diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4fd58016..a18fa662 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,10 +3,10 @@ name: build on: push: branches: - - master + - main pull_request: branches: - - master + - main release: types: - released @@ -14,28 +14,23 @@ on: env: # See https://github.com/nodejs/release#release-schedule - # Node.js v16 EOL = 2023-09-11. v21 EOL = 2024-06-01. - NODE_BUILD_CMD: npx --no-install prebuild -r node -t 18.0.0 -t 20.0.0 -t 22.0.0 -t 23.0.0 --include-regex 'better_sqlite3.node$' - # Merge with NODE_BUILD_CMD when Node.js v18 is EOL - NO_V18_NODE_BUILD_CMD: npx --no-install prebuild -r node -t 20.0.0 -t 22.0.0 -t 23.0.0 --include-regex 'better_sqlite3.node$' - # See https://www.electronjs.org/docs/latest/tutorial/electron-timelines#version-support-policy - # Electron v25 EOL = 2023-12-05. v26 EOL = 2024-02-20. v27 EOL = 2024-04-16. v28 EOL = 2024-06-11. v29 EOL = 2024-08-20. - ELECTRON_BUILD_CMD: npx --no-install prebuild -r electron -t 26.0.0 -t 27.0.0 -t 28.0.0 -t 29.0.0 -t 30.0.0 -t 31.0.0 -t 32.0.0 -t 33.0.0 --include-regex 'better_sqlite3.node$' + # Node.js v20 EOL = 2026-04-30. v22 EOL = 2027-04-30. v23 EOL = 2025-06-01. v24 EOL = 2028-04-30. + NODE_BUILD_CMD: npx --no-install prebuild -r node -t 20.0.0 -t 22.0.0 -t 23.0.0 -t 24.0.0 --include-regex '_sqlite3' jobs: test: strategy: matrix: os: - - ubuntu-20.04 + - ubuntu-22.04 - macos-13 - macos-14 - - windows-2019 + - windows-2022 node: - - 18 - 20 - 22 - 23 + - 24 name: Testing Node ${{ matrix.node }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: @@ -52,32 +47,27 @@ jobs: - if: ${{ startsWith(matrix.os, 'ubuntu') }} run: | sudo apt update - sudo apt install gcc-10 g++-10 -y + sudo apt install -y gcc-10 g++-10 libreadline-dev libncurses5-dev sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 - run: npm install --ignore-scripts - run: npm run build-debug - run: npm test - - name: Test SpatiaLite extension - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt update - sudo apt install libsqlite3-mod-spatialite -y - node -e "require('./lib/index.js')(':memory:').loadExtension('mod_spatialite').exec('SELECT InitSpatialMetaData();')" publish: if: ${{ github.event_name == 'release' }} name: Publishing to NPM - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: - prebuild - prebuild-alpine - prebuild-alpine-arm + - prebuild-linux-x64 - prebuild-linux-arm steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 registry-url: https://registry.npmjs.org - run: npm publish env: @@ -89,10 +79,9 @@ jobs: fail-fast: false matrix: os: - - ubuntu-20.04 - macos-13 - macos-14 - - windows-2019 + - windows-2022 name: Prebuild on ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: test @@ -100,39 +89,38 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 - if: ${{ startsWith(matrix.os, 'windows') }} run: pip.exe install setuptools - if: ${{ startsWith(matrix.os, 'macos') }} run: brew install python-setuptools - - if: ${{ !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos') }} - run: python3 -m pip install setuptools - - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt update - sudo apt install gcc-10 g++-10 -y - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 - run: npm install --ignore-scripts - run: ${{ env.NODE_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }} - - run: ${{ env.ELECTRON_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }} - - if: matrix.os == 'windows-2019' + - if: matrix.os == 'windows-2022' run: | ${{ env.NODE_BUILD_CMD }} --arch ia32 -u ${{ secrets.GITHUB_TOKEN }} - ${{ env.NO_V18_NODE_BUILD_CMD }} --arch arm64 -u ${{ secrets.GITHUB_TOKEN }} - ${{ env.ELECTRON_BUILD_CMD }} --arch ia32 -u ${{ secrets.GITHUB_TOKEN }} - ${{ env.ELECTRON_BUILD_CMD }} --arch arm64 -u ${{ secrets.GITHUB_TOKEN }} + ${{ env.NODE_BUILD_CMD }} --arch arm64 -u ${{ secrets.GITHUB_TOKEN }} + + prebuild-linux-x64: + if: ${{ github.event_name == 'release' }} + name: Prebuild on Linux x64 + runs-on: ubuntu-latest + container: node:20-bullseye + needs: test + steps: + - uses: actions/checkout@v4 + - run: npm install --ignore-scripts + - run: ${{ env.NODE_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }} prebuild-alpine: if: ${{ github.event_name == 'release' }} - strategy: - fail-fast: false name: Prebuild on alpine runs-on: ubuntu-latest - container: node:18-alpine + container: node:20-alpine needs: test steps: - uses: actions/checkout@v4 - - run: apk add build-base git python3 py3-setuptools --update-cache + - run: apk add build-base git python3 py3-setuptools libstdc++ readline-dev ncurses-dev --update-cache - run: npm install --ignore-scripts - run: ${{ env.NODE_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }} @@ -151,8 +139,8 @@ jobs: - uses: actions/checkout@v4 - uses: docker/setup-qemu-action@v3 - run: | - docker run --rm -v $(pwd):/tmp/project --entrypoint /bin/sh --platform linux/${{ matrix.arch }} node:18-alpine -c "\ - apk add build-base git python3 py3-setuptools --update-cache && \ + docker run --rm -v $(pwd):/tmp/project --entrypoint /bin/sh --platform linux/${{ matrix.arch }} node:20-alpine -c "\ + apk add build-base git python3 py3-setuptools libstdc++ readline-dev ncurses-dev --update-cache && \ cd /tmp/project && \ npm install --ignore-scripts && \ ${{ env.NODE_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }}" @@ -172,7 +160,7 @@ jobs: - uses: actions/checkout@v4 - uses: docker/setup-qemu-action@v3 - run: | - docker run --rm -v $(pwd):/tmp/project --entrypoint /bin/sh --platform linux/${{ matrix.arch }} node:18-bullseye -c "\ + docker run --rm -v $(pwd):/tmp/project --entrypoint /bin/sh --platform linux/${{ matrix.arch }} node:20-bullseye -c "\ cd /tmp/project && \ npm install --ignore-scripts && \ ${{ env.NODE_BUILD_CMD }} -u ${{ secrets.GITHUB_TOKEN }}" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..e1c56c7d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +hello@roci.dev. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/README.md b/README.md index b324c532..a0b54012 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,25 @@ -# better-sqlite3 [![Build Status](https://github.com/JoshuaWise/better-sqlite3/actions/workflows/build.yml/badge.svg)](https://github.com/JoshuaWise/better-sqlite3/actions/workflows/build.yml?query=branch%3Amaster) +This is a fork of [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) with changes needed by [Zero](https://zerosync.dev/). -The fastest and simplest library for SQLite3 in Node.js. +So far the changes are: -- Full transaction support -- High performance, efficiency, and safety -- Easy-to-use synchronous API *(better concurrency than an asynchronous API... yes, you read that correctly)* -- Support for user-defined functions, aggregates, virtual tables, and extensions -- 64-bit integers *(invisible until you need them)* -- Worker thread support *(for large/slow queries)* +* Build the [bedrock](https://sqlite.org/src/timeline?r=bedrock) branch of SQLite to enable [`begin concurrent`](https://www.sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md). +* Create a shell too, so we can debug db files created -## Help this project stay strong! 💪 +Other changes will be likely be made over time. -`better-sqlite3` is used by thousands of developers and engineers on a daily basis. Long nights and weekends were spent keeping this project strong and dependable, with no ask for compensation or funding, until now. If your company uses `better-sqlite3`, ask your manager to consider supporting the project: +## Environment Variables -- [Become a GitHub sponsor](https://github.com/sponsors/JoshuaWise) -- [Become a backer on Patreon](https://www.patreon.com/joshuawise) -- [Make a one-time donation on PayPal](https://www.paypal.me/joshuathomaswise) +### `ZERO_SKIP_SQLITE3_BUILD` -## How other libraries compare +Set `ZERO_SKIP_SQLITE3_BUILD=true` to skip the native module build during installation. This is useful in Docker builds and CI/CD pipelines where: -| |select 1 row  `get()` |select 100 rows   `all()`  |select 100 rows `iterate()` 1-by-1|insert 1 row `run()`|insert 100 rows in a transaction| -|---|---|---|---|---|---| -|better-sqlite3|1x|1x|1x|1x|1x| -|[sqlite](https://www.npmjs.com/package/sqlite) and [sqlite3](https://www.npmjs.com/package/sqlite3)|11.7x slower|2.9x slower|24.4x slower|2.8x slower|15.6x slower| - -> You can verify these results by [running the benchmark yourself](./docs/benchmark.md). - -## Installation +- The native module is pre-built in a different layer +- Build tools are not available in the runtime environment +- You want to reduce installation time in monorepo setups +Example: ```bash -npm install better-sqlite3 -``` - -> You must be using Node.js v14.21.1 or above. Prebuilt binaries are available for [LTS versions](https://nodejs.org/en/about/releases/). If you have trouble installing, check the [troubleshooting guide](./docs/troubleshooting.md). - -## Usage - -```js -const db = require('better-sqlite3')('foobar.db', options); - -const row = db.prepare('SELECT * FROM users WHERE id = ?').get(userId); -console.log(row.firstName, row.lastName, row.email); -``` - -Though not required, [it is generally important to set the WAL pragma for performance reasons](https://github.com/WiseLibs/better-sqlite3/blob/master/docs/performance.md). - -```js -db.pragma('journal_mode = WAL'); +ZERO_SKIP_SQLITE3_BUILD=true npm install ``` -##### In ES6 module notation: - -```js -import Database from 'better-sqlite3'; -const db = new Database('foobar.db', options); -db.pragma('journal_mode = WAL'); -``` - -## Why should I use this instead of [node-sqlite3](https://github.com/mapbox/node-sqlite3)? - -- `node-sqlite3` uses asynchronous APIs for tasks that are either CPU-bound or serialized. That's not only bad design, but it wastes tons of resources. It also causes [mutex thrashing](https://en.wikipedia.org/wiki/Resource_contention) which has devastating effects on performance. -- `node-sqlite3` exposes low-level (C language) memory management functions. `better-sqlite3` does it the JavaScript way, allowing the garbage collector to worry about memory management. -- `better-sqlite3` is simpler to use, and it provides nice utilities for some operations that are very difficult or impossible in `node-sqlite3`. -- `better-sqlite3` is much faster than `node-sqlite3` in most cases, and just as fast in all other cases. - -#### When is this library not appropriate? - -In most cases, if you're attempting something that cannot be reasonably accomplished with `better-sqlite3`, it probably cannot be reasonably accomplished with SQLite3 in general. For example, if you're executing queries that take one second to complete, and you expect to have many concurrent users executing those queries, no amount of asynchronicity will save you from SQLite3's serialized nature. Fortunately, SQLite3 is very *very* fast. With proper indexing, we've been able to achieve upward of 2000 queries per second with 5-way-joins in a 60 GB database, where each query was handling 5–50 kilobytes of real data. - -If you have a performance problem, the most likely causes are inefficient queries, improper indexing, or a lack of [WAL mode](./docs/performance.md)—not `better-sqlite3` itself. However, there are some cases where `better-sqlite3` could be inappropriate: - -- If you expect a high volume of concurrent reads each returning many megabytes of data (i.e., videos) -- If you expect a high volume of concurrent writes (i.e., a social media site) -- If your database's size is near the terabyte range - -For these situations, you should probably use a full-fledged RDBMS such as [PostgreSQL](https://www.postgresql.org/). - -# Documentation - -- [API documentation](./docs/api.md) -- [Performance](./docs/performance.md) (also see [benchmark results](./docs/benchmark.md)) -- [64-bit integer support](./docs/integer.md) -- [Worker thread support](./docs/threads.md) -- [Unsafe mode (advanced)](./docs/unsafe.md) -- [SQLite3 compilation (advanced)](./docs/compilation.md) -- [Contribution rules](./docs/contribution.md) -- [Code of conduct](./docs/conduct.md) - -# License - -[MIT](./LICENSE) +**Warning:** When using this flag, ensure the native module is available through other means (pre-built, copied from another stage, etc.) or the package will not function correctly. diff --git a/binding.gyp b/binding.gyp index f517a8d6..8ca67458 100644 --- a/binding.gyp +++ b/binding.gyp @@ -4,6 +4,9 @@ { 'includes': ['deps/common.gypi'], + 'variables': { + 'is_alpine%': ' "$TEMP/source.zip" || exit 1 +curl -#f "https://sqlite.org/src/zip/$CHECKIN/SQLite-$CHECKIN.zip" > "$TEMP/source.zip" || exit 1 echo "extracting source..." unzip "$TEMP/source.zip" -d "$TEMP" > /dev/null || exit 1 -cd "$TEMP/sqlite-src-$VERSION" || exit 1 +cd "$TEMP/SQLite-$CHECKIN" || exit 1 echo "configuring amalgamation..." sh configure > /dev/null || exit 1 @@ -86,7 +86,7 @@ echo "building amalgamation..." make sqlite3.c > /dev/null || exit 1 echo "copying amalgamation..." -cp sqlite3.c sqlite3.h sqlite3ext.h "$OUTPUT/" || exit 1 +cp sqlite3.c sqlite3.h sqlite3ext.h shell.c "$OUTPUT/" || exit 1 echo "applying patches..." cd "$DEPS" || exit 1 diff --git a/deps/sqlite3.gyp b/deps/sqlite3.gyp index d5307b14..9bbf2bba 100755 --- a/deps/sqlite3.gyp +++ b/deps/sqlite3.gyp @@ -6,6 +6,9 @@ { 'includes': ['common.gypi'], + 'variables': { + 'is_alpine%': ' +#include +#include +#include +#include +#include "sqlite3.h" +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; +typedef unsigned char u8; +#if SQLITE_USER_AUTHENTICATION +# include "sqlite3userauth.h" +#endif +#include +#include + +#if !defined(_WIN32) && !defined(WIN32) +# include +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) +# include +# endif +#endif +#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__) +# include +# include +# define GETPID getpid +# if defined(__MINGW32__) +# define DIRENT dirent +# ifndef S_ISLNK +# define S_ISLNK(mode) (0) +# endif +# endif +#else +# define GETPID (int)GetCurrentProcessId +#endif +#include +#include + +#if HAVE_READLINE +# include +# include +#endif + +#if HAVE_EDITLINE +# include +#endif + +#if HAVE_EDITLINE || HAVE_READLINE + +# define shell_add_history(X) add_history(X) +# define shell_read_history(X) read_history(X) +# define shell_write_history(X) write_history(X) +# define shell_stifle_history(X) stifle_history(X) +# define shell_readline(X) readline(X) + +#elif HAVE_LINENOISE + +# include "linenoise.h" +# define shell_add_history(X) linenoiseHistoryAdd(X) +# define shell_read_history(X) linenoiseHistoryLoad(X) +# define shell_write_history(X) linenoiseHistorySave(X) +# define shell_stifle_history(X) linenoiseHistorySetMaxLen(X) +# define shell_readline(X) linenoise(X) + +#else + +# define shell_read_history(X) +# define shell_write_history(X) +# define shell_stifle_history(X) + +# define SHELL_USE_LOCAL_GETLINE 1 +#endif + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +#if defined(_WIN32) || defined(WIN32) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef pclose +# define pclose _pclose +# endif +#else + /* Make sure isatty() has a prototype. */ + extern int isatty(int); + +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) + /* popen and pclose are not C89 functions and so are + ** sometimes omitted from the header */ + extern FILE *popen(const char*,const char*); + extern int pclose(FILE*); +# else +# define SQLITE_OMIT_POPEN 1 +# endif +#endif + +#if defined(_WIN32_WCE) +/* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() + * thus we always assume that we have a console. That can be + * overridden with the -batch command line option. + */ +#define isatty(x) 1 +#endif + +/* ctype macros that work with signed characters */ +#define IsSpace(X) isspace((unsigned char)X) +#define IsDigit(X) isdigit((unsigned char)X) +#define ToLower(X) (char)tolower((unsigned char)X) + +#if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include +#endif +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include + +/* string conversion routines only needed on Win32 */ +extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); +extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); +#endif + +/************************* Begin ../ext/misc/sqlite3_stdio.h ******************/ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This header file contains definitions of interfaces that provide +** cross-platform I/O for UTF-8 content. +** +** On most platforms, the interfaces definitions in this file are +** just #defines. For example sqlite3_fopen() is a macro that resolves +** to the standard fopen() in the C-library. +** +** But Windows does not have a standard C-library, at least not one that +** can handle UTF-8. So for windows build, the interfaces resolve to new +** C-language routines contained in the separate sqlite3_stdio.c source file. +** +** So on all non-Windows platforms, simply #include this header file and +** use the interfaces defined herein. Then to run your application on Windows, +** also link in the accompanying sqlite3_stdio.c source file when compiling +** to get compatible interfaces. +*/ +#ifndef _SQLITE3_STDIO_H_ +#define _SQLITE3_STDIO_H_ 1 +#ifdef _WIN32 +/**** Definitions For Windows ****/ +#include +#include + +FILE *sqlite3_fopen(const char *zFilename, const char *zMode); +FILE *sqlite3_popen(const char *zCommand, const char *type); +char *sqlite3_fgets(char *s, int size, FILE *stream); +int sqlite3_fputs(const char *s, FILE *stream); +int sqlite3_fprintf(FILE *stream, const char *format, ...); +void sqlite3_fsetmode(FILE *stream, int mode); + + +#else +/**** Definitions For All Other Platforms ****/ +#include +#define sqlite3_fopen fopen +#define sqlite3_popen popen +#define sqlite3_fgets fgets +#define sqlite3_fputs fputs +#define sqlite3_fprintf fprintf +#define sqlite3_fsetmode(F,X) /*no-op*/ + +#endif +#endif /* _SQLITE3_STDIO_H_ */ + +/************************* End ../ext/misc/sqlite3_stdio.h ********************/ +/************************* Begin ../ext/misc/sqlite3_stdio.c ******************/ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Implementation of standard I/O interfaces for UTF-8 that are missing +** on Windows. +*/ +#ifdef _WIN32 /* This file is a no-op on all platforms except Windows */ +#ifndef _SQLITE3_STDIO_H_ +/* #include "sqlite3_stdio.h" */ +#endif +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +/* #include "sqlite3.h" */ +#include +#include +#include +#include + +/* +** If the SQLITE_U8TEXT_ONLY option is defined, then use O_U8TEXT +** when appropriate on all output. (Sometimes use O_BINARY when +** rendering ASCII text in cases where NL-to-CRLF expansion would +** not be correct.) +** +** If the SQLITE_U8TEXT_STDIO option is defined, then use O_U8TEXT +** when appropriate when writing to stdout or stderr. Use O_BINARY +** or O_TEXT (depending on things like the .mode and the .crlf setting +** in the CLI, or other context clues in other applications) for all +** other output channels. +** +** The default behavior, if neither of the above is defined is to +** use O_U8TEXT when writing to the Windows console (or anything +** else for which _isatty() returns true) and to use O_BINARY or O_TEXT +** for all other output channels. +*/ +#if defined(SQLITE_U8TEXT_ONLY) +# define UseWtextForOutput(fd) 1 +# define UseWtextForInput(fd) 1 +# define IsConsole(fd) _isatty(_fileno(fd)) +#elif defined(SQLITE_U8TEXT_STDIO) +# define UseWtextForOutput(fd) ((fd)==stdout || (fd)==stderr) +# define UseWtextForInput(fd) ((fd)==stdin) +# define IsConsole(fd) _isatty(_fileno(fd)) +#else +# define UseWtextForOutput(fd) _isatty(_fileno(fd)) +# define UseWtextForInput(fd) _isatty(_fileno(fd)) +# define IsConsole(fd) 1 +#endif + +/* +** Global variables determine if simulated O_BINARY mode is to be +** used for stdout or other, respectively. Simulated O_BINARY mode +** means the mode is usually O_BINARY, but switches to O_U8TEXT for +** unicode characters U+0080 or greater (any character that has a +** multi-byte representation in UTF-8). This is the only way we +** have found to render Unicode characters on a Windows console while +** at the same time avoiding undesirable \n to \r\n translation. +*/ +static int simBinaryStdout = 0; +static int simBinaryOther = 0; + + +/* +** Determine if simulated binary mode should be used for output to fd +*/ +static int UseBinaryWText(FILE *fd){ + if( fd==stdout || fd==stderr ){ + return simBinaryStdout; + }else{ + return simBinaryOther; + } +} + + +/* +** Work-alike for the fopen() routine from the standard C library. +*/ +FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zFilename); + sz2 = (int)strlen(zMode); + b1 = malloc( (sz1+1)*sizeof(b1[0]) ); + b2 = malloc( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wfopen(b1, b2); + } + free(b1); + free(b2); + simBinaryOther = 0; + return fp; +} + + +/* +** Work-alike for the popen() routine from the standard C library. +*/ +FILE *sqlite3_popen(const char *zCommand, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zCommand); + sz2 = (int)strlen(zMode); + b1 = malloc( (sz1+1)*sizeof(b1[0]) ); + b2 = malloc( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wpopen(b1, b2); + } + free(b1); + free(b2); + return fp; +} + +/* +** Work-alike for fgets() from the standard C library. +*/ +char *sqlite3_fgets(char *buf, int sz, FILE *in){ + if( UseWtextForInput(in) ){ + /* When reading from the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode to read UTF-16 characters, then translate + ** that into UTF-8. Otherwise, non-ASCII characters all get translated + ** into '?'. + */ + wchar_t *b1 = malloc( sz*sizeof(wchar_t) ); + if( b1==0 ) return 0; + _setmode(_fileno(in), IsConsole(in) ? _O_WTEXT : _O_U8TEXT); + if( fgetws(b1, sz/4, in)==0 ){ + sqlite3_free(b1); + return 0; + } + WideCharToMultiByte(CP_UTF8, 0, b1, -1, buf, sz, 0, 0); + sqlite3_free(b1); + return buf; + }else{ + /* Reading from a file or other input source, just read bytes without + ** any translation. */ + return fgets(buf, sz, in); + } +} + +/* +** Send ASCII text as O_BINARY. But for Unicode characters U+0080 and +** greater, switch to O_U8TEXT. +*/ +static void piecemealOutput(wchar_t *b1, int sz, FILE *out){ + int i; + wchar_t c; + while( sz>0 ){ + for(i=0; i=0x80; i++){} + if( i>0 ){ + c = b1[i]; + b1[i] = 0; + fflush(out); + _setmode(_fileno(out), _O_U8TEXT); + fputws(b1, out); + fflush(out); + b1 += i; + b1[0] = c; + sz -= i; + }else{ + fflush(out); + _setmode(_fileno(out), _O_TEXT); + _setmode(_fileno(out), _O_BINARY); + fwrite(&b1[0], 1, 1, out); + for(i=1; iiVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include +#include + +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. +*/ +static void endTimer(FILE *out){ + if( enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", + (iEnd - iBegin)*0.001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { +#if !SQLITE_OS_WINRT + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } +#endif + } + return 0; +} + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer && getProcessTimesAddr ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +/* +** Print the timing results. +*/ +static void endTimer(FILE *out){ + if( enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER +#define END_TIMER(X) /*no-op*/ +#define HAS_TIMER 0 +#endif + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* +** If the following flag is set, then command execution stops +** at an error if we are not interactive. +*/ +static int bail_on_error = 0; + +/* +** Treat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; + +/* +** On Windows systems we need to know if standard output is a console +** in order to show that UTF-16 translation is done in the sign-on +** banner. The following variable is true if it is the console. +*/ +static int stdout_is_console = 1; + +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *globalDb = 0; + +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; + +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; + +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +#define PROMPT_LEN_MAX 20 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; + +/* This is variant of the standard-library strncpy() routine with the +** one change that the destination string is always zero-terminated, even +** if there is no zero-terminator in the first n-1 characters of the source +** string. +*/ +static char *shell_strncpy(char *dest, const char *src, size_t n){ + size_t i; + for(i=0; iinParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} + +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; + } +} + +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + size_t ncp = strlen(continuePrompt); + size_t ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); + }else if( dynPrompt.inParenLevel<0 ){ + shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); + }else{ + shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; +} +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ + +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + eputz("Error: out of memory\n"); + exit(1); +} + +/* Check a pointer to see if it is NULL. If it is NULL, exit with an +** out-of-memory error. +*/ +static void shell_check_oom(const void *p){ + if( p==0 ) shell_out_of_memory(); +} + +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_fprintf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif + +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int cli_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<=0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; + while( iFirst c ){ + iLast = iMid - 1; + }else{ + return aUWidth[iMid].w; + } + } + if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; + return aUWidth[iLast].w; +} + +/* +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. +*/ +static int decodeUtf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; + } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; + } + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[4] & 0x3f); + return 4; + } + *pU = 0; + return 1; +} + + +#if 0 /* NOT USED */ +/* +** Return the width, in display columns, of a UTF-8 string. +** +** Each normal character counts as 1. Zero-width characters count +** as zero, and double-width characters count as 2. +*/ +int cli_wcswidth(const char *z){ + const unsigned char *a = (const unsigned char*)z; + int n = 0; + int i = 0; + unsigned char c; + while( (c = a[i])!=0 ){ + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&a[i], &u); + i += len; + n += cli_wcwidth(u); + }else if( c>=' ' ){ + n++; + i++; + }else{ + i++; + } + } + return n; +} +#endif + +/* +** Output string zUtf to stdout as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. +** +** Take into account zero-width and double-width Unicode characters. +** In other words, a zero-width character does not count toward the +** the w limit. A double-width character counts as two. +*/ +static void utf8_width_print(FILE *out, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + unsigned char c; + int i = 0; + int n = 0; + int aw = w<0 ? -w : w; + if( zUtf==0 ) zUtf = ""; + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = decodeUtf8(a+i, &u); + int x = cli_wcwidth(u); + if( x+n>aw ){ + break; + } + i += len; + n += x; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; + } + } + if( n>=aw ){ + sqlite3_fprintf(out, "%.*s", i, zUtf); + }else if( w<0 ){ + sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); + }else{ + sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); + } +} + + +/* +** Determines if a string is a number of not. +*/ +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !IsDigit(*z) ){ + return 0; + } + z++; + if( realnum ) *realnum = 0; + while( IsDigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + return *z==0; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0xc0&*(z++))!=0x80 ) n++; + } + return n; +} + +/* +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. +*/ +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; + } + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); + }else{ + return 0; + } +#endif +#undef STAT_CHR_SRC +} + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. +*/ +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; + + while( 1 ){ + if( n+100>nLine ){ + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + shell_check_oom(zLine); + } + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } + } + return zLine; +} + +/* +** Retrieve a single line of input text. +** +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. +** +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. +** +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. +*/ +#ifndef SQLITE_SHELL_FIDDLE +static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + if( in!=0 ){ + zResult = local_getline(zPrior, in); + }else{ + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + sputz(stdout, zPrompt); + fflush(stdout); + do{ + zResult = local_getline(zPrior, stdin); + zPrior = 0; + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); + } + if( zResult && *zResult ) shell_add_history(zResult); +#endif + } + return zResult; +} +#endif /* !SQLITE_SHELL_FIDDLE */ + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; iz); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, const char *zAppend, char quote){ + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; iz==0 || p->n+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + shell_check_oom(p->z); + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( zName==0 ) return '"'; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + +/* +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +*/ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; + + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + shell_check_oom(zSql); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + if( zCol==0 ) zCol = ""; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.z = 0; + } + return s.z; +} + +/* +** SQL function: strtod(X) +** +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. +*/ +static void shellStrtod( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + char *z = (char*)sqlite3_value_text(apVal[0]); + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} + +/* +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. +*/ +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +} + + +/* +** SQL function: shell_module_schema(X) +** +** Return a fake schema for the table-valued function or eponymous virtual +** table X. +*/ +static void shellModuleSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zName; + char *zFake; + UNUSED_PARAMETER(nVal); + zName = (const char*)sqlite3_value_text(apVal[0]); + zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} + +/* +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. +*/ +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; i +#include +#include +#include +#include +#include +#include + +/* +** We may need several defines that should have been in "sys/stat.h". +*/ + +#ifndef S_ISREG +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif + +#ifndef S_ISDIR +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif + +#ifndef S_ISLNK +#define S_ISLNK(mode) (0) +#endif + +/* +** We may need to provide the "mode_t" type. +*/ + +#ifndef MODE_T_DEFINED + #define MODE_T_DEFINED + typedef unsigned short mode_t; +#endif + +/* +** We may need to provide the "ino_t" type. +*/ + +#ifndef INO_T_DEFINED + #define INO_T_DEFINED + typedef unsigned short ino_t; +#endif + +/* +** We need to define "NAME_MAX" if it was not present in "limits.h". +*/ + +#ifndef NAME_MAX +# ifdef FILENAME_MAX +# define NAME_MAX (FILENAME_MAX) +# else +# define NAME_MAX (260) +# endif +#endif + +/* +** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". +*/ + +#ifndef NULL_INTPTR_T +# define NULL_INTPTR_T ((intptr_t)(0)) +#endif + +#ifndef BAD_INTPTR_T +# define BAD_INTPTR_T ((intptr_t)(-1)) +#endif + +/* +** We need to provide the necessary structures and related types. +*/ + +#ifndef DIRENT_DEFINED +#define DIRENT_DEFINED +typedef struct DIRENT DIRENT; +typedef DIRENT *LPDIRENT; +struct DIRENT { + ino_t d_ino; /* Sequence number, do not use. */ + unsigned d_attributes; /* Win32 file attributes. */ + char d_name[NAME_MAX + 1]; /* Name within the directory. */ +}; +#endif + +#ifndef DIR_DEFINED +#define DIR_DEFINED +typedef struct DIR DIR; +typedef DIR *LPDIR; +struct DIR { + intptr_t d_handle; /* Value returned by "_findfirst". */ + DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ + DIRENT d_next; /* DIRENT constructed based on "_findnext". */ +}; +#endif + +/* +** Provide a macro, for use by the implementation, to determine if a +** particular directory entry should be skipped over when searching for +** the next directory entry that should be returned by the readdir() or +** readdir_r() functions. +*/ + +#ifndef is_filtered +# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) +#endif + +/* +** Provide the function prototype for the POSIX compatible getenv() +** function. This function is not thread-safe. +*/ + +extern const char *windirent_getenv(const char *name); + +/* +** Finally, we can provide the function prototypes for the opendir(), +** readdir(), readdir_r(), and closedir() POSIX functions. +*/ + +extern LPDIR opendir(const char *dirname); +extern LPDIRENT readdir(LPDIR dirp); +extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); +extern INT closedir(LPDIR dirp); + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.h ********************/ +/************************* Begin test_windirent.c ******************/ +/* +** 2015 November 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement most of the opendir() family of +** POSIX functions on Win32 using the MSVCRT. +*/ + +#if defined(_WIN32) && defined(_MSC_VER) +/* #include "test_windirent.h" */ + +/* +** Implementation of the POSIX getenv() function using the Win32 API. +** This function is not thread-safe. +*/ +const char *windirent_getenv( + const char *name +){ + static char value[32768]; /* Maximum length, per MSDN */ + DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ + DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ + + memset(value, 0, sizeof(value)); + dwRet = GetEnvironmentVariableA(name, value, dwSize); + if( dwRet==0 || dwRet>dwSize ){ + /* + ** The function call to GetEnvironmentVariableA() failed -OR- + ** the buffer is not large enough. Either way, return NULL. + */ + return 0; + }else{ + /* + ** The function call to GetEnvironmentVariableA() succeeded + ** -AND- the buffer contains the entire value. + */ + return value; + } +} + +/* +** Implementation of the POSIX opendir() function using the MSVCRT. +*/ +LPDIR opendir( + const char *dirname +){ + struct _finddata_t data; + LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); + SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); + + if( dirp==NULL ) return NULL; + memset(dirp, 0, sizeof(DIR)); + + /* TODO: Remove this if Unix-style root paths are not used. */ + if( sqlite3_stricmp(dirname, "/")==0 ){ + dirname = windirent_getenv("SystemDrive"); + } + + memset(&data, 0, sizeof(struct _finddata_t)); + _snprintf(data.name, namesize, "%s\\*", dirname); + dirp->d_handle = _findfirst(data.name, &data); + + if( dirp->d_handle==BAD_INTPTR_T ){ + closedir(dirp); + return NULL; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ){ +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + closedir(dirp); + return NULL; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + } + + dirp->d_first.d_attributes = data.attrib; + strncpy(dirp->d_first.d_name, data.name, NAME_MAX); + dirp->d_first.d_name[NAME_MAX] = '\0'; + + return dirp; +} + +/* +** Implementation of the POSIX readdir() function using the MSVCRT. +*/ +LPDIRENT readdir( + LPDIR dirp +){ + struct _finddata_t data; + + if( dirp==NULL ) return NULL; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + return &dirp->d_first; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + dirp->d_next.d_ino++; + dirp->d_next.d_attributes = data.attrib; + strncpy(dirp->d_next.d_name, data.name, NAME_MAX); + dirp->d_next.d_name[NAME_MAX] = '\0'; + + return &dirp->d_next; +} + +/* +** Implementation of the POSIX readdir_r() function using the MSVCRT. +*/ +INT readdir_r( + LPDIR dirp, + LPDIRENT entry, + LPDIRENT *result +){ + struct _finddata_t data; + + if( dirp==NULL ) return EBADF; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + entry->d_ino = dirp->d_first.d_ino; + entry->d_attributes = dirp->d_first.d_attributes; + strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + *result = NULL; + return ENOENT; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + entry->d_ino = (ino_t)-1; /* not available */ + entry->d_attributes = data.attrib; + strncpy(entry->d_name, data.name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; +} + +/* +** Implementation of the POSIX closedir() function using the MSVCRT. +*/ +INT closedir( + LPDIR dirp +){ + INT result = 0; + + if( dirp==NULL ) return EINVAL; + + if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ + result = _findclose(dirp->d_handle); + } + + sqlite3_free(dirp); + return result; +} + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.c ********************/ +#define dirent DIRENT +#endif +/************************* Begin ../ext/misc/memtrace.c ******************/ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/memtrace.c ********************/ +/************************* Begin ../ext/misc/pcachetrace.c ******************/ +/* +** 2023-06-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 +** mechanism to add a tracing layer on top of pluggable page cache of +** SQLite. If this extension is registered prior to sqlite3_initialize(), +** it will cause all page cache activities to be logged on standard output, +** or to some other FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --pcachetrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/pcachetrace.c ********************/ +/************************* Begin ../ext/misc/shathree.c ******************/ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_agg(Y,SIZE) +** sha3_query(Z,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. If inputs X is text, the UTF-8 rendering of that text is +** used to compute the hash. If X is a BLOB, then the binary data of the +** blob is used to compute the hash. If X is an integer or real number, +** then that number if converted into UTF-8 text and the hash is computed +** over the text. +** +** The sha3_agg(Y) function computes the SHA3 hash of all Y inputs. Since +** order is important for the hash, it is recommended that the Y expression +** by followed by an ORDER BY clause to guarantee that the inputs occur +** in the desired order. +** +** The sha3_query(Y) function evaluates all queries in the SQL statements of Y +** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. +** +** Because the sha3_agg() and sha3_query() functions compute a hash over +** multiple values, the values are encode to use include type information. +** +** In sha3_agg(), the sequence of bytes that gets hashed for each input +** Y depends on the datatype of Y: +** +** typeof(Y)='null' A single "N" is hashed. (One byte) +** +** typeof(Y)='integer' The data hash is the character "I" followed +** by an 8-byte big-endian binary of the +** 64-bit signed integer. (Nine bytes total.) +** +** typeof(Y)='real' The character "F" followed by an 8-byte +** big-ending binary of the double. (Nine +** bytes total.) +** +** typeof(Y)='text' The hash is over prefix "Tnnn:" followed +** by the UTF8 encoding of the text. The "nnn" +** in the prefix is the minimum-length decimal +** representation of the octet_length of the text. +** Notice the ":" at the end of the prefix, which +** is needed to separate the prefix from the +** content in cases where the content starts +** with a digit. +** +** typeof(Y)='blob' The hash is taken over prefix "Bnnn:" followed +** by the binary content of the blob. The "nnn" +** in the prefix is the mimimum-length decimal +** representation of the byte-length of the blob. +** +** According to the rules above, all of the following SELECT statements +** should return TRUE: +** +** SELECT sha3(1) = sha3('1'); +** +** SELECT sha3('hello') = sha3(x'68656c6c6f'); +** +** WITH a(x) AS (VALUES('xyzzy')) +** SELECT sha3_agg(x) = sha3('T5:xyzzy') FROM a; +** +** WITH a(x) AS (VALUES(x'010203')) +** SELECT sha3_agg(x) = sha3(x'42333a010203') FROM a; +** +** WITH a(x) AS (VALUES(0x123456)) +** SELECT sha3_agg(x) = sha3(x'490000000000123456') FROM a; +** +** WITH a(x) AS (VALUES(100.015625)) +** SELECT sha3_agg(x) = sha3(x'464059010000000000') FROM a; +** +** WITH a(x) AS (VALUES(NULL)) +** SELECT sha3_agg(x) = sha3('N') FROM a; +** +** +** In sha3_query(), individual column values are encoded as with +** sha3_agg(), but with the addition that a single "R" character is +** inserted at the start of each row. +** +** Note that sha3_agg() hashes rows for which Y is NULL. Add a FILTER +** clause if NULL rows should be excluded: +** +** SELECT sha3_agg(x ORDER BY rowid) FILTER(WHERE x NOT NULL) FROM t1; +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_uint64 u64; */ +#endif /* SQLITE_AMALGAMATION */ + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + c0 = a00^a20^a40^a10^a30; + c1 = a11^a31^a01^a21^a41; + c2 = a22^a42^a12^a32^a02; + c3 = a33^a03^a23^a43^a13; + c4 = a44^a14^a34^a04^a24; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a31^d1), 44); + b2 = ROL64((a12^d2), 43); + b3 = ROL64((a43^d3), 21); + b4 = ROL64((a24^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+1]; + a31 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a40^d0), 3); + b3 = ROL64((a21^d1), 45); + b4 = ROL64((a02^d2), 61); + b0 = ROL64((a33^d3), 28); + b1 = ROL64((a14^d4), 20); + a40 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a30^d0), 18); + b0 = ROL64((a11^d1), 1); + b1 = ROL64((a42^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a04^d4), 8); + a30 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a20^d0), 36); + b2 = ROL64((a01^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a13^d3), 56); + b0 = ROL64((a44^d4), 27); + a20 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a10^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a22^d2), 62); + b1 = ROL64((a03^d3), 55); + b2 = ROL64((a34^d4), 39); + a10 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + c0 = a00^a40^a30^a20^a10; + c1 = a31^a21^a11^a01^a41; + c2 = a12^a02^a42^a32^a22; + c3 = a43^a33^a23^a13^a03; + c4 = a24^a14^a04^a44^a34; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a21^d1), 44); + b2 = ROL64((a42^d2), 43); + b3 = ROL64((a13^d3), 21); + b4 = ROL64((a34^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+2]; + a21 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a30^d0), 3); + b3 = ROL64((a01^d1), 45); + b4 = ROL64((a22^d2), 61); + b0 = ROL64((a43^d3), 28); + b1 = ROL64((a14^d4), 20); + a30 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a10^d0), 18); + b0 = ROL64((a31^d1), 1); + b1 = ROL64((a02^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a44^d4), 8); + a10 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a40^d0), 36); + b2 = ROL64((a11^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a03^d3), 56); + b0 = ROL64((a24^d4), 27); + a40 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a20^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a12^d2), 62); + b1 = ROL64((a33^d3), 55); + b2 = ROL64((a04^d4), 39); + a20 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + c0 = a00^a30^a10^a40^a20; + c1 = a21^a01^a31^a11^a41; + c2 = a42^a22^a02^a32^a12; + c3 = a13^a43^a23^a03^a33; + c4 = a34^a14^a44^a24^a04; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a01^d1), 44); + b2 = ROL64((a02^d2), 43); + b3 = ROL64((a03^d3), 21); + b4 = ROL64((a04^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+3]; + a01 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a10^d0), 3); + b3 = ROL64((a11^d1), 45); + b4 = ROL64((a12^d2), 61); + b0 = ROL64((a13^d3), 28); + b1 = ROL64((a14^d4), 20); + a10 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a20^d0), 18); + b0 = ROL64((a21^d1), 1); + b1 = ROL64((a22^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a24^d4), 8); + a20 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a30^d0), 36); + b2 = ROL64((a31^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a33^d3), 56); + b0 = ROL64((a34^d4), 27); + a30 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a40^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a42^d2), 62); + b1 = ROL64((a43^d3), 55); + b2 = ROL64((a44^d4), 39); + a40 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + p->iSize = iSize; + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; + if( aData==0 ) return; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void sha3_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Update a SHA3Context using a single sqlite3_value. +*/ +static void sha3UpdateFromValue(SHA3Context *p, sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: { + SHA3Update(p, (const unsigned char*)"N",1); + break; + } + case SQLITE_INTEGER: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + sqlite3_int64 v = sqlite3_value_int64(pVal); + memcpy(&u, &v, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(p, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_value_double(pVal); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(p,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_text(pVal); + sha3_step_vformat(p,"T%d:",n2); + SHA3Update(p, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_blob(pVal); + sha3_step_vformat(p,"B%d:",n2); + SHA3Update(p, z2, n2); + break; + } + } +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z ){ + n = (int)strlen(z); + sha3_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + } + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; inRate==0 ){ + int sz = 256; + if( argc==2 ){ + sz = sqlite3_value_int(argv[1]); + if( sz!=224 && sz!=384 && sz!=512 ){ + sz = 256; + } + } + SHA3Init(p, sz); + } + sha3UpdateFromValue(p, argv[0]); +} + + +/* +** xFinal function for sha3_agg(). +*/ +static void sha3AggFinal(sqlite3_context *context){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->iSize ){ + sqlite3_result_blob(context, SHA3Final(p), p->iSize/8, SQLITE_TRANSIENT); + } +} + + + +#ifdef _WIN32 + +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ../ext/misc/shathree.c ********************/ +/************************* Begin ../ext/misc/sha1.c ******************/ +/* +** 2017-01-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha1(X) +** sha1_query(Y) +** +** The sha1(X) function computes the SHA1 hash of the input X, or NULL if +** X is NULL. +** +** The sha1_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/****************************************************************************** +** The Hash Engine +*/ +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +static void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut, /* Store hex or binary hash here */ + int bAsBinary /* 1 for binary hash, 0 for hex hash */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + if( bAsBinary ){ + memcpy(zOut, digest, 20); + }else{ + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; + } +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha1(X) function. +** +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hash without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + char zOut[44]; + + assert( argc==1 ); + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + } + if( sqlite3_user_data(context)!=0 ){ + hash_finish(&cx, zOut, 1); + sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); + }else{ + hash_finish(&cx, zOut, 0); + sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + } +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + assert( argc==1 ); + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut, 0); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static int one = 1; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha1", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1b", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + (void*)&one, sha1Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1_query", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sha1QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ../ext/misc/sha1.c ********************/ +/************************* Begin ../ext/misc/uint.c ******************/ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the maginitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + (void)notUsed; + while( i +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. +*/ +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; + int iExp = 0; + + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_from_text_failed; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_from_text_failed; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. +*/ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc( nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 1); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 1); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. A := A + B. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pB ){ + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + } + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + decimalMul(pA, pB); + if( pA->oom ){ + goto mul_end; + } + decimal_result(context, pA); + +mul_end: + decimal_free(pA); + decimal_free(pB); +} + +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + +#ifdef _WIN32 + +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + int iArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "decimal_sum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, + decimalSumStep, decimalSumFinalize, + decimalSumValue, decimalSumInverse, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_collation(db, "decimal", SQLITE_UTF8, + 0, decimalCollFunc); + } + return rc; +} + +/************************* End ../ext/misc/decimal.c ********************/ +/************************* Begin ../ext/misc/percentile.c ******************/ +/* +** 2013-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the percentile(Y,P) SQL function +** and similar as described below: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. +*/ +#if defined(SQLITE3_H) + /* no-op */ +#elif defined(SQLITE_STATIC_PERCENTILE) +/* # include "sqlite3.h" */ +#else +/* # include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#endif +#include +#include +#include + +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + unsigned nAlloc; /* Number of slots allocated for a[] */ + unsigned nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ + double *a; /* Array of Y values */ +}; + +/* Details of each function in the percentile family */ +typedef struct PercentileFunc PercentileFunc; +struct PercentileFunc { + const char *zName; /* Function name */ + char nArg; /* Number of arguments */ + char mxFrac; /* Maximum value of the "fraction" input */ + char bDiscrete; /* True for percentile_disc() */ +}; +static const PercentileFunc aPercentFunc[] = { + { "median", 1, 1, 0 }, + { "percentile", 2, 100, 0 }, + { "percentile_cont", 2, 1, 0 }, + { "percentile_disc", 2, 1, 1 }, +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int percentIsInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less. +*/ +static int percentSameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static int percentBinarySearch(Percentile *p, double y, int bExact){ + int iFirst = 0; /* First element of search range */ + int iLast = p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + int iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrance of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* Requirement 3: P must be a number between 0 and 100 */ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)pFunc->mxFrac); + return; + } + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + percentError(pCtx, "input to %%s() is not numeric"); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + unsigned n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + int i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } +} + +/* +** Interchange two doubles. +*/ +#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} + +/* +** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. +*/ +static void percentSort(double *a, unsigned int n){ + int iLt; /* Entries before a[iLt] are less than rPivot */ + int iGt; /* Entries at or after a[iGt] are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + + assert( n>=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; ibSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + unsigned i1, i2; + double v1, v2; + double ix, vx; + p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); + if( p==0 ) return; + if( p->a==0 ) return; + if( p->nUsed ){ + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); + i1 = (unsigned)ix; + if( pFunc->bDiscrete ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } + sqlite3_result_double(pCtx, vx); + } + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} + +#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) + +#endif +int sqlite3_percentile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + unsigned int i; +#if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE) + (void)pApi; /* Unused parameter */ +#else + SQLITE_EXTENSION_INIT2(pApi); +#endif + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +/* #include "sqlite3ext.h" */ + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +/************************* End ../ext/misc/base64.c ********************/ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#define OMIT_BASE85_CHECKER +/************************* Begin ../ext/misc/base85.c ******************/ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +/* # include "sqlite3ext.h" */ + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (((unsigned long)pIn[0])<<24) | + (pIn[1]<<16) | (pIn[2]<<8) | pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + case 3: + *pOut++ = (qv >> 16)&0xff; + case 2: + *pOut++ = (qv >> 8)&0xff; + case 1: + *pOut++ = qv&0xff; + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +# endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif + +/************************* End ../ext/misc/base85.c ********************/ +/************************* Begin ../ext/misc/ieee754.c ******************/ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(2,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + if( 1-e >= 64 ){ + m = 0; + }else{ + m >>= 1-e; + } + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_inc", 2, 0, ieee754inc }, + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. +** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** The virtual table also has a rowid, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return that member of a generate_series(...) sequence whose 0-based +** index is ix. The 0th member is given by smBase. The sequence members +** progress per ix increment by smStep. +*/ +static sqlite3_int64 genSeqMember( + sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix +){ + static const sqlite3_uint64 mxI64 = + ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; + if( ix>=mxI64 ){ + /* Get ix into signed i64 range. */ + ix -= mxI64; + /* With 2's complement ALU, this next can be 1 step, but is split into + * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ + smBase += (mxI64/2) * smStep; + smBase += (mxI64 - mxI64/2) * smStep; + } + /* Under UBSAN (or on 1's complement machines), must do this last term + * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ + if( ix>=2 ){ + sqlite3_int64 ix2 = (sqlite3_int64)ix/2; + smBase += ix2*smStep; + ix -= ix2; + } + return smBase + ((sqlite3_int64)ix)*smStep; +} + +/* typedef unsigned char u8; */ + +typedef struct SequenceSpec { + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_int64 iStep; /* Increment ("step") */ + sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ + sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ + sqlite3_int64 iValueNow; /* Current value during generation */ + u8 isNotEOF; /* Sequence generation not exhausted */ + u8 isReversing; /* Sequence is being reverse generated */ +} SequenceSpec; + +/* +** Prepare a SequenceSpec for use in generating an integer series +** given initialized iBase, iTerm and iStep values. Sequence is +** initialized per given isReversing. Other members are computed. +*/ +static void setupSequence( SequenceSpec *pss ){ + int bSameSigns; + pss->uSeqIndexMax = 0; + pss->isNotEOF = 0; + bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); + if( pss->iTerm < pss->iBase ){ + sqlite3_uint64 nuspan = 0; + if( bSameSigns ){ + nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iBase>=0 and iTerm<0 . */ + nuspan = 1; + nuspan += pss->iBase; + nuspan += -(pss->iTerm+1); + } + if( pss->iStep<0 ){ + pss->isNotEOF = 1; + if( nuspan==ULONG_MAX ){ + pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; + }else if( pss->iStep>LLONG_MIN ){ + pss->uSeqIndexMax = nuspan/-pss->iStep; + } + } + }else if( pss->iTerm > pss->iBase ){ + sqlite3_uint64 puspan = 0; + if( bSameSigns ){ + puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iTerm>=0 and iBase<0 . */ + puspan = 1; + puspan += pss->iTerm; + puspan += -(pss->iBase+1); + } + if( pss->iStep>0 ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = puspan/pss->iStep; + } + }else if( pss->iTerm == pss->iBase ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = 0; + } + pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; + pss->iValueNow = (pss->isReversing) + ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) + : pss->iBase; +} + +/* +** Progress sequence generator to yield next value, if any. +** Leave its state to either yield next value or be at EOF. +** Return whether there is a next value, or 0 at EOF. +*/ +static int progressSequence( SequenceSpec *pss ){ + if( !pss->isNotEOF ) return 0; + if( pss->isReversing ){ + if( pss->uSeqIndexNow > 0 ){ + pss->uSeqIndexNow--; + pss->iValueNow -= pss->iStep; + }else{ + pss->isNotEOF = 0; + } + }else{ + if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ + pss->uSeqIndexNow++; + pss->iValueNow += pss->iStep; + }else{ + pss->isNotEOF = 0; + } + } + return pss->isNotEOF; +} + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + SequenceSpec ss; /* (this) Derived class data */ +}; + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pUnused, + int argcUnused, const char *const*argvUnused, + sqlite3_vtab **ppVtab, + char **pzErrUnused +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + (void)pUnused; + (void)argcUnused; + (void)argvUnused; + (void)pzErrUnused; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + (void)pUnused; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + progressSequence( & pCur->ss ); + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + default: x = pCur->ss.iValueNow; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +#ifndef LARGEST_UINT64 +#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) +#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** Return the rowid for the current row, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_uint64 n = pCur->ss.uSeqIndexNow; + *pRowid = (sqlite3_int64)((nss.isNotEOF; +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: valuess.iBase = sqlite3_value_int64(argv[i++]); + }else{ + pCur->ss.iBase = 0; + } + if( idxNum & 0x02 ){ + pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); + }else{ + pCur->ss.iTerm = 0xffffffff; + } + if( idxNum & 0x04 ){ + pCur->ss.iStep = sqlite3_value_int64(argv[i++]); + if( pCur->ss.iStep==0 ){ + pCur->ss.iStep = 1; + }else if( pCur->ss.iStep<0 ){ + if( (idxNum & 0x10)==0 ) idxNum |= 0x08; + } + }else{ + pCur->ss.iStep = 1; + } + + /* If there are constraints on the value column but there are + ** no constraints on the start, stop, and step columns, then + ** initialize the default range to be the entire range of 64-bit signed + ** integers. This range will contracted by the value column constraints + ** further below. + */ + if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){ + pCur->ss.iBase = SMALLEST_INT64; + } + if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){ + pCur->ss.iTerm = LARGEST_INT64; + } + pCur->ss.iOBase = pCur->ss.iBase; + pCur->ss.iOTerm = pCur->ss.iTerm; + + /* Extract the LIMIT and OFFSET values, but do not apply them yet. + ** The range must first be constrained by the limits on value. + */ + if( idxNum & 0x20 ){ + iLimit = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x40 ){ + iOffset = sqlite3_value_int64(argv[i++]); + } + } + + if( idxNum & 0x3380 ){ + /* Extract the maximum range of output values determined by + ** constraints on the "value" column. + */ + if( idxNum & 0x0080 ){ + iMin = iMax = sqlite3_value_int64(argv[i++]); + }else{ + if( idxNum & 0x0300 ){ + iMin = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x0200 ){ + if( iMin==LARGEST_INT64 ){ + returnNoRows = 1; + }else{ + iMin++; + } + } + } + if( idxNum & 0x3000 ){ + iMax = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x2000 ){ + if( iMax==SMALLEST_INT64 ){ + returnNoRows = 1; + }else{ + iMax--; + } + } + } + if( iMin>iMax ){ + returnNoRows = 1; + } + } + + /* Try to reduce the range of values to be generated based on + ** constraints on the "value" column. + */ + if( pCur->ss.iStep>0 ){ + sqlite3_int64 szStep = pCur->ss.iStep; + if( pCur->ss.iBasess.iBase; + pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTerm>iMax ){ + sqlite3_uint64 d = pCur->ss.iTerm - iMax; + pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep; + } + }else{ + sqlite3_int64 szStep = -pCur->ss.iStep; + assert( szStep>0 ); + if( pCur->ss.iBase>iMax ){ + sqlite3_uint64 d = pCur->ss.iBase - iMax; + pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTermss.iTerm; + pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep; + } + } + } + + /* Apply LIMIT and OFFSET constraints, if any */ + if( idxNum & 0x20 ){ + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } + if( iLimit>=0 ){ + sqlite3_int64 iTerm; + iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; + if( pCur->ss.iStep<0 ){ + if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + }else{ + if( iTermss.iTerm ) pCur->ss.iTerm = iTerm; + } + } + } + + + for(i=0; iss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; + } + if( idxNum & 0x08 ){ + pCur->ss.isReversing = pCur->ss.iStep > 0; + }else{ + pCur->ss.isReversing = pCur->ss.iStep < 0; + } + setupSequence( &pCur->ss ); + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** 0x0001 start = $num +** 0x0002 stop = $num +** 0x0004 step = $num +** 0x0008 output is in descending order +** 0x0010 output is in ascending order +** 0x0020 LIMIT $num +** 0x0040 OFFSET $num +** 0x0080 value = $num +** 0x0100 value >= $num +** 0x0200 value > $num +** 0x1000 value <= $num +** 0x2000 value < $num +** +** Only one of 0x0100 or 0x0200 will be returned. Similarly, only +** one of 0x1000 or 0x2000 will be returned. If the 0x0080 is set, then +** none of the 0xff00 bits will be set. +** +** The order of parameters passed to xFilter is as follows: +** +** * The argument to start= if bit 0x0001 is in the idxNum mask +** * The argument to stop= if bit 0x0002 is in the idxNum mask +** * The argument to step= if bit 0x0004 is in the idxNum mask +** * The argument to LIMIT if bit 0x0020 is in the idxNum mask +** * The argument to OFFSET if bit 0x0040 is in the idxNum mask +** * The argument to value=, or value>= or value> if any of +** bits 0x0380 are in the idxNum mask +** * The argument to value<= or value< if either of bits 0x3000 +** are in the mask +** +*/ +static int seriesBestIndex( + sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo +){ + int i, j; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif + int unusableMask = 0; /* Mask of unusable constraints */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + int aIdx[7]; /* Constraints on start, stop, step, LIMIT, OFFSET, + ** and value. aIdx[5] covers value=, value>=, and + ** value>, aIdx[6] covers value<= and value< */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that the start, stop, and step columns + ** are the last three columns in the virtual table. */ + assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); + assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); + + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = aIdx[5] = aIdx[6] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; /* 0 for start, 1 for stop, 2 for step */ + int iMask; /* bitmask for those column */ + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } + if( pConstraint->iColumniColumn==SERIES_COLUMN_VALUE ){ + switch( op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: { + idxNum |= 0x0080; + idxNum &= ~0x3300; + aIdx[5] = i; + aIdx[6] = -1; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0100; + idxNum &= ~0x0200; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0200; + idxNum &= ~0x0100; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x1000; + idxNum &= ~0x2000; + aIdx[6] = i; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x2000; + idxNum &= ~0x1000; + aIdx[6] = i; + break; + } + } + } + continue; + } + iCol = pConstraint->iColumn - SERIES_COLUMN_START; + assert( iCol>=0 && iCol<=2 ); + iMask = 1 << iCol; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + continue; + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idxNum |= iMask; + aIdx[iCol] = i; + } + } + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<7; i++){ + if( (j = aIdx[i])>=0 ){ + pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; + } + } + /* The current generate_column() implementation requires at least one + ** argument (the START value). Legacy versions assumed START=0 if the + ** first argument was omitted. Compile with -DZERO_ARGUMENT_GENERATE_SERIES + ** to obtain the legacy behavior */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( !bStartSeen ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf( + "first argument to \"generate_series()\" missing or unusable"); + return SQLITE_ERROR; + } +#endif + if( (unusableMask & ~idxNum)!=0 ){ + /* The start, stop, and step columns are inputs. Therefore if there + ** are unusable constraints on any of start, stop, or step then + ** this plan is unusable */ + return SQLITE_CONSTRAINT; + } + if( (idxNum & 0x03)==0x03 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ + if( pIdxInfo->aOrderBy[0].desc ){ + idxNum |= 0x08; + }else{ + idxNum |= 0x10; + } + pIdxInfo->orderByConsumed = 1; + } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; + }else{ + /* If either boundary is missing, we have to generate a huge span + ** of numbers. Make this case very expensive so that the query + ** planner will work hard to avoid it. */ + pIdxInfo->estimatedRows = 2147483647; + } + pIdxInfo->idxNum = idxNum; +#ifdef SQLITE_INDEX_SCAN_HEX + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_HEX; +#endif + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 + +#endif +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3_libversion_number()<3008012 && pzErrMsg!=0 ){ + *pzErrMsg = sqlite3_mprintf( + "generate_series() requires SQLite 3.8.12 or later"); + return SQLITE_ERROR; + } + rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); +#endif + return rc; +} + +/************************* End ../ext/misc/series.c ********************/ +/************************* Begin ../ext/misc/regexp.c ******************/ +/* +** 2012-11-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** The code in this file implements a compact but reasonably +** efficient regular-expression matcher for posix extended regular +** expressions against UTF8 text. +** +** This file is an SQLite extension. It registers a single function +** named "regexp(A,B)" where A is the regular expression and B is the +** string to be matched. By registering this function, SQLite will also +** then implement the "B regexp A" operator. Note that with the function +** the regular expression comes first, but with the operator it comes +** second. +** +** The following regular expression syntax is supported: +** +** X* zero or more occurrences of X +** X+ one or more occurrences of X +** X? zero or one occurrences of X +** X{p,q} between p and q occurrences of X +** (X) match X +** X|Y X or Y +** ^X X occurring at the beginning of the string +** X$ X occurring at the end of the string +** . Match any single character +** \c Character c where c is one of \{}()[]|*+?. +** \c C-language escapes for c in afnrtv. ex: \t or \n +** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX +** \xXX Where XX is exactly 2 hex digits, unicode value XX +** [abc] Any single character from the set abc +** [^abc] Any single character not in the set abc +** [a-z] Any single character in the range a-z +** [^a-z] Any single character not in the range a-z +** \b Word boundary +** \w Word character. [A-Za-z0-9_] +** \W Non-word character +** \d Digit +** \D Non-digit +** \s Whitespace character +** \S Non-whitespace character +** +** A nondeterministic finite automaton (NFA) is used for matching, so the +** performance is bounded by O(N*M) where N is the size of the regular +** expression and M is the size of the input string. The matcher never +** exhibits exponential behavior. Note that the X{p,q} operator expands +** to p copies of X following by q-p copies of X? and that the size of the +** regular expression in the O(N*M) performance bound is computed after +** this expansion. +*/ +#include +#include +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 + +/* +** The following #defines change the names of some functions implemented in +** this file to prevent name collisions with C-library functions of the +** same name. +*/ +#define re_match sqlite3re_match +#define re_compile sqlite3re_compile +#define re_free sqlite3re_free + +/* The end-of-input character */ +#define RE_EOF 0 /* End of input */ +#define RE_START 0xfffffff /* Start of input - larger than an UTF-8 */ + +/* The NFA is implemented as sequence of opcodes taken from the following +** set. Each opcode has a single integer argument. +*/ +#define RE_OP_MATCH 1 /* Match the one character in the argument */ +#define RE_OP_ANY 2 /* Match any one character. (Implements ".") */ +#define RE_OP_ANYSTAR 3 /* Special optimized version of .* */ +#define RE_OP_FORK 4 /* Continue to both next and opcode at iArg */ +#define RE_OP_GOTO 5 /* Jump to opcode at iArg */ +#define RE_OP_ACCEPT 6 /* Halt and indicate a successful match */ +#define RE_OP_CC_INC 7 /* Beginning of a [...] character class */ +#define RE_OP_CC_EXC 8 /* Beginning of a [^...] character class */ +#define RE_OP_CC_VALUE 9 /* Single value in a character class */ +#define RE_OP_CC_RANGE 10 /* Range of values in a character class */ +#define RE_OP_WORD 11 /* Perl word character [A-Za-z0-9_] */ +#define RE_OP_NOTWORD 12 /* Not a perl word character */ +#define RE_OP_DIGIT 13 /* digit: [0-9] */ +#define RE_OP_NOTDIGIT 14 /* Not a digit */ +#define RE_OP_SPACE 15 /* space: [ \t\n\r\v\f] */ +#define RE_OP_NOTSPACE 16 /* Not a digit */ +#define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ +#define RE_OP_ATSTART 18 /* Currently at the start of the string */ + +#if defined(SQLITE_DEBUG) +/* Opcode names used for symbolic debugging */ +static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", + "ATSTART", +}; +#endif /* SQLITE_DEBUG */ + + +/* Each opcode is a "state" in the NFA */ +typedef unsigned short ReStateNumber; + +/* Because this is an NFA and not a DFA, multiple states can be active at +** once. An instance of the following object records all active states in +** the NFA. The implementation is optimized for the common case where the +** number of actives states is small. +*/ +typedef struct ReStateSet { + unsigned nState; /* Number of current states */ + ReStateNumber *aState; /* Current states */ +} ReStateSet; + +/* An input string read one character at a time. +*/ +typedef struct ReInput ReInput; +struct ReInput { + const unsigned char *z; /* All text */ + int i; /* Next byte to read */ + int mx; /* EOF when i>=mx */ +}; + +/* A compiled NFA (or an NFA that is in the process of being compiled) is +** an instance of the following object. +*/ +typedef struct ReCompiled ReCompiled; +struct ReCompiled { + ReInput sIn; /* Regular expression text */ + const char *zErr; /* Error message to return */ + char *aOp; /* Operators for the virtual machine */ + int *aArg; /* Arguments to each operator */ + unsigned (*xNextChar)(ReInput*); /* Next character function */ + unsigned char zInit[12]; /* Initial text to match */ + int nInit; /* Number of bytes in zInit */ + unsigned nState; /* Number of entries in aOp[] and aArg[] */ + unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ +}; + +/* Add a state to the given state set if it is not already there */ +static void re_add_state(ReStateSet *pSet, int newState){ + unsigned i; + for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; + pSet->aState[pSet->nState++] = (ReStateNumber)newState; +} + +/* Extract the next unicode character from *pzIn and return it. Advance +** *pzIn to the first byte past the end of the character returned. To +** be clear: this routine converts utf8 to unicode. This routine is +** optimized for the common case where the next character is a single byte. +*/ +static unsigned re_next_char(ReInput *p){ + unsigned c; + if( p->i>=p->mx ) return 0; + c = p->z[p->i++]; + if( c>=0x80 ){ + if( (c&0xe0)==0xc0 && p->imx && (p->z[p->i]&0xc0)==0x80 ){ + c = (c&0x1f)<<6 | (p->z[p->i++]&0x3f); + if( c<0x80 ) c = 0xfffd; + }else if( (c&0xf0)==0xe0 && p->i+1mx && (p->z[p->i]&0xc0)==0x80 + && (p->z[p->i+1]&0xc0)==0x80 ){ + c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f); + p->i += 2; + if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd; + }else if( (c&0xf8)==0xf0 && p->i+2mx && (p->z[p->i]&0xc0)==0x80 + && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){ + c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6) + | (p->z[p->i+2]&0x3f); + p->i += 3; + if( c<=0xffff || c>0x10ffff ) c = 0xfffd; + }else{ + c = 0xfffd; + } + } + return c; +} +static unsigned re_next_char_nocase(ReInput *p){ + unsigned c = re_next_char(p); + if( c>='A' && c<='Z' ) c += 'a' - 'A'; + return c; +} + +/* Return true if c is a perl "word" character: [A-Za-z0-9_] */ +static int re_word_char(int c){ + return (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +/* Return true if c is a "digit" character: [0-9] */ +static int re_digit_char(int c){ + return (c>='0' && c<='9'); +} + +/* Return true if c is a perl "space" character: [ \t\r\n\v\f] */ +static int re_space_char(int c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* Run a compiled regular expression on the zero-terminated input +** string zIn[]. Return true on a match and false if there is no match. +*/ +static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ + ReStateSet aStateSet[2], *pThis, *pNext; + ReStateNumber aSpace[100]; + ReStateNumber *pToFree; + unsigned int i = 0; + unsigned int iSwap = 0; + int c = RE_START; + int cPrev = 0; + int rc = 0; + ReInput in; + + in.z = zIn; + in.i = 0; + in.mx = nIn>=0 ? nIn : (int)strlen((char const*)zIn); + + /* Look for the initial prefix match, if there is one. */ + if( pRe->nInit ){ + unsigned char x = pRe->zInit[0]; + while( in.i+pRe->nInit<=in.mx + && (zIn[in.i]!=x || + strncmp((const char*)zIn+in.i, (const char*)pRe->zInit, pRe->nInit)!=0) + ){ + in.i++; + } + if( in.i+pRe->nInit>in.mx ) return 0; + c = RE_START-1; + } + + if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ + pToFree = 0; + aStateSet[0].aState = aSpace; + }else{ + pToFree = sqlite3_malloc64( sizeof(ReStateNumber)*2*pRe->nState ); + if( pToFree==0 ) return -1; + aStateSet[0].aState = pToFree; + } + aStateSet[1].aState = &aStateSet[0].aState[pRe->nState]; + pNext = &aStateSet[1]; + pNext->nState = 0; + re_add_state(pNext, 0); + while( c!=RE_EOF && pNext->nState>0 ){ + cPrev = c; + c = pRe->xNextChar(&in); + pThis = pNext; + pNext = &aStateSet[iSwap]; + iSwap = 1 - iSwap; + pNext->nState = 0; + for(i=0; inState; i++){ + int x = pThis->aState[i]; + switch( pRe->aOp[x] ){ + case RE_OP_MATCH: { + if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); + break; + } + case RE_OP_ATSTART: { + if( cPrev==RE_START ) re_add_state(pThis, x+1); + break; + } + case RE_OP_ANY: { + if( c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_WORD: { + if( re_word_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTWORD: { + if( !re_word_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_DIGIT: { + if( re_digit_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTDIGIT: { + if( !re_digit_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_SPACE: { + if( re_space_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTSPACE: { + if( !re_space_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_BOUNDARY: { + if( re_word_char(c)!=re_word_char(cPrev) ) re_add_state(pThis, x+1); + break; + } + case RE_OP_ANYSTAR: { + re_add_state(pNext, x); + re_add_state(pThis, x+1); + break; + } + case RE_OP_FORK: { + re_add_state(pThis, x+pRe->aArg[x]); + re_add_state(pThis, x+1); + break; + } + case RE_OP_GOTO: { + re_add_state(pThis, x+pRe->aArg[x]); + break; + } + case RE_OP_ACCEPT: { + rc = 1; + goto re_match_end; + } + case RE_OP_CC_EXC: { + if( c==0 ) break; + /* fall-through */ goto re_op_cc_inc; + } + case RE_OP_CC_INC: re_op_cc_inc: { + int j = 1; + int n = pRe->aArg[x]; + int hit = 0; + for(j=1; j>0 && jaOp[x+j]==RE_OP_CC_VALUE ){ + if( pRe->aArg[x+j]==c ){ + hit = 1; + j = -1; + } + }else{ + if( pRe->aArg[x+j]<=c && pRe->aArg[x+j+1]>=c ){ + hit = 1; + j = -1; + }else{ + j++; + } + } + } + if( pRe->aOp[x]==RE_OP_CC_EXC ) hit = !hit; + if( hit ) re_add_state(pNext, x+n); + break; + } + } + } + } + for(i=0; inState; i++){ + int x = pNext->aState[i]; + while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x]; + if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; } + } +re_match_end: + sqlite3_free(pToFree); + return rc; +} + +/* Resize the opcode and argument arrays for an RE under construction. +*/ +static int re_resize(ReCompiled *p, int N){ + char *aOp; + int *aArg; + aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); + if( aOp==0 ) return 1; + p->aOp = aOp; + aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); + if( aArg==0 ) return 1; + p->aArg = aArg; + p->nAlloc = N; + return 0; +} + +/* Insert a new opcode and argument into an RE under construction. The +** insertion point is just prior to existing opcode iBefore. +*/ +static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ + int i; + if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0; + for(i=p->nState; i>iBefore; i--){ + p->aOp[i] = p->aOp[i-1]; + p->aArg[i] = p->aArg[i-1]; + } + p->nState++; + p->aOp[iBefore] = (char)op; + p->aArg[iBefore] = arg; + return iBefore; +} + +/* Append a new opcode and argument to the end of the RE under construction. +*/ +static int re_append(ReCompiled *p, int op, int arg){ + return re_insert(p, p->nState, op, arg); +} + +/* Make a copy of N opcodes starting at iStart onto the end of the RE +** under construction. +*/ +static void re_copy(ReCompiled *p, int iStart, int N){ + if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; + memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); + memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); + p->nState += N; +} + +/* Return true if c is a hexadecimal digit character: [0-9a-fA-F] +** If c is a hex digit, also set *pV = (*pV)*16 + valueof(c). If +** c is not a hex digit *pV is unchanged. +*/ +static int re_hex(int c, int *pV){ + if( c>='0' && c<='9' ){ + c -= '0'; + }else if( c>='a' && c<='f' ){ + c -= 'a' - 10; + }else if( c>='A' && c<='F' ){ + c -= 'A' - 10; + }else{ + return 0; + } + *pV = (*pV)*16 + (c & 0xff); + return 1; +} + +/* A backslash character has been seen, read the next character and +** return its interpretation. +*/ +static unsigned re_esc_char(ReCompiled *p){ + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; + static const char zTrans[] = "\a\f\n\r\t\v"; + int i, v = 0; + char c; + if( p->sIn.i>=p->sIn.mx ) return 0; + c = p->sIn.z[p->sIn.i]; + if( c=='u' && p->sIn.i+4sIn.mx ){ + const unsigned char *zIn = p->sIn.z + p->sIn.i; + if( re_hex(zIn[1],&v) + && re_hex(zIn[2],&v) + && re_hex(zIn[3],&v) + && re_hex(zIn[4],&v) + ){ + p->sIn.i += 5; + return v; + } + } + if( c=='x' && p->sIn.i+2sIn.mx ){ + const unsigned char *zIn = p->sIn.z + p->sIn.i; + if( re_hex(zIn[1],&v) + && re_hex(zIn[2],&v) + ){ + p->sIn.i += 3; + return v; + } + } + for(i=0; zEsc[i] && zEsc[i]!=c; i++){} + if( zEsc[i] ){ + if( i<6 ) c = zTrans[i]; + p->sIn.i++; + }else{ + p->zErr = "unknown \\ escape"; + } + return c; +} + +/* Forward declaration */ +static const char *re_subcompile_string(ReCompiled*); + +/* Peek at the next byte of input */ +static unsigned char rePeek(ReCompiled *p){ + return p->sIn.isIn.mx ? p->sIn.z[p->sIn.i] : 0; +} + +/* Compile RE text into a sequence of opcodes. Continue up to the +** first unmatched ")" character, then return. If an error is found, +** return a pointer to the error message string. +*/ +static const char *re_subcompile_re(ReCompiled *p){ + const char *zErr; + int iStart, iEnd, iGoto; + iStart = p->nState; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + while( rePeek(p)=='|' ){ + iEnd = p->nState; + re_insert(p, iStart, RE_OP_FORK, iEnd + 2 - iStart); + iGoto = re_append(p, RE_OP_GOTO, 0); + p->sIn.i++; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + p->aArg[iGoto] = p->nState - iGoto; + } + return 0; +} + +/* Compile an element of regular expression text (anything that can be +** an operand to the "|" operator). Return NULL on success or a pointer +** to the error message if there is a problem. +*/ +static const char *re_subcompile_string(ReCompiled *p){ + int iPrev = -1; + int iStart; + unsigned c; + const char *zErr; + while( (c = p->xNextChar(&p->sIn))!=0 ){ + iStart = p->nState; + switch( c ){ + case '|': + case ')': { + p->sIn.i--; + return 0; + } + case '(': { + zErr = re_subcompile_re(p); + if( zErr ) return zErr; + if( rePeek(p)!=')' ) return "unmatched '('"; + p->sIn.i++; + break; + } + case '.': { + if( rePeek(p)=='*' ){ + re_append(p, RE_OP_ANYSTAR, 0); + p->sIn.i++; + }else{ + re_append(p, RE_OP_ANY, 0); + } + break; + } + case '*': { + if( iPrev<0 ) return "'*' without operand"; + re_insert(p, iPrev, RE_OP_GOTO, p->nState - iPrev + 1); + re_append(p, RE_OP_FORK, iPrev - p->nState + 1); + break; + } + case '+': { + if( iPrev<0 ) return "'+' without operand"; + re_append(p, RE_OP_FORK, iPrev - p->nState); + break; + } + case '?': { + if( iPrev<0 ) return "'?' without operand"; + re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); + break; + } + case '$': { + re_append(p, RE_OP_MATCH, RE_EOF); + break; + } + case '^': { + re_append(p, RE_OP_ATSTART, 0); + break; + } + case '{': { + int m = 0, n = 0; + int sz, j; + if( iPrev<0 ) return "'{m,n}' without operand"; + while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } + n = m; + if( c==',' ){ + p->sIn.i++; + n = 0; + while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } + } + if( c!='}' ) return "unmatched '{'"; + if( n>0 && nsIn.i++; + sz = p->nState - iPrev; + if( m==0 ){ + if( n==0 ) return "both m and n are zero in '{m,n}'"; + re_insert(p, iPrev, RE_OP_FORK, sz+1); + iPrev++; + n--; + }else{ + for(j=1; j0 ){ + re_append(p, RE_OP_FORK, -sz); + } + break; + } + case '[': { + unsigned int iFirst = p->nState; + if( rePeek(p)=='^' ){ + re_append(p, RE_OP_CC_EXC, 0); + p->sIn.i++; + }else{ + re_append(p, RE_OP_CC_INC, 0); + } + while( (c = p->xNextChar(&p->sIn))!=0 ){ + if( c=='[' && rePeek(p)==':' ){ + return "POSIX character classes not supported"; + } + if( c=='\\' ) c = re_esc_char(p); + if( rePeek(p)=='-' ){ + re_append(p, RE_OP_CC_RANGE, c); + p->sIn.i++; + c = p->xNextChar(&p->sIn); + if( c=='\\' ) c = re_esc_char(p); + re_append(p, RE_OP_CC_RANGE, c); + }else{ + re_append(p, RE_OP_CC_VALUE, c); + } + if( rePeek(p)==']' ){ p->sIn.i++; break; } + } + if( c==0 ) return "unclosed '['"; + if( p->nState>iFirst ) p->aArg[iFirst] = p->nState - iFirst; + break; + } + case '\\': { + int specialOp = 0; + switch( rePeek(p) ){ + case 'b': specialOp = RE_OP_BOUNDARY; break; + case 'd': specialOp = RE_OP_DIGIT; break; + case 'D': specialOp = RE_OP_NOTDIGIT; break; + case 's': specialOp = RE_OP_SPACE; break; + case 'S': specialOp = RE_OP_NOTSPACE; break; + case 'w': specialOp = RE_OP_WORD; break; + case 'W': specialOp = RE_OP_NOTWORD; break; + } + if( specialOp ){ + p->sIn.i++; + re_append(p, specialOp, 0); + }else{ + c = re_esc_char(p); + re_append(p, RE_OP_MATCH, c); + } + break; + } + default: { + re_append(p, RE_OP_MATCH, c); + break; + } + } + iPrev = iStart; + } + return 0; +} + +/* Free and reclaim all the memory used by a previously compiled +** regular expression. Applications should invoke this routine once +** for every call to re_compile() to avoid memory leaks. +*/ +static void re_free(ReCompiled *pRe){ + if( pRe ){ + sqlite3_free(pRe->aOp); + sqlite3_free(pRe->aArg); + sqlite3_free(pRe); + } +} + +/* +** Compile a textual regular expression in zIn[] into a compiled regular +** expression suitable for us by re_match() and return a pointer to the +** compiled regular expression in *ppRe. Return NULL on success or an +** error message if something goes wrong. +*/ +static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ + ReCompiled *pRe; + const char *zErr; + int i, j; + + *ppRe = 0; + pRe = sqlite3_malloc( sizeof(*pRe) ); + if( pRe==0 ){ + return "out of memory"; + } + memset(pRe, 0, sizeof(*pRe)); + pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; + if( re_resize(pRe, 30) ){ + re_free(pRe); + return "out of memory"; + } + if( zIn[0]=='^' ){ + zIn++; + }else{ + re_append(pRe, RE_OP_ANYSTAR, 0); + } + pRe->sIn.z = (unsigned char*)zIn; + pRe->sIn.i = 0; + pRe->sIn.mx = (int)strlen(zIn); + zErr = re_subcompile_re(pRe); + if( zErr ){ + re_free(pRe); + return zErr; + } + if( pRe->sIn.i>=pRe->sIn.mx ){ + re_append(pRe, RE_OP_ACCEPT, 0); + *ppRe = pRe; + }else{ + re_free(pRe); + return "unrecognized character"; + } + + /* The following is a performance optimization. If the regex begins with + ** ".*" (if the input regex lacks an initial "^") and afterwards there are + ** one or more matching characters, enter those matching characters into + ** zInit[]. The re_match() routine can then search ahead in the input + ** string looking for the initial match without having to run the whole + ** regex engine over the string. Do not worry about trying to match + ** unicode characters beyond plane 0 - those are very rare and this is + ** just an optimization. */ + if( pRe->aOp[0]==RE_OP_ANYSTAR && !noCase ){ + for(j=0, i=1; j<(int)sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ + unsigned x = pRe->aArg[i]; + if( x<=0x7f ){ + pRe->zInit[j++] = (unsigned char)x; + }else if( x<=0x7ff ){ + pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); + pRe->zInit[j++] = 0x80 | (x&0x3f); + }else if( x<=0xffff ){ + pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12)); + pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); + pRe->zInit[j++] = 0x80 | (x&0x3f); + }else{ + break; + } + } + if( j>0 && pRe->zInit[j-1]==0 ) j--; + pRe->nInit = j; + } + return pRe->zErr; +} + +/* +** Implementation of the regexp() SQL function. This function implements +** the build-in REGEXP operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A REGEXP B +** +** is implemented as regexp(B,A). +*/ +static void re_sql_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ReCompiled *pRe; /* Compiled regular expression */ + const char *zPattern; /* The regular expression */ + const unsigned char *zStr;/* String being searched */ + const char *zErr; /* Compile error message */ + int setAux = 0; /* True to invoke sqlite3_set_auxdata() */ + + (void)argc; /* Unused */ + pRe = sqlite3_get_auxdata(context, 0); + if( pRe==0 ){ + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + setAux = 1; + } + zStr = (const unsigned char*)sqlite3_value_text(argv[1]); + if( zStr!=0 ){ + sqlite3_result_int(context, re_match(pRe, zStr, -1)); + } + if( setAux ){ + sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + } +} + +#if defined(SQLITE_DEBUG) +/* +** This function is used for testing and debugging only. It is only available +** if the SQLITE_DEBUG compile-time option is used. +** +** Compile a regular expression and then convert the compiled expression into +** text and return that text. +*/ +static void re_bytecode_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern; + const char *zErr; + ReCompiled *pRe; + sqlite3_str *pStr; + int i; + int n; + char *z; + (void)argc; + + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + pStr = sqlite3_str_new(0); + if( pStr==0 ) goto re_bytecode_func_err; + if( pRe->nInit>0 ){ + sqlite3_str_appendf(pStr, "INIT "); + for(i=0; inInit; i++){ + sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]); + } + sqlite3_str_appendf(pStr, "\n"); + } + for(i=0; (unsigned)inState; i++){ + sqlite3_str_appendf(pStr, "%-8s %4d\n", + ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); + } + n = sqlite3_str_length(pStr); + z = sqlite3_str_finish(pStr); + if( n==0 ){ + sqlite3_free(z); + }else{ + sqlite3_result_text(context, z, n-1, sqlite3_free); + } + +re_bytecode_func_err: + re_free(pRe); +} + +#endif /* SQLITE_DEBUG */ + + +/* +** Invoke this routine to register the regexp() function with the +** SQLite database connection. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_regexp_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused */ + rc = sqlite3_create_function(db, "regexp", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_sql_func, 0, 0); + if( rc==SQLITE_OK ){ + /* The regexpi(PATTERN,STRING) function is a case-insensitive version + ** of regexp(PATTERN,STRING). */ + rc = sqlite3_create_function(db, "regexpi", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + (void*)db, re_sql_func, 0, 0); +#if defined(SQLITE_DEBUG) + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "regexp_bytecode", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_bytecode_func, 0, 0); + } +#endif /* SQLITE_DEBUG */ + } + return rc; +} + +/************************* End ../ext/misc/regexp.c ********************/ +#ifndef SQLITE_SHELL_FIDDLE +/************************* Begin ../ext/misc/fileio.c ******************/ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL functions readfile() and +** writefile(), and eponymous virtual type "fsdir". +** +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): +** +** If neither of the optional arguments is present, then this UDF +** function writes blob DATA to file FILE. If successful, the number +** of bytes written is returned. If an error occurs, NULL is returned. +** +** If the first option argument - MODE - is present, then it must +** be passed an integer value that corresponds to a POSIX mode +** value (file type + permissions, as returned in the stat.st_mode +** field by the stat() system call). Three types of files may +** be written/created: +** +** regular files: (mode & 0170000)==0100000 +** symbolic links: (mode & 0170000)==0120000 +** directories: (mode & 0170000)==0040000 +** +** For a directory, the DATA is ignored. For a symbolic link, it is +** interpreted as text and used as the target of the link. For a +** regular file, it is interpreted as a blob and written into the +** named file. Regardless of the type of file, its permissions are +** set to (mode & 0777) before returning. +** +** If the optional MTIME argument is present, then it is interpreted +** as an integer - the number of seconds since the unix epoch. The +** modification-time of the target file is set to this value before +** returning. +** +** If five or more arguments are passed to this function and an +** error is encountered, an exception is raised. +** +** READFILE(FILE): +** +** Read and return the contents of file FILE (type blob) from disk. +** +** FSDIR: +** +** Used as follows: +** +** SELECT * FROM fsdir($path [, $dir]); +** +** Parameter $path is an absolute or relative pathname. If the file that it +** refers to does not exist, it is an error. If the path refers to a regular +** file or symbolic link, it returns a single row. Or, if the path refers +** to a directory, it returns one row for the directory, and one row for each +** file within the hierarchy rooted at $path. +** +** Each row has the following columns: +** +** name: Path to file or directory (text value). +** mode: Value of stat.st_mode for directory entry (an integer). +** mtime: Value of stat.st_mtime for directory entry (an integer). +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. +** +** If a non-NULL value is specified for the optional $dir parameter and +** $path is a relative path, then $path is interpreted relative to $dir. +** And the paths returned in the "name" column of the table are also +** relative to directory $dir. +** +** Notes on building this extension for Windows: +** Unless linked statically with the SQLite library, a preprocessor +** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone +** DLL form of this extension for WIN32. See its use below for details. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#include +#include +#include +#if !defined(_WIN32) && !defined(WIN32) +# include +# include +# include +# include +#else +# include "windows.h" +# include +# include +/* # include "test_windirent.h" */ +# define dirent DIRENT +# ifndef chmod +# define chmod _chmod +# endif +# ifndef stat +# define stat _stat +# endif +# define mkdir(path,mode) _mkdir(path) +# define lstat(path,buf) stat(path,buf) +#endif +#include +#include + +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif + +/* +** Structure of the fsdir() table-valued function +*/ + /* 0 1 2 3 4 5 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +#define FSDIR_COLUMN_NAME 0 /* Name of the file */ +#define FSDIR_COLUMN_MODE 1 /* Access mode */ +#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ +#define FSDIR_COLUMN_DATA 3 /* File content */ +#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ + + +/* +** Set the result stored by context ctx to a blob containing the +** contents of file zName. Or, leave the result unchanged (NULL) +** if the file does not exist or is unreadable. +** +** If the file exceeds the SQLite blob size limit, through an +** SQLITE_TOOBIG error. +** +** Throw an SQLITE_IOERR if there are difficulties pulling the file +** off of disk. +*/ +static void readFileContents(sqlite3_context *ctx, const char *zName){ + FILE *in; + sqlite3_int64 nIn; + void *pBuf; + sqlite3 *db; + int mxBlob; + + in = sqlite3_fopen(zName, "rb"); + if( in==0 ){ + /* File does not exist or is unreadable. Leave the result set to NULL. */ + return; + } + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + db = sqlite3_context_db_handle(ctx); + mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + if( nIn>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + fclose(in); + return; + } + pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + if( pBuf==0 ){ + sqlite3_result_error_nomem(ctx); + fclose(in); + return; + } + if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ + sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_result_error_code(ctx, SQLITE_IOERR); + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + readFileContents(context, zName); +} + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +#if defined(_WIN32) +/* +** This function is designed to convert a Win32 FILETIME structure into the +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). +*/ +static sqlite3_uint64 fileTimeToUnixTime( + LPFILETIME pFileTime +){ + SYSTEMTIME epochSystemTime; + ULARGE_INTEGER epochIntervals; + FILETIME epochFileTime; + ULARGE_INTEGER fileIntervals; + + memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); + epochSystemTime.wYear = 1970; + epochSystemTime.wMonth = 1; + epochSystemTime.wDay = 1; + SystemTimeToFileTime(&epochSystemTime, &epochFileTime); + epochIntervals.LowPart = epochFileTime.dwLowDateTime; + epochIntervals.HighPart = epochFileTime.dwHighDateTime; + + fileIntervals.LowPart = pFileTime->dwLowDateTime; + fileIntervals.HighPart = pFileTime->dwHighDateTime; + + return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; +} + + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +# /* To allow a standalone DLL, use this next replacement function: */ +# undef sqlite3_win32_utf8_to_unicode +# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 +# +LPWSTR utf8_to_utf16(const char *z){ + int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); + LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); + if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) + return rv; + sqlite3_free(rv); + return 0; +} +#endif + +/* +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. +*/ +static void statTimesToUtc( + const char *zPath, + struct stat *pStatBuf +){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(zUnicodeName, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + sqlite3_free(zUnicodeName); + } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = stat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return stat(zPath, pStatBuf); +#endif +} + +/* +** This function is used in place of lstat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls lstat(). +*/ +static int fileLinkStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = lstat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return lstat(zPath, pStatBuf); +#endif +} + +/* +** Argument zFile is the name of a file that will be created and/or written +** by SQL function writefile(). This function ensures that the directory +** zFile will be written to exists, creating it if required. The permissions +** for any path components created by this function are set in accordance +** with the current umask. +** +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, +** SQLITE_OK is returned if the directory is successfully created, or +** SQLITE_ERROR otherwise. +*/ +static int makeDirectory( + const char *zFile +){ + char *zCopy = sqlite3_mprintf("%s", zFile); + int rc = SQLITE_OK; + + if( zCopy==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nCopy = (int)strlen(zCopy); + int i = 1; + + while( rc==SQLITE_OK ){ + struct stat sStat; + int rc2; + + for(; zCopy[i]!='/' && i=0 ){ +#if defined(_WIN32) +#if !SQLITE_OS_WINRT + /* Windows */ + FILETIME lastAccess; + FILETIME lastWrite; + SYSTEMTIME currentTime; + LONGLONG intervals; + HANDLE hFile; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + + GetSystemTime(¤tTime); + SystemTimeToFileTime(¤tTime, &lastAccess); + intervals = Int32x32To64(mtime, 10000000) + 116444736000000000; + lastWrite.dwLowDateTime = (DWORD)intervals; + lastWrite.dwHighDateTime = intervals >> 32; + zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); + if( zUnicodeName==0 ){ + return 1; + } + hFile = CreateFileW( + zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL + ); + sqlite3_free(zUnicodeName); + if( hFile!=INVALID_HANDLE_VALUE ){ + BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite); + CloseHandle(hFile); + return !bResult; + }else{ + return 1; + } +#endif +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ + /* Recent unix */ + struct timespec times[2]; + times[0].tv_nsec = times[1].tv_nsec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){ + return 1; + } +#else + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } + } +#endif + } + + return 0; +} + +/* +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. +** Refer to header comments at the top of this file for details. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zFile; + mode_t mode = 0; + int res; + sqlite3_int64 mtime = -1; + + if( argc<2 || argc>4 ){ + sqlite3_result_error(context, + "wrong number of arguments to function writefile()", -1 + ); + return; + } + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + if( argc>=3 ){ + mode = (mode_t)sqlite3_value_int(argv[2]); + } + if( argc==4 ){ + mtime = sqlite3_value_int64(argv[3]); + } + + res = writeFile(context, zFile, argv[1], mode, mtime); + if( res==1 && errno==ENOENT ){ + if( makeDirectory(zFile)==SQLITE_OK ){ + res = writeFile(context, zFile, argv[1], mode, mtime); + } + } + + if( argc>2 && res!=0 ){ + if( S_ISLNK(mode) ){ + ctxErrorMsg(context, "failed to create symlink: %s", zFile); + }else if( S_ISDIR(mode) ){ + ctxErrorMsg(context, "failed to create directory: %s", zFile); + }else{ + ctxErrorMsg(context, "failed to write file: %s", zFile); + } + } +} + +/* +** SQL function: lsmode(MODE) +** +** Given a numberic st_mode from stat(), convert it into a human-readable +** text string in the style of "ls -l". +*/ +static void lsModeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int iMode = sqlite3_value_int(argv[0]); + char z[16]; + (void)argc; + if( S_ISLNK(iMode) ){ + z[0] = 'l'; + }else if( S_ISREG(iMode) ){ + z[0] = '-'; + }else if( S_ISDIR(iMode) ){ + z[0] = 'd'; + }else{ + z[0] = '?'; + } + for(i=0; i<3; i++){ + int m = (iMode >> ((2-i)*3)); + char *a = &z[1 + i*3]; + a[0] = (m & 0x4) ? 'r' : '-'; + a[1] = (m & 0x2) ? 'w' : '-'; + a[2] = (m & 0x1) ? 'x' : '-'; + } + z[10] = '\0'; + sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT); +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Cursor type for recursively iterating through a directory structure. +*/ +typedef struct fsdir_cursor fsdir_cursor; +typedef struct FsdirLevel FsdirLevel; + +struct FsdirLevel { + DIR *pDir; /* From opendir() */ + char *zDir; /* Name of directory (nul-terminated) */ +}; + +struct fsdir_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + + int nLvl; /* Number of entries in aLvl[] array */ + int iLvl; /* Index of current entry */ + FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ + + const char *zBase; + int nBase; + + struct stat sStat; /* Current lstat() results */ + char *zPath; /* Path to current entry */ + sqlite3_int64 iRowid; /* Current rowid */ +}; + +typedef struct fsdir_tab fsdir_tab; +struct fsdir_tab { + sqlite3_vtab base; /* Base class - must be first */ +}; + +/* +** Construct a new fsdir virtual table object. +*/ +static int fsdirConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fsdir_tab *pNew = 0; + int rc; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** This method is the destructor for fsdir vtab objects. +*/ +static int fsdirDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new fsdir_cursor object. +*/ +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + fsdir_cursor *pCur; + (void)p; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iLvl = -1; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset a cursor back to the state it was in when first returned +** by fsdirOpen(). +*/ +static void fsdirResetCursor(fsdir_cursor *pCur){ + int i; + for(i=0; i<=pCur->iLvl; i++){ + FsdirLevel *pLvl = &pCur->aLvl[i]; + if( pLvl->pDir ) closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + } + sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); + pCur->aLvl = 0; + pCur->zPath = 0; + pCur->zBase = 0; + pCur->nBase = 0; + pCur->nLvl = 0; + pCur->iLvl = -1; + pCur->iRowid = 1; +} + +/* +** Destructor for an fsdir_cursor. +*/ +static int fsdirClose(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + + fsdirResetCursor(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Set the error message for the virtual table associated with cursor +** pCur to the results of vprintf(zFmt, ...). +*/ +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + + +/* +** Advance an fsdir_cursor to its next row of output. +*/ +static int fsdirNext(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + mode_t m = pCur->sStat.st_mode; + + pCur->iRowid++; + if( S_ISDIR(m) ){ + /* Descend into this directory */ + int iNew = pCur->iLvl + 1; + FsdirLevel *pLvl; + if( iNew>=pCur->nLvl ){ + int nNew = iNew+1; + sqlite3_int64 nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc64(pCur->aLvl, nByte); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); + pCur->aLvl = aNew; + pCur->nLvl = nNew; + } + pCur->iLvl = iNew; + pLvl = &pCur->aLvl[iNew]; + + pLvl->zDir = pCur->zPath; + pCur->zPath = 0; + pLvl->pDir = opendir(pLvl->zDir); + if( pLvl->pDir==0 ){ + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + return SQLITE_ERROR; + } + } + + while( pCur->iLvl>=0 ){ + FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl]; + struct dirent *pEntry = readdir(pLvl->pDir); + if( pEntry ){ + if( pEntry->d_name[0]=='.' ){ + if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue; + if( pEntry->d_name[1]=='\0' ) continue; + } + sqlite3_free(pCur->zPath); + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + pLvl->pDir = 0; + pLvl->zDir = 0; + pCur->iLvl--; + } + + /* EOF */ + sqlite3_free(pCur->zPath); + pCur->zPath = 0; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int fsdirColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + switch( i ){ + case FSDIR_COLUMN_NAME: { + sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); + break; + } + + case FSDIR_COLUMN_MODE: + sqlite3_result_int64(ctx, pCur->sStat.st_mode); + break; + + case FSDIR_COLUMN_MTIME: + sqlite3_result_int64(ctx, pCur->sStat.st_mtime); + break; + + case FSDIR_COLUMN_DATA: { + mode_t m = pCur->sStat.st_mode; + if( S_ISDIR(m) ){ + sqlite3_result_null(ctx); +#if !defined(_WIN32) && !defined(WIN32) + }else if( S_ISLNK(m) ){ + char aStatic[64]; + char *aBuf = aStatic; + sqlite3_int64 nBuf = 64; + int n; + + while( 1 ){ + n = readlink(pCur->zPath, aBuf, nBuf); + if( nzPath); + } + } + case FSDIR_COLUMN_PATH: + default: { + /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. + ** always return their values as NULL */ + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fsdirEof(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + return (pCur->zPath==0); +} + +/* +** xFilter callback. +** +** idxNum==1 PATH parameter only +** idxNum==2 Both PATH and DIR supplied +*/ +static int fsdirFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + const char *zDir = 0; + fsdir_cursor *pCur = (fsdir_cursor*)cur; + (void)idxStr; + fsdirResetCursor(pCur); + + if( idxNum==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); + return SQLITE_ERROR; + } + + assert( argc==idxNum && (argc==1 || argc==2) ); + zDir = (const char*)sqlite3_value_text(argv[0]); + if( zDir==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); + return SQLITE_ERROR; + } + if( argc==2 ){ + pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + } + if( pCur->zBase ){ + pCur->nBase = (int)strlen(pCur->zBase)+1; + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", zDir); + } + + if( pCur->zPath==0 ){ + return SQLITE_NOMEM; + } + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by values of idxNum: +** +** (1) The path value is supplied by argv[0] +** (2) Path is in argv[0] and dir is in argv[1] +*/ +static int fsdirBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ + int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int seenPath = 0; /* True if an unusable PATH= constraint is seen */ + int seenDir = 0; /* True if an unusable DIR= constraint is seen */ + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; + } + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + } + } + if( seenPath || seenDir ){ + /* If input parameters are unusable, disallow this plan */ + return SQLITE_CONSTRAINT; + } + + if( idxPath<0 ){ + pIdxInfo->idxNum = 0; + /* The pIdxInfo->estimatedCost should have been initialized to a huge + ** number. Leave it unchanged. */ + pIdxInfo->estimatedRows = 0x7fffffff; + }else{ + pIdxInfo->aConstraintUsage[idxPath].omit = 1; + pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + if( idxDir>=0 ){ + pIdxInfo->aConstraintUsage[idxDir].omit = 1; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; + pIdxInfo->idxNum = 2; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 100.0; + } + } + + return SQLITE_OK; +} + +/* +** Register the "fsdir" virtual table. +*/ +static int fsdirRegister(sqlite3 *db){ + static sqlite3_module fsdirModule = { + 0, /* iVersion */ + 0, /* xCreate */ + fsdirConnect, /* xConnect */ + fsdirBestIndex, /* xBestIndex */ + fsdirDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + fsdirOpen, /* xOpen - open a cursor */ + fsdirClose, /* xClose - close a cursor */ + fsdirFilter, /* xFilter - configure scan constraints */ + fsdirNext, /* xNext - advance a cursor */ + fsdirEof, /* xEof - check for end of scan */ + fsdirColumn, /* xColumn - read data */ + fsdirRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + + int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define fsdirRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 + +#endif +int sqlite3_fileio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readfile", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writefile", -1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + writefileFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0, + lsModeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = fsdirRegister(db); + } + return rc; +} + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +/* To allow a standalone DLL, make test_windirent.c use the same + * redefined SQLite API calls as the above extension code does. + * Just pull in this .c to accomplish this. As a beneficial side + * effect, this extension becomes a single translation unit. */ +# include "test_windirent.c" +#endif + +/************************* End ../ext/misc/fileio.c ********************/ +/************************* Begin ../ext/misc/completion.c ******************/ +/* +** 2017-07-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an eponymous virtual table that returns suggested +** completions for a partial SQL input. +** +** Suggested usage: +** +** SELECT DISTINCT candidate COLLATE nocase +** FROM completion($prefix,$wholeline) +** ORDER BY 1; +** +** The two query parameters are optional. $prefix is the text of the +** current word being typed and that is to be completed. $wholeline is +** the complete input line, used for context. +** +** The raw completion() table might return the same candidate multiple +** times, for example if the same column name is used to two or more +** tables. And the candidates are returned in an arbitrary order. Hence, +** the DISTINCT and ORDER BY are recommended. +** +** This virtual table operates at the speed of human typing, and so there +** is no attempt to make it fast. Even a slow implementation will be much +** faster than any human can type. +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* completion_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a completion virtual table +*/ +typedef struct completion_vtab completion_vtab; +struct completion_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this completion vtab */ +}; + +/* completion_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct completion_cursor completion_cursor; +struct completion_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int nPrefix, nLine; /* Number of bytes in zPrefix and zLine */ + char *zPrefix; /* The prefix for the word we want to complete */ + char *zLine; /* The whole that we want to complete */ + const char *zCurrentRow; /* Current output row */ + int szRow; /* Length of the zCurrentRow string */ + sqlite3_stmt *pStmt; /* Current statement */ + sqlite3_int64 iRowid; /* The rowid */ + int ePhase; /* Current phase */ + int j; /* inter-phase counter */ +}; + +/* Values for ePhase: +*/ +#define COMPLETION_FIRST_PHASE 1 +#define COMPLETION_KEYWORDS 1 +#define COMPLETION_PRAGMAS 2 +#define COMPLETION_FUNCTIONS 3 +#define COMPLETION_COLLATIONS 4 +#define COMPLETION_INDEXES 5 +#define COMPLETION_TRIGGERS 6 +#define COMPLETION_DATABASES 7 +#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */ +#define COMPLETION_COLUMNS 9 +#define COMPLETION_MODULES 10 +#define COMPLETION_EOF 11 + +/* +** The completionConnect() method is invoked to create a new +** completion_vtab that describes the completion virtual table. +** +** Think of this routine as the constructor for completion_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the completion_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against completion will look like. +*/ +static int completionConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + completion_vtab *pNew; + int rc; + + (void)(pAux); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + (void)(argv); /* Unused parameter */ + (void)(pzErr); /* Unused parameter */ + +/* Column numbers */ +#define COMPLETION_COLUMN_CANDIDATE 0 /* Suggested completion of the input */ +#define COMPLETION_COLUMN_PREFIX 1 /* Prefix of the word to be completed */ +#define COMPLETION_COLUMN_WHOLELINE 2 /* Entire line seen so far */ +#define COMPLETION_COLUMN_PHASE 3 /* ePhase - used for debugging only */ + + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(" + " candidate TEXT," + " prefix TEXT HIDDEN," + " wholeline TEXT HIDDEN," + " phase INT HIDDEN" /* Used for debugging only */ + ")"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for completion_cursor objects. +*/ +static int completionDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new completion_cursor object. +*/ +static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + completion_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((completion_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset the completion_cursor. +*/ +static void completionCursorReset(completion_cursor *pCur){ + sqlite3_free(pCur->zPrefix); pCur->zPrefix = 0; pCur->nPrefix = 0; + sqlite3_free(pCur->zLine); pCur->zLine = 0; pCur->nLine = 0; + sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; + pCur->j = 0; +} + +/* +** Destructor for a completion_cursor. +*/ +static int completionClose(sqlite3_vtab_cursor *cur){ + completionCursorReset((completion_cursor*)cur); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Advance a completion_cursor to its next row of output. +** +** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object +** record the current state of the scan. This routine sets ->zCurrentRow +** to the current row of output and then returns. If no more rows remain, +** then ->ePhase is set to COMPLETION_EOF which will signal the virtual +** table that has reached the end of its scan. +** +** The current implementation just lists potential identifiers and +** keywords and filters them by zPrefix. Future enhancements should +** take zLine into account to try to restrict the set of identifiers and +** keywords based on what would be legal at the current point of input. +*/ +static int completionNext(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + int eNextPhase = 0; /* Next phase to try if current phase reaches end */ + int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + pCur->iRowid++; + while( pCur->ePhase!=COMPLETION_EOF ){ + switch( pCur->ePhase ){ + case COMPLETION_KEYWORDS: { + if( pCur->j >= sqlite3_keyword_count() ){ + pCur->zCurrentRow = 0; + pCur->ePhase = COMPLETION_DATABASES; + }else{ + sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow); + } + iCol = -1; + break; + } + case COMPLETION_DATABASES: { + if( pCur->pStmt==0 ){ + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, + &pCur->pStmt, 0); + } + iCol = 1; + eNextPhase = COMPLETION_TABLES; + break; + } + case COMPLETION_TABLES: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT name FROM \"%w\".sqlite_schema", + zSql, zSep, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_COLUMNS; + break; + } + case COMPLETION_COLUMNS: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" + " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" + " WHERE sm.type='table'", + zSql, zSep, zDb, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_EOF; + break; + } + } + if( iCol<0 ){ + /* This case is when the phase presets zCurrentRow */ + if( pCur->zCurrentRow==0 ) continue; + }else{ + if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){ + /* Extract the next row of content */ + pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol); + pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); + }else{ + /* When all rows are finished, advance to the next phase */ + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + pCur->ePhase = eNextPhase; + continue; + } + } + if( pCur->nPrefix==0 ) break; + if( pCur->nPrefix<=pCur->szRow + && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 + ){ + break; + } + } + + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the completion_cursor +** is currently pointing. +*/ +static int completionColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + completion_cursor *pCur = (completion_cursor*)cur; + switch( i ){ + case COMPLETION_COLUMN_CANDIDATE: { + sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PREFIX: { + sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_WHOLELINE: { + sqlite3_result_text(ctx, pCur->zLine, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PHASE: { + sqlite3_result_int(ctx, pCur->ePhase); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + completion_cursor *pCur = (completion_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int completionEof(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + return pCur->ePhase >= COMPLETION_EOF; +} + +/* +** This method is called to "rewind" the completion_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to completionColumn() or completionRowid() or +** completionEof(). +*/ +static int completionFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + completion_cursor *pCur = (completion_cursor *)pVtabCursor; + int iArg = 0; + (void)(idxStr); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + completionCursorReset(pCur); + if( idxNum & 1 ){ + pCur->nPrefix = sqlite3_value_bytes(argv[iArg]); + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + iArg = 1; + } + if( idxNum & 2 ){ + pCur->nLine = sqlite3_value_bytes(argv[iArg]); + if( pCur->nLine>0 ){ + pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zLine==0 ) return SQLITE_NOMEM; + } + } + if( pCur->zLine!=0 && pCur->zPrefix==0 ){ + int i = pCur->nLine; + while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ + i--; + } + pCur->nPrefix = pCur->nLine - i; + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + } + pCur->iRowid = 0; + pCur->ePhase = COMPLETION_FIRST_PHASE; + return completionNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the completion virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** There are two hidden parameters that act as arguments to the table-valued +** function: "prefix" and "wholeline". Bit 0 of idxNum is set if "prefix" +** is available and bit 1 is set if "wholeline" is available. +*/ +static int completionBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int prefixIdx = -1; /* Index of the start= constraint, or -1 if none */ + int wholelineIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that completeFilter() expects */ + const struct sqlite3_index_constraint *pConstraint; + + (void)(tab); /* Unused parameter */ + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case COMPLETION_COLUMN_PREFIX: + prefixIdx = i; + idxNum |= 1; + break; + case COMPLETION_COLUMN_WHOLELINE: + wholelineIdx = i; + idxNum |= 2; + break; + } + } + if( prefixIdx>=0 ){ + pIdxInfo->aConstraintUsage[prefixIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[prefixIdx].omit = 1; + } + if( wholelineIdx>=0 ){ + pIdxInfo->aConstraintUsage[wholelineIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[wholelineIdx].omit = 1; + } + pIdxInfo->idxNum = idxNum; + pIdxInfo->estimatedCost = (double)5000 - 1000*nArg; + pIdxInfo->estimatedRows = 500 - 100*nArg; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** completion virtual table. +*/ +static sqlite3_module completionModule = { + 0, /* iVersion */ + 0, /* xCreate */ + completionConnect, /* xConnect */ + completionBestIndex, /* xBestIndex */ + completionDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + completionOpen, /* xOpen - open a cursor */ + completionClose, /* xClose - close a cursor */ + completionFilter, /* xFilter - configure scan constraints */ + completionNext, /* xNext - advance a cursor */ + completionEof, /* xEof - check for end of scan */ + completionColumn, /* xColumn - read data */ + completionRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3CompletionVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "completion", &completionModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 + +#endif +int sqlite3_completion_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)(pzErrMsg); /* Unused parameter */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3CompletionVtabInit(db); +#endif + return rc; +} + +/************************* End ../ext/misc/completion.c ********************/ +/************************* Begin ../ext/misc/appendvfs.c ******************/ +/* +** 2017-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that allows an SQLite database to be +** appended onto the end of some other file, such as an executable. +** +** A special record must appear at the end of the file that identifies the +** file as an appended database and provides the offset to the first page +** of the exposed content. (Or, it is the length of the content prefix.) +** For best performance page 1 should be located at a disk page boundary, +** though that is not required. +** +** When opening a database using this VFS, the connection might treat +** the file as an ordinary SQLite database, or it might treat it as a +** database appended onto some other file. The decision is made by +** applying the following rules in order: +** +** (1) An empty file is an ordinary database. +** +** (2) If the file ends with the appendvfs trailer string +** "Start-Of-SQLite3-NNNNNNNN" that file is an appended database. +** +** (3) If the file begins with the standard SQLite prefix string +** "SQLite format 3", that file is an ordinary database. +** +** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is +** set, then a new database is appended to the already existing file. +** +** (5) Otherwise, SQLITE_CANTOPEN is returned. +** +** To avoid unnecessary complications with the PENDING_BYTE, the size of +** the file containing the database is limited to 1GiB. (1073741824 bytes) +** This VFS will not read or write past the 1GiB mark. This restriction +** might be lifted in future versions. For now, if you need a larger +** database, then keep it in a separate file. +** +** If the file being opened is a plain database (not an appended one), then +** this shim is a pass-through into the default underlying VFS. (rule 3) +**/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* The append mark at the end of the database is: +** +** Start-Of-SQLite3-NNNNNNNN +** 123456789 123456789 12345 +** +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is +** the offset to page 1, and also the length of the prefix content. +*/ +#define APND_MARK_PREFIX "Start-Of-SQLite3-" +#define APND_MARK_PREFIX_SZ 17 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) + +/* +** Maximum size of the combined prefix + database + append-mark. This +** must be less than 0x40000000 to avoid locking issues on Windows. +*/ +#define APND_MAX_SIZE (0x40000000) + +/* +** Try to align the database to an even multiple of APND_ROUNDUP bytes. +*/ +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 +#endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs ApndVfs; +typedef struct ApndFile ApndFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) + +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ +struct ApndFile { + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ +}; + +/* +** Methods for ApndFile +*/ +static int apndClose(sqlite3_file*); +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int apndTruncate(sqlite3_file*, sqlite3_int64 size); +static int apndSync(sqlite3_file*, int flags); +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int apndLock(sqlite3_file*, int); +static int apndUnlock(sqlite3_file*, int); +static int apndCheckReservedLock(sqlite3_file*, int *pResOut); +static int apndFileControl(sqlite3_file*, int op, void *pArg); +static int apndSectorSize(sqlite3_file*); +static int apndDeviceCharacteristics(sqlite3_file*); +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int apndShmLock(sqlite3_file*, int offset, int n, int flags); +static void apndShmBarrier(sqlite3_file*); +static int apndShmUnmap(sqlite3_file*, int deleteFlag); +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for ApndVfs +*/ +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void apndDlClose(sqlite3_vfs*, void*); +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int apndSleep(sqlite3_vfs*, int microseconds); +static int apndCurrentTime(sqlite3_vfs*, double*); +static int apndGetLastError(sqlite3_vfs*, int, char *); +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs apnd_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "apndvfs", /* zName */ + 0, /* pAppData (set when registered) */ + apndOpen, /* xOpen */ + apndDelete, /* xDelete */ + apndAccess, /* xAccess */ + apndFullPathname, /* xFullPathname */ + apndDlOpen, /* xDlOpen */ + apndDlError, /* xDlError */ + apndDlSym, /* xDlSym */ + apndDlClose, /* xDlClose */ + apndRandomness, /* xRandomness */ + apndSleep, /* xSleep */ + apndCurrentTime, /* xCurrentTime */ + apndGetLastError, /* xGetLastError */ + apndCurrentTimeInt64, /* xCurrentTimeInt64 */ + apndSetSystemCall, /* xSetSystemCall */ + apndGetSystemCall, /* xGetSystemCall */ + apndNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods apnd_io_methods = { + 3, /* iVersion */ + apndClose, /* xClose */ + apndRead, /* xRead */ + apndWrite, /* xWrite */ + apndTruncate, /* xTruncate */ + apndSync, /* xSync */ + apndFileSize, /* xFileSize */ + apndLock, /* xLock */ + apndUnlock, /* xUnlock */ + apndCheckReservedLock, /* xCheckReservedLock */ + apndFileControl, /* xFileControl */ + apndSectorSize, /* xSectorSize */ + apndDeviceCharacteristics, /* xDeviceCharacteristics */ + apndShmMap, /* xShmMap */ + apndShmLock, /* xShmLock */ + apndShmBarrier, /* xShmBarrier */ + apndShmUnmap, /* xShmUnmap */ + apndFetch, /* xFetch */ + apndUnfetch /* xUnfetch */ +}; + +/* +** Close an apnd-file. +*/ +static int apndClose(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from an apnd-file. +*/ +static int apndRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. +*/ +static int apndWriteMark( + ApndFile *paf, + sqlite3_file *pFile, + sqlite_int64 iWriteEnd +){ + sqlite_int64 iPgOne = paf->iPgOne; + unsigned char a[APND_MARK_SIZE]; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); + memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); + while( --i >= 0 ){ + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; + } + iWriteEnd += paf->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; + } + return rc; +} + +/* +** Write data to an apnd-file. +*/ +static int apndWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; + pFile = ORIGFILE(pFile); + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); + if( SQLITE_OK!=rc ) return rc; + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Truncate an apnd-file. +*/ +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); +} + +/* +** Sync an apnd-file. +*/ +static int apndSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. +*/ +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; +} + +/* +** Lock an apnd-file. +*/ +static int apndLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock an apnd-file. +*/ +static int apndUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an apnd-file. +*/ +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on an apnd-file. +*/ +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ + ApndFile *paf = (ApndFile *)pFile; + int rc; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_SIZE_HINT ) *(sqlite3_int64*)pArg += paf->iPgOne; + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne,*(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an apnd-file. +*/ +static int apndSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by an apnd-file. +*/ +static int apndDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int apndShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void apndShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int apndFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ + return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); +} + +/* +** Try to read the append-mark off the end of a file. Return the +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; +** +** An append-mark is only valid if the NNNNNNNN start-of-database offset +** indicates that the appended database contains at least one page. The +** start-of-database value must be a multiple of 512. +*/ +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc, i; + sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); + unsigned char a[APND_MARK_SIZE]; + + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc ) return -1; + if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]< (sz - APND_MARK_SIZE - 512) ) return -1; + if( iMark & 0x1ff ) return -1; + return iMark; +} + +static const char apvfsSqliteHdr[] = "SQLite format 3"; +/* +** Check to see if the file is an appendvfs SQLite database file. +** Return true iff it is such. Parameter sz is the file's size. +*/ +static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + sqlite3_int64 iMark = apndReadMark(sz, pFile); + if( iMark>=0 ){ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ + return 1; /* It's an appendvfs database */ + } + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 + ){ + return 0; + }else{ + return 1; + } +} + +/* +** Open an apnd file handle. +*/ +static int apndOpen( + sqlite3_vfs *pApndVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); + int rc; + sqlite3_int64 sz = 0; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + /* The appendvfs is not to be used for transient or temporary databases. + ** Just use the base VFS open to initialize the given file object and + ** open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); + } + memset(pApndFile, 0, sizeof(ApndFile)); + pFile->pMethods = &apnd_io_methods; + pApndFile->iMark = -1; /* Append mark not yet written */ + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + if( rc ){ + pBaseFile->pMethods->xClose(pBaseFile); + } + } + if( rc ){ + pFile->pMethods = 0; + return rc; + } + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + ** the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); + return SQLITE_OK; + } + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ + return SQLITE_OK; + } + if( (flags & SQLITE_OPEN_CREATE)==0 ){ + pBaseFile->pMethods->xClose(pBaseFile); + rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); + } + return rc; +} + +/* +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int apndAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int apndFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + + +#ifdef _WIN32 + +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + (void)db; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 0); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} + +/************************* End ../ext/misc/appendvfs.c ********************/ +#endif +#ifdef SQLITE_HAVE_ZLIB +/************************* Begin ../ext/misc/zipfile.c ******************/ +/* +** 2017-12-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a virtual table for reading and writing ZIP archive +** files. +** +** Usage example: +** +** SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename); +** +** Current limitations: +** +** * No support for encryption +** * No support for ZIP archives spanning multiple files +** * No support for zip64 extensions +** * Only the "inflate/deflate" (zlib) compression method is supported +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#ifndef SQLITE_NO_STDINT +# include +#endif + +#include + +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +#ifndef SQLITE_AMALGAMATION + +#ifndef UINT32_TYPE +# ifdef HAVE_UINT32_T +# define UINT32_TYPE uint32_t +# else +# define UINT32_TYPE unsigned int +# endif +#endif +#ifndef UINT16_TYPE +# ifdef HAVE_UINT16_T +# define UINT16_TYPE uint16_t +# else +# define UINT16_TYPE unsigned short int +# endif +#endif +/* typedef sqlite3_int64 i64; */ +/* typedef unsigned char u8; */ +/* typedef UINT32_TYPE u32; // 4-byte unsigned integer // */ +/* typedef UINT16_TYPE u16; // 2-byte unsigned integer // */ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +/* +** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. +** +** In some ways it would be better to obtain these values from system +** header files. But, the dependency is undesirable and (a) these +** have been stable for decades, (b) the values are part of POSIX and +** are also made explicit in [man stat], and (c) are part of the +** file format for zip archives. +*/ +#ifndef S_IFDIR +# define S_IFDIR 0040000 +#endif +#ifndef S_IFREG +# define S_IFREG 0100000 +#endif +#ifndef S_IFLNK +# define S_IFLNK 0120000 +#endif + +static const char ZIPFILE_SCHEMA[] = + "CREATE TABLE y(" + "name PRIMARY KEY," /* 0: Name of file in zip archive */ + "mode," /* 1: POSIX mode for file */ + "mtime," /* 2: Last modification time (secs since 1970)*/ + "sz," /* 3: Size of object */ + "rawdata," /* 4: Raw data */ + "data," /* 5: Uncompressed data */ + "method," /* 6: Compression method (integer) */ + "z HIDDEN" /* 7: Name of zip file */ + ") WITHOUT ROWID;"; + +#define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */ +#define ZIPFILE_BUFFER_SIZE (64*1024) + + +/* +** Magic numbers used to read and write zip files. +** +** ZIPFILE_NEWENTRY_MADEBY: +** Use this value for the "version-made-by" field in new zip file +** entries. The upper byte indicates "unix", and the lower byte +** indicates that the zip file matches pkzip specification 3.0. +** This is what info-zip seems to do. +** +** ZIPFILE_NEWENTRY_REQUIRED: +** Value for "version-required-to-extract" field of new entries. +** Version 2.0 is required to support folders and deflate compression. +** +** ZIPFILE_NEWENTRY_FLAGS: +** Value for "general-purpose-bit-flags" field of new entries. Bit +** 11 means "utf-8 filename and comment". +** +** ZIPFILE_SIGNATURE_CDS: +** First 4 bytes of a valid CDS record. +** +** ZIPFILE_SIGNATURE_LFH: +** First 4 bytes of a valid LFH record. +** +** ZIPFILE_SIGNATURE_EOCD +** First 4 bytes of a valid EOCD record. +*/ +#define ZIPFILE_EXTRA_TIMESTAMP 0x5455 +#define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) +#define ZIPFILE_NEWENTRY_REQUIRED 20 +#define ZIPFILE_NEWENTRY_FLAGS 0x800 +#define ZIPFILE_SIGNATURE_CDS 0x02014b50 +#define ZIPFILE_SIGNATURE_LFH 0x04034b50 +#define ZIPFILE_SIGNATURE_EOCD 0x06054b50 + +/* +** The sizes of the fixed-size part of each of the three main data +** structures in a zip archive. +*/ +#define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 +#define ZIPFILE_CDS_FIXED_SZ 46 + +/* +*** 4.3.16 End of central directory record: +*** +*** end of central dir signature 4 bytes (0x06054b50) +*** number of this disk 2 bytes +*** number of the disk with the +*** start of the central directory 2 bytes +*** total number of entries in the +*** central directory on this disk 2 bytes +*** total number of entries in +*** the central directory 2 bytes +*** size of the central directory 4 bytes +*** offset of start of central +*** directory with respect to +*** the starting disk number 4 bytes +*** .ZIP file comment length 2 bytes +*** .ZIP file comment (variable size) +*/ +typedef struct ZipfileEOCD ZipfileEOCD; +struct ZipfileEOCD { + u16 iDisk; + u16 iFirstDisk; + u16 nEntry; + u16 nEntryTotal; + u32 nSize; + u32 iOffset; +}; + +/* +*** 4.3.12 Central directory structure: +*** +*** ... +*** +*** central file header signature 4 bytes (0x02014b50) +*** version made by 2 bytes +*** version needed to extract 2 bytes +*** general purpose bit flag 2 bytes +*** compression method 2 bytes +*** last mod file time 2 bytes +*** last mod file date 2 bytes +*** crc-32 4 bytes +*** compressed size 4 bytes +*** uncompressed size 4 bytes +*** file name length 2 bytes +*** extra field length 2 bytes +*** file comment length 2 bytes +*** disk number start 2 bytes +*** internal file attributes 2 bytes +*** external file attributes 4 bytes +*** relative offset of local header 4 bytes +*/ +typedef struct ZipfileCDS ZipfileCDS; +struct ZipfileCDS { + u16 iVersionMadeBy; + u16 iVersionExtract; + u16 flags; + u16 iCompression; + u16 mTime; + u16 mDate; + u32 crc32; + u32 szCompressed; + u32 szUncompressed; + u16 nFile; + u16 nExtra; + u16 nComment; + u16 iDiskStart; + u16 iInternalAttr; + u32 iExternalAttr; + u32 iOffset; + char *zFile; /* Filename (sqlite3_malloc()) */ +}; + +/* +*** 4.3.7 Local file header: +*** +*** local file header signature 4 bytes (0x04034b50) +*** version needed to extract 2 bytes +*** general purpose bit flag 2 bytes +*** compression method 2 bytes +*** last mod file time 2 bytes +*** last mod file date 2 bytes +*** crc-32 4 bytes +*** compressed size 4 bytes +*** uncompressed size 4 bytes +*** file name length 2 bytes +*** extra field length 2 bytes +*** +*/ +typedef struct ZipfileLFH ZipfileLFH; +struct ZipfileLFH { + u16 iVersionExtract; + u16 flags; + u16 iCompression; + u16 mTime; + u16 mDate; + u32 crc32; + u32 szCompressed; + u32 szUncompressed; + u16 nFile; + u16 nExtra; +}; + +typedef struct ZipfileEntry ZipfileEntry; +struct ZipfileEntry { + ZipfileCDS cds; /* Parsed CDS record */ + u32 mUnixTime; /* Modification time, in UNIX format */ + u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ + i64 iDataOff; /* Offset to data in file (if aData==0) */ + u8 *aData; /* cds.szCompressed bytes of compressed data */ + ZipfileEntry *pNext; /* Next element in in-memory CDS */ +}; + +/* +** Cursor type for zipfile tables. +*/ +typedef struct ZipfileCsr ZipfileCsr; +struct ZipfileCsr { + sqlite3_vtab_cursor base; /* Base class - must be first */ + i64 iId; /* Cursor ID */ + u8 bEof; /* True when at EOF */ + u8 bNoop; /* If next xNext() call is no-op */ + + /* Used outside of write transactions */ + FILE *pFile; /* Zip file */ + i64 iNextOff; /* Offset of next record in central directory */ + ZipfileEOCD eocd; /* Parse of central directory record */ + + ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ + ZipfileEntry *pCurrent; /* Current entry */ + ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ +}; + +typedef struct ZipfileTab ZipfileTab; +struct ZipfileTab { + sqlite3_vtab base; /* Base class - must be first */ + char *zFile; /* Zip file this table accesses (may be NULL) */ + sqlite3 *db; /* Host database connection */ + u8 *aBuffer; /* Temporary buffer used for various tasks */ + + ZipfileCsr *pCsrList; /* List of cursors */ + i64 iNextCsrid; + + /* The following are used by write transactions only */ + ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */ + ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */ + FILE *pWriteFd; /* File handle open on zip archive */ + i64 szCurrent; /* Current size of zip archive */ + i64 szOrig; /* Size of archive at start of transaction */ +}; + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +/* +** If string zIn is quoted, dequote it in place. Otherwise, if the string +** is not quoted, do nothing. +*/ +static void zipfileDequote(char *zIn){ + char q = zIn[0]; + if( q=='"' || q=='\'' || q=='`' || q=='[' ){ + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + while( ALWAYS(zIn[iIn]) ){ + char c = zIn[iIn++]; + if( c==q && zIn[iIn++]!=q ) break; + zIn[iOut++] = c; + } + zIn[iOut] = '\0'; + } +} + +/* +** Construct a new ZipfileTab virtual table object. +** +** argv[0] -> module name ("zipfile") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int zipfileConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; + int nFile = 0; + const char *zFile = 0; + ZipfileTab *pNew = 0; + int rc; + (void)pAux; + + /* If the table name is not "zipfile", require that the argument be + ** specified. This stops zipfile tables from being created as: + ** + ** CREATE VIRTUAL TABLE zzz USING zipfile(); + ** + ** It does not prevent: + ** + ** CREATE VIRTUAL TABLE zipfile USING zipfile(); + */ + assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); + if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ + *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); + return SQLITE_ERROR; + } + + if( argc>3 ){ + zFile = argv[3]; + nFile = (int)strlen(zFile)+1; + } + + rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, nByte+nFile); + pNew->db = db; + pNew->aBuffer = (u8*)&pNew[1]; + if( zFile ){ + pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; + memcpy(pNew->zFile, zFile, nFile); + zipfileDequote(pNew->zFile); + } + } + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** Free the ZipfileEntry structure indicated by the only argument. +*/ +static void zipfileEntryFree(ZipfileEntry *p){ + if( p ){ + sqlite3_free(p->cds.zFile); + sqlite3_free(p); + } +} + +/* +** Release resources that should be freed at the end of a write +** transaction. +*/ +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + if( pTab->pWriteFd ){ + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; + } + for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + zipfileEntryFree(pEntry); + } + pTab->pFirstEntry = 0; + pTab->pLastEntry = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; +} + +/* +** This method is the destructor for zipfile vtab objects. +*/ +static int zipfileDisconnect(sqlite3_vtab *pVtab){ + zipfileCleanupTransaction((ZipfileTab*)pVtab); + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new ZipfileCsr object. +*/ +static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + ZipfileTab *pTab = (ZipfileTab*)p; + ZipfileCsr *pCsr; + pCsr = sqlite3_malloc(sizeof(*pCsr)); + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(*pCsr)); + pCsr->iId = ++pTab->iNextCsrid; + pCsr->pCsrNext = pTab->pCsrList; + pTab->pCsrList = pCsr; + return SQLITE_OK; +} + +/* +** Reset a cursor back to the state it was in when first returned +** by zipfileOpen(). +*/ +static void zipfileResetCursor(ZipfileCsr *pCsr){ + ZipfileEntry *p; + ZipfileEntry *pNext; + + pCsr->bEof = 0; + if( pCsr->pFile ){ + fclose(pCsr->pFile); + pCsr->pFile = 0; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + } + + for(p=pCsr->pFreeEntry; p; p=pNext){ + pNext = p->pNext; + zipfileEntryFree(p); + } +} + +/* +** Destructor for an ZipfileCsr. +*/ +static int zipfileClose(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab); + ZipfileCsr **pp; + zipfileResetCursor(pCsr); + + /* Remove this cursor from the ZipfileTab.pCsrList list. */ + for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); + *pp = pCsr->pCsrNext; + + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Set the error message for the virtual table associated with cursor +** pCsr to the results of vprintf(zFmt, ...). +*/ +static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} +static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + sqlite3_free(pCsr->base.pVtab->zErrMsg); + pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + +/* +** Read nRead bytes of data from offset iOff of file pFile into buffer +** aRead[]. Return SQLITE_OK if successful, or an SQLite error code +** otherwise. +** +** If an error does occur, output variable (*pzErrmsg) may be set to point +** to an English language error message. It is the responsibility of the +** caller to eventually free this buffer using +** sqlite3_free(). +*/ +static int zipfileReadData( + FILE *pFile, /* Read from this file */ + u8 *aRead, /* Read into this buffer */ + int nRead, /* Number of bytes to read */ + i64 iOff, /* Offset to read from */ + char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ +){ + size_t n; + fseek(pFile, (long)iOff, SEEK_SET); + n = fread(aRead, 1, nRead, pFile); + if( (int)n!=nRead ){ + *pzErrmsg = sqlite3_mprintf("error in fread()"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +static int zipfileAppendData( + ZipfileTab *pTab, + const u8 *aWrite, + int nWrite +){ + if( nWrite>0 ){ + size_t n = nWrite; + fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); + n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); + if( (int)n!=nWrite ){ + pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); + return SQLITE_ERROR; + } + pTab->szCurrent += nWrite; + } + return SQLITE_OK; +} + +/* +** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +*/ +static u16 zipfileGetU16(const u8 *aBuf){ + return (aBuf[1] << 8) + aBuf[0]; +} + +/* +** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +*/ +static u32 zipfileGetU32(const u8 *aBuf){ + if( aBuf==0 ) return 0; + return ((u32)(aBuf[3]) << 24) + + ((u32)(aBuf[2]) << 16) + + ((u32)(aBuf[1]) << 8) + + ((u32)(aBuf[0]) << 0); +} + +/* +** Write a 16-bit little endiate integer into buffer aBuf. +*/ +static void zipfilePutU16(u8 *aBuf, u16 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; +} + +/* +** Write a 32-bit little endiate integer into buffer aBuf. +*/ +static void zipfilePutU32(u8 *aBuf, u32 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; + aBuf[2] = (val>>16) & 0xFF; + aBuf[3] = (val>>24) & 0xFF; +} + +#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) ) +#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) ) + +#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } +#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } + +/* +** Magic numbers used to read CDS records. +*/ +#define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 + +/* +** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ + u8 *aRead = aBuf; + u32 sig = zipfileRead32(aRead); + int rc = SQLITE_OK; + if( sig!=ZIPFILE_SIGNATURE_CDS ){ + rc = SQLITE_ERROR; + }else{ + pCDS->iVersionMadeBy = zipfileRead16(aRead); + pCDS->iVersionExtract = zipfileRead16(aRead); + pCDS->flags = zipfileRead16(aRead); + pCDS->iCompression = zipfileRead16(aRead); + pCDS->mTime = zipfileRead16(aRead); + pCDS->mDate = zipfileRead16(aRead); + pCDS->crc32 = zipfileRead32(aRead); + pCDS->szCompressed = zipfileRead32(aRead); + pCDS->szUncompressed = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + pCDS->nFile = zipfileRead16(aRead); + pCDS->nExtra = zipfileRead16(aRead); + pCDS->nComment = zipfileRead16(aRead); + pCDS->iDiskStart = zipfileRead16(aRead); + pCDS->iInternalAttr = zipfileRead16(aRead); + pCDS->iExternalAttr = zipfileRead32(aRead); + pCDS->iOffset = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] ); + } + + return rc; +} + +/* +** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadLFH( + u8 *aBuffer, + ZipfileLFH *pLFH +){ + u8 *aRead = aBuffer; + int rc = SQLITE_OK; + + u32 sig = zipfileRead32(aRead); + if( sig!=ZIPFILE_SIGNATURE_LFH ){ + rc = SQLITE_ERROR; + }else{ + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); + } + return rc; +} + + +/* +** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. +** Scan through this buffer to find an "extra-timestamp" field. If one +** exists, extract the 32-bit modification-timestamp from it and store +** the value in output parameter *pmTime. +** +** Zero is returned if no extra-timestamp record could be found (and so +** *pmTime is left unchanged), or non-zero otherwise. +** +** The general format of an extra field is: +** +** Header ID 2 bytes +** Data Size 2 bytes +** Data N bytes +*/ +static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ + int ret = 0; + u8 *p = aExtra; + u8 *pEnd = &aExtra[nExtra]; + + while( p modtime is present */ + *pmTime = zipfileGetU32(&p[1]); + ret = 1; + } + break; + } + } + + p += nByte; + } + return ret; +} + +/* +** Convert the standard MS-DOS timestamp stored in the mTime and mDate +** fields of the CDS structure passed as the only argument to a 32-bit +** UNIX seconds-since-the-epoch timestamp. Return the result. +** +** "Standard" MS-DOS time format: +** +** File modification time: +** Bits 00-04: seconds divided by 2 +** Bits 05-10: minute +** Bits 11-15: hour +** File modification date: +** Bits 00-04: day +** Bits 05-08: month (1-12) +** Bits 09-15: years from 1980 +** +** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx +*/ +static u32 zipfileMtime(ZipfileCDS *pCDS){ + int Y,M,D,X1,X2,A,B,sec,min,hr; + i64 JDsec; + Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); + M = ((pCDS->mDate >> 5) & 0x0F); + D = (pCDS->mDate & 0x1F); + sec = (pCDS->mTime & 0x1F)*2; + min = (pCDS->mTime >> 5) & 0x3F; + hr = (pCDS->mTime >> 11) & 0x1F; + if( M<=2 ){ + Y--; + M += 12; + } + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + A = Y/100; + B = 2 - A + (A/4); + JDsec = (i64)((X1 + X2 + D + B - 1524.5)*86400) + hr*3600 + min*60 + sec; + return (u32)(JDsec - (i64)24405875*(i64)8640); +} + +/* +** The opposite of zipfileMtime(). This function populates the mTime and +** mDate fields of the CDS structure passed as the first argument according +** to the UNIX timestamp value passed as the second. +*/ +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ + /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ + i64 JD = (i64)2440588 + mUnixTime / (24*60*60); + + int A, B, C, D, E; + int yr, mon, day; + int hr, min, sec; + + A = (int)((JD - 1867216.25)/36524.25); + A = (int)(JD + 1 + A - (A/4)); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); + + day = B - D - (int)(30.6001*E); + mon = (E<14 ? E-1 : E-13); + yr = mon>2 ? C-4716 : C-4715; + + hr = (mUnixTime % (24*60*60)) / (60*60); + min = (mUnixTime % (60*60)) / 60; + sec = (mUnixTime % 60); + + if( yr>=1980 ){ + pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); + pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); + }else{ + pCds->mDate = pCds->mTime = 0; + } + + assert( mUnixTime<315507600 + || mUnixTime==zipfileMtime(pCds) + || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) + /* || (mUnixTime % 2) */ + ); +} + +/* +** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in +** size) containing an entire zip archive image. Or, if aBlob is NULL, +** then pFile is a file-handle open on a zip file. In either case, this +** function creates a ZipfileEntry object based on the zip archive entry +** for which the CDS record is at offset iOff. +** +** If successful, SQLITE_OK is returned and (*ppEntry) set to point to +** the new object. Otherwise, an SQLite error code is returned and the +** final value of (*ppEntry) undefined. +*/ +static int zipfileGetEntry( + ZipfileTab *pTab, /* Store any error message here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* If aBlob==0, read from this file */ + i64 iOff, /* Offset of CDS record */ + ZipfileEntry **ppEntry /* OUT: Pointer to new object */ +){ + u8 *aRead; + char **pzErr = &pTab->base.zErrMsg; + int rc = SQLITE_OK; + (void)nBlob; + + if( aBlob==0 ){ + aRead = pTab->aBuffer; + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + }else{ + aRead = (u8*)&aBlob[iOff]; + } + + if( rc==SQLITE_OK ){ + sqlite3_int64 nAlloc; + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + nAlloc = sizeof(ZipfileEntry) + nExtra; + if( aBlob ){ + nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + } + + pNew = (ZipfileEntry*)sqlite3_malloc64(nAlloc); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else if( aBlob==0 ){ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); + }else{ + aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; + } + } + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + pNew->aExtra = (u8*)&pNew[1]; + memcpy(pNew->aExtra, &aRead[nFile], nExtra); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc==SQLITE_OK ){ + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + ZipfileLFH lfh; + if( pFile ){ + rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); + }else{ + aRead = (u8*)&aBlob[pNew->cds.iOffset]; + } + + if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); + if( rc==SQLITE_OK ){ + pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff += lfh.nFile + lfh.nExtra; + if( aBlob && pNew->cds.szCompressed ){ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } + }else{ + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + (int)pNew->cds.iOffset + ); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); + }else{ + *ppEntry = pNew; + } + } + + return rc; +} + +/* +** Advance an ZipfileCsr to its next row of output. +*/ +static int zipfileNext(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + int rc = SQLITE_OK; + + if( pCsr->pFile ){ + i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + if( pCsr->iNextOff>=iEof ){ + pCsr->bEof = 1; + }else{ + ZipfileEntry *p = 0; + ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); + rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); + if( rc==SQLITE_OK ){ + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; + } + pCsr->pCurrent = p; + } + }else{ + if( !pCsr->bNoop ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + } + if( pCsr->pCurrent==0 ){ + pCsr->bEof = 1; + } + } + + pCsr->bNoop = 0; + return rc; +} + +static void zipfileFree(void *p) { + sqlite3_free(p); +} + +/* +** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the +** size is nOut bytes. This function uncompresses the data and sets the +** return value in context pCtx to the result (a blob). +** +** If an error occurs, an error code is left in pCtx instead. +*/ +static void zipfileInflate( + sqlite3_context *pCtx, /* Store result here */ + const u8 *aIn, /* Compressed data */ + int nIn, /* Size of buffer aIn[] in bytes */ + int nOut /* Expected output size */ +){ + u8 *aRes = sqlite3_malloc(nOut); + if( aRes==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + int err; + z_stream str; + memset(&str, 0, sizeof(str)); + + str.next_in = (Byte*)aIn; + str.avail_in = nIn; + str.next_out = (Byte*)aRes; + str.avail_out = nOut; + + err = inflateInit2(&str, -15); + if( err!=Z_OK ){ + zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err); + }else{ + err = inflate(&str, Z_NO_FLUSH); + if( err!=Z_STREAM_END ){ + zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); + }else{ + sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + aRes = 0; + } + } + sqlite3_free(aRes); + inflateEnd(&str); + } +} + +/* +** Buffer aIn (size nIn bytes) contains uncompressed data. This function +** compresses it and sets (*ppOut) to point to a buffer containing the +** compressed data. The caller is responsible for eventually calling +** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) +** is set to the size of buffer (*ppOut) in bytes. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned and an error message left in virtual-table handle +** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this +** case. +*/ +static int zipfileDeflate( + const u8 *aIn, int nIn, /* Input */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + sqlite3_int64 nAlloc; + z_stream str; + u8 *aOut; + + memset(&str, 0, sizeof(str)); + str.next_in = (Bytef*)aIn; + str.avail_in = nIn; + deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + + nAlloc = deflateBound(&str, nIn); + aOut = (u8*)sqlite3_malloc64(nAlloc); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + int res; + str.next_out = aOut; + str.avail_out = nAlloc; + res = deflate(&str, Z_FINISH); + if( res==Z_STREAM_END ){ + *ppOut = aOut; + *pnOut = (int)str.total_out; + }else{ + sqlite3_free(aOut); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); + rc = SQLITE_ERROR; + } + deflateEnd(&str); + } + + return rc; +} + + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int zipfileColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; + int rc = SQLITE_OK; + switch( i ){ + case 0: /* name */ + sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); + break; + case 1: /* mode */ + /* TODO: Whether or not the following is correct surely depends on + ** the platform on which the archive was created. */ + sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); + break; + case 2: { /* mtime */ + sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); + break; + } + case 3: { /* sz */ + if( sqlite3_vtab_nochange(ctx)==0 ){ + sqlite3_result_int64(ctx, pCDS->szUncompressed); + } + break; + } + case 4: /* rawdata */ + if( sqlite3_vtab_nochange(ctx) ) break; + case 5: { /* data */ + if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ + int sz = pCDS->szCompressed; + int szFinal = pCDS->szUncompressed; + if( szFinal>0 ){ + u8 *aBuf; + u8 *aFree = 0; + if( pCsr->pCurrent->aData ){ + aBuf = pCsr->pCurrent->aData; + }else{ + aBuf = aFree = sqlite3_malloc64(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = pCsr->pFile; + if( pFile==0 ){ + pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; + } + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } + } + if( rc==SQLITE_OK ){ + if( i==5 && pCDS->iCompression ){ + zipfileInflate(ctx, aBuf, sz, szFinal); + }else{ + sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); + } + } + sqlite3_free(aFree); + }else{ + /* Figure out if this is a directory or a zero-sized file. Consider + ** it to be a directory either if the mode suggests so, or if + ** the final character in the name is '/'. */ + u32 mode = pCDS->iExternalAttr >> 16; + if( !(mode & S_IFDIR) + && pCDS->nFile>=1 + && pCDS->zFile[pCDS->nFile-1]!='/' + ){ + sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); + } + } + } + break; + } + case 6: /* method */ + sqlite3_result_int(ctx, pCDS->iCompression); + break; + default: /* z */ + assert( i==7 ); + sqlite3_result_int64(ctx, pCsr->iId); + break; + } + + return rc; +} + +/* +** Return TRUE if the cursor is at EOF. +*/ +static int zipfileEof(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + return pCsr->bEof; +} + +/* +** If aBlob is not NULL, then it points to a buffer nBlob bytes in size +** containing an entire zip archive image. Or, if aBlob is NULL, then pFile +** is guaranteed to be a file-handle open on a zip file. +** +** This function attempts to locate the EOCD record within the zip archive +** and populate *pEOCD with the results of decoding it. SQLITE_OK is +** returned if successful. Otherwise, an SQLite error code is returned and +** an English language error message may be left in virtual-table pTab. +*/ +static int zipfileReadEOCD( + ZipfileTab *pTab, /* Return errors here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* Read from this file if aBlob==0 */ + ZipfileEOCD *pEOCD /* Object to populate */ +){ + u8 *aRead = pTab->aBuffer; /* Temporary buffer */ + int nRead; /* Bytes to read from file */ + int rc = SQLITE_OK; + + memset(pEOCD, 0, sizeof(ZipfileEOCD)); + if( aBlob==0 ){ + i64 iOff; /* Offset to read from */ + i64 szFile; /* Total size of file in bytes */ + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + return SQLITE_OK; + } + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; + rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); + }else{ + nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); + aRead = (u8*)&aBlob[nBlob-nRead]; + } + + if( rc==SQLITE_OK ){ + int i; + + /* Scan backwards looking for the signature bytes */ + for(i=nRead-20; i>=0; i--){ + if( aRead[i]==0x50 && aRead[i+1]==0x4b + && aRead[i+2]==0x05 && aRead[i+3]==0x06 + ){ + break; + } + } + if( i<0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "cannot find end of central directory record" + ); + return SQLITE_ERROR; + } + + aRead += i+4; + pEOCD->iDisk = zipfileRead16(aRead); + pEOCD->iFirstDisk = zipfileRead16(aRead); + pEOCD->nEntry = zipfileRead16(aRead); + pEOCD->nEntryTotal = zipfileRead16(aRead); + pEOCD->nSize = zipfileRead32(aRead); + pEOCD->iOffset = zipfileRead32(aRead); + } + + return rc; +} + +/* +** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry +** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added +** to the end of the list. Otherwise, it is added to the list immediately +** before pBefore (which is guaranteed to be a part of said list). +*/ +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew +){ + assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); + assert( pNew->pNext==0 ); + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } + }else{ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; + } +} + +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ + ZipfileEOCD eocd; + int rc; + int i; + i64 iOff; + + rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); + + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; + } + } + return rc; +} + +/* +** xFilter callback. +*/ +static int zipfileFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + const char *zFile = 0; /* Zip file to scan */ + int rc = SQLITE_OK; /* Return Code */ + int bInMemory = 0; /* True for an in-memory zipfile */ + + (void)idxStr; + (void)argc; + + zipfileResetCursor(pCsr); + + if( pTab->zFile ){ + zFile = pTab->zFile; + }else if( idxNum==0 ){ + zipfileCursorErr(pCsr, "zipfile() function requires an argument"); + return SQLITE_ERROR; + }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + static const u8 aEmptyBlob = 0; + const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( pTab->pFirstEntry==0 ); + if( aBlob==0 ){ + aBlob = &aEmptyBlob; + nBlob = 0; + } + rc = zipfileLoadDirectory(pTab, aBlob, nBlob); + pCsr->pFreeEntry = pTab->pFirstEntry; + pTab->pFirstEntry = pTab->pLastEntry = 0; + if( rc!=SQLITE_OK ) return rc; + bInMemory = 1; + }else{ + zFile = (const char*)sqlite3_value_text(argv[0]); + } + + if( 0==pTab->pWriteFd && 0==bInMemory ){ + pCsr->pFile = zFile ? sqlite3_fopen(zFile, "rb") : 0; + if( pCsr->pFile==0 ){ + zipfileCursorErr(pCsr, "cannot open file: %s", zFile); + rc = SQLITE_ERROR; + }else{ + rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); + if( rc==SQLITE_OK ){ + if( pCsr->eocd.nEntry==0 ){ + pCsr->bEof = 1; + }else{ + pCsr->iNextOff = pCsr->eocd.iOffset; + rc = zipfileNext(cur); + } + } + } + }else{ + pCsr->bNoop = 1; + pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; + rc = zipfileNext(cur); + } + + return rc; +} + +/* +** xBestIndex callback. +*/ +static int zipfileBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int idx = -1; + int unusable = 0; + (void)tab; + + for(i=0; inConstraint; i++){ + const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; + if( pCons->usable==0 ){ + unusable = 1; + }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idx = i; + } + } + pIdxInfo->estimatedCost = 1000.0; + if( idx>=0 ){ + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; + pIdxInfo->idxNum = 1; + }else if( unusable ){ + return SQLITE_CONSTRAINT; + } + return SQLITE_OK; +} + +static ZipfileEntry *zipfileNewEntry(const char *zPath){ + ZipfileEntry *pNew; + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + if( pNew ){ + memset(pNew, 0, sizeof(ZipfileEntry)); + pNew->cds.zFile = sqlite3_mprintf("%s", zPath); + if( pNew->cds.zFile==0 ){ + sqlite3_free(pNew); + pNew = 0; + } + } + return pNew; +} + +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; + + pCds->nExtra = 9; + + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); + + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; + + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + + return a-aBuf; +} + +static int zipfileAppendEntry( + ZipfileTab *pTab, + ZipfileEntry *pEntry, + const u8 *pData, + int nData +){ + u8 *aBuf = pTab->aBuffer; + int nBuf; + int rc; + + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); + if( rc==SQLITE_OK ){ + pEntry->iDataOff = pTab->szCurrent; + rc = zipfileAppendData(pTab, pData, nData); + } + + return rc; +} + +static int zipfileGetMode( + sqlite3_value *pVal, + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ +){ + const char *z = (const char*)sqlite3_value_text(pVal); + u32 mode = 0; + if( z==0 ){ + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); + }else if( z[0]>='0' && z[0]<='9' ){ + mode = (unsigned int)sqlite3_value_int(pVal); + }else{ + const char zTemplate[11] = "-rwxrwxrwx"; + int i; + if( strlen(z)!=10 ) goto parse_error; + switch( z[0] ){ + case '-': mode |= S_IFREG; break; + case 'd': mode |= S_IFDIR; break; + case 'l': mode |= S_IFLNK; break; + default: goto parse_error; + } + for(i=1; i<10; i++){ + if( z[i]==zTemplate[i] ) mode |= 1 << (9-i); + else if( z[i]!='-' ) goto parse_error; + } + } + if( ((mode & S_IFDIR)==0)==bIsDir ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + *pzErr = sqlite3_mprintf("zipfile: mode does not match data"); + return SQLITE_CONSTRAINT; + } + *pMode = mode; + return SQLITE_OK; + + parse_error: + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + return SQLITE_ERROR; +} + +/* +** Both (const char*) arguments point to nul-terminated strings. Argument +** nB is the value of strlen(zB). This function returns 0 if the strings are +** identical, ignoring any trailing '/' character in either path. */ +static int zipfileComparePath(const char *zA, const char *zB, int nB){ + int nA = (int)strlen(zA); + if( nA>0 && zA[nA-1]=='/' ) nA--; + if( nB>0 && zB[nB-1]=='/' ) nB--; + if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; + return 1; +} + +static int zipfileBegin(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); + if( pTab->zFile==0 || pTab->zFile[0]==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); + return SQLITE_ERROR; + } + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = sqlite3_fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } + + return rc; +} + +/* +** Return the current time as a 32-bit timestamp in UNIX epoch format (like +** time(2)). +*/ +static u32 zipfileTime(void){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + u32 ret; + if( pVfs==0 ) return 0; + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + i64 ms; + pVfs->xCurrentTimeInt64(pVfs, &ms); + ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); + }else{ + double day; + pVfs->xCurrentTime(pVfs, &day); + ret = (u32)((day - 2440587.5) * 86400); + } + return ret; +} + +/* +** Return a 32-bit timestamp in UNIX epoch format. +** +** If the value passed as the only argument is either NULL or an SQL NULL, +** return the current time. Otherwise, return the value stored in (*pVal) +** cast to a 32-bit unsigned integer. +*/ +static u32 zipfileGetTime(sqlite3_value *pVal){ + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + return zipfileTime(); + } + return (u32)sqlite3_value_int64(pVal); +} + +/* +** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry +** linked list. Remove it from the list and free the object. +*/ +static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ + if( pOld ){ + if( pTab->pFirstEntry==pOld ){ + pTab->pFirstEntry = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = 0; + }else{ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( p->pNext==pOld ){ + p->pNext = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = p; + break; + } + } + } + zipfileEntryFree(pOld); + } +} + +/* +** xUpdate method. +*/ +static int zipfileUpdate( + sqlite3_vtab *pVtab, + int nVal, + sqlite3_value **apVal, + sqlite_int64 *pRowid +){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; /* Return Code */ + ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ + + u32 mode = 0; /* Mode for new entry */ + u32 mTime = 0; /* Modification time for new entry */ + i64 sz = 0; /* Uncompressed size */ + const char *zPath = 0; /* Path for new entry */ + int nPath = 0; /* strlen(zPath) */ + const u8 *pData = 0; /* Pointer to buffer containing content */ + int nData = 0; /* Size of pData buffer in bytes */ + int iMethod = 0; /* Compression method for new entry */ + u8 *pFree = 0; /* Free this */ + char *zFree = 0; /* Also free this */ + ZipfileEntry *pOld = 0; + ZipfileEntry *pOld2 = 0; + int bUpdate = 0; /* True for an update that modifies "name" */ + int bIsDir = 0; + u32 iCrc32 = 0; + + (void)pRowid; + + if( pTab->pWriteFd==0 ){ + rc = zipfileBegin(pVtab); + if( rc!=SQLITE_OK ) return rc; + } + + /* If this is a DELETE or UPDATE, find the archive entry to delete. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); + int nDelete = (int)strlen(zDelete); + if( nVal>1 ){ + const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]); + if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){ + bUpdate = 1; + } + } + for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ + if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ + break; + } + assert( pOld->pNext ); + } + } + + if( nVal>1 ){ + /* Check that "sz" and "rawdata" are both NULL: */ + if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "sz must be NULL"); + rc = SQLITE_CONSTRAINT; + } + if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "rawdata must be NULL"); + rc = SQLITE_CONSTRAINT; + } + + if( rc==SQLITE_OK ){ + if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + /* data=NULL. A directory */ + bIsDir = 1; + }else{ + /* Value specified for "data", and possibly "method". This must be + ** a regular file or a symlink. */ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + pData = aIn; + nData = nIn; + if( iMethod!=0 && iMethod!=8 ){ + zipfileTableErr(pTab, "unknown compression method: %d", iMethod); + rc = SQLITE_CONSTRAINT; + }else{ + if( bAuto || iMethod ){ + int nCmp; + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); + if( rc==SQLITE_OK ){ + if( iMethod || nCmpbase.zErrMsg); + } + + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); + if( zPath==0 ) zPath = ""; + nPath = (int)strlen(zPath); + mTime = zipfileGetTime(apVal[4]); + } + + if( rc==SQLITE_OK && bIsDir ){ + /* For a directory, check that the last character in the path is a + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ + if( nPath<=0 || zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); + zPath = (const char*)zFree; + if( zFree==0 ){ + rc = SQLITE_NOMEM; + nPath = 0; + }else{ + nPath = (int)strlen(zPath); + } + } + } + + /* Check that we're not inserting a duplicate entry -OR- updating an + ** entry with a path, thereby making it into a duplicate. */ + if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ + switch( sqlite3_vtab_on_conflict(pTab->db) ){ + case SQLITE_IGNORE: { + goto zipfile_update_done; + } + case SQLITE_REPLACE: { + pOld2 = p; + break; + } + default: { + zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath); + rc = SQLITE_CONSTRAINT; + break; + } + } + break; + } + } + } + + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + pNew = zipfileNewEntry(zPath); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; + pNew->cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&pNew->cds, mTime); + pNew->cds.crc32 = iCrc32; + pNew->cds.szCompressed = nData; + pNew->cds.szUncompressed = (u32)sz; + pNew->cds.iExternalAttr = (mode<<16); + pNew->cds.iOffset = (u32)pTab->szCurrent; + pNew->cds.nFile = (u16)nPath; + pNew->mUnixTime = (u32)mTime; + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + zipfileAddEntry(pTab, pOld, pNew); + } + } + } + + if( rc==SQLITE_OK && (pOld || pOld2) ){ + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + pCsr->bNoop = 1; + } + } + + zipfileRemoveEntryFromList(pTab, pOld); + zipfileRemoveEntryFromList(pTab, pOld2); + } + +zipfile_update_done: + sqlite3_free(pFree); + sqlite3_free(zFree); + return rc; +} + +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ + + return a-aBuf; +} + +static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); +} + +/* +** Serialize the CDS structure into buffer aBuf[]. Return the number +** of bytes written. +*/ +static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ + u8 *a = aBuf; + ZipfileCDS *pCDS = &pEntry->cds; + + if( pEntry->aExtra==0 ){ + pCDS->nExtra = 9; + } + + zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(a, pCDS->iVersionMadeBy); + zipfileWrite16(a, pCDS->iVersionExtract); + zipfileWrite16(a, pCDS->flags); + zipfileWrite16(a, pCDS->iCompression); + zipfileWrite16(a, pCDS->mTime); + zipfileWrite16(a, pCDS->mDate); + zipfileWrite32(a, pCDS->crc32); + zipfileWrite32(a, pCDS->szCompressed); + zipfileWrite32(a, pCDS->szUncompressed); + assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + zipfileWrite16(a, pCDS->nFile); + zipfileWrite16(a, pCDS->nExtra); + zipfileWrite16(a, pCDS->nComment); + zipfileWrite16(a, pCDS->iDiskStart); + zipfileWrite16(a, pCDS->iInternalAttr); + zipfileWrite32(a, pCDS->iExternalAttr); + zipfileWrite32(a, pCDS->iOffset); + + memcpy(a, pCDS->zFile, pCDS->nFile); + a += pCDS->nFile; + + if( pEntry->aExtra ){ + int n = (int)pCDS->nExtra + (int)pCDS->nComment; + memcpy(a, pEntry->aExtra, n); + a += n; + }else{ + assert( pCDS->nExtra==9 ); + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + } + + return a-aBuf; +} + +static int zipfileCommit(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + if( pTab->pWriteFd ){ + i64 iOffset = pTab->szCurrent; + ZipfileEntry *p; + ZipfileEOCD eocd; + int nEntry = 0; + + /* Write out all entries */ + for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ + int n = zipfileSerializeCDS(p, pTab->aBuffer); + rc = zipfileAppendData(pTab, pTab->aBuffer, n); + nEntry++; + } + + /* Write out the EOCD record */ + eocd.iDisk = 0; + eocd.iFirstDisk = 0; + eocd.nEntry = (u16)nEntry; + eocd.nEntryTotal = (u16)nEntry; + eocd.nSize = (u32)(pTab->szCurrent - iOffset); + eocd.iOffset = (u32)iOffset; + rc = zipfileAppendEOCD(pTab, &eocd); + + zipfileCleanupTransaction(pTab); + } + return rc; +} + +static int zipfileRollback(sqlite3_vtab *pVtab){ + return zipfileCommit(pVtab); +} + +static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){ + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( iId==pCsr->iId ) break; + } + return pCsr; +} + +static void zipfileFunctionCds( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ZipfileCsr *pCsr; + ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); + assert( argc>0 ); + + pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); + if( pCsr ){ + ZipfileCDS *p = &pCsr->pCurrent->cds; + char *zRes = sqlite3_mprintf("{" + "\"version-made-by\" : %u, " + "\"version-to-extract\" : %u, " + "\"flags\" : %u, " + "\"compression\" : %u, " + "\"time\" : %u, " + "\"date\" : %u, " + "\"crc32\" : %u, " + "\"compressed-size\" : %u, " + "\"uncompressed-size\" : %u, " + "\"file-name-length\" : %u, " + "\"extra-field-length\" : %u, " + "\"file-comment-length\" : %u, " + "\"disk-number-start\" : %u, " + "\"internal-attr\" : %u, " + "\"external-attr\" : %u, " + "\"offset\" : %u }", + (u32)p->iVersionMadeBy, (u32)p->iVersionExtract, + (u32)p->flags, (u32)p->iCompression, + (u32)p->mTime, (u32)p->mDate, + (u32)p->crc32, (u32)p->szCompressed, + (u32)p->szUncompressed, (u32)p->nFile, + (u32)p->nExtra, (u32)p->nComment, + (u32)p->iDiskStart, (u32)p->iInternalAttr, + (u32)p->iExternalAttr, (u32)p->iOffset + ); + + if( zRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT); + sqlite3_free(zRes); + } + } +} + +/* +** xFindFunction method. +*/ +static int zipfileFindFunction( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* OUT: User data for *pxFunc */ +){ + (void)nArg; + if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ + *pxFunc = zipfileFunctionCds; + *ppArg = (void*)pVtab; + return 1; + } + return 0; +} + +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNewa, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = (int)nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... +*/ +static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; + }else{ + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; + } + } + + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = (int)sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + } + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; + }else{ + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOut0 && zName[nName-1]=='/' ){ + zErr = sqlite3_mprintf("non-directory name must not end with /"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + }else{ + if( nName==0 || zName[nName-1]!='/' ){ + zName = zFree = sqlite3_mprintf("%s/", zName); + if( zName==0 ){ + rc = SQLITE_NOMEM; + goto zipfile_step_out; + } + nName = (int)strlen(zName); + }else{ + while( nName>1 && zName[nName-2]=='/' ) nName--; + } + } + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = (u16)nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( nData>0 ){ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + } + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); +} + +/* +** xFinalize() callback for zipfile aggregate function. +*/ +static void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + sqlite3_int64 nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + if( p->nEntry>0 ){ + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = (u16)p->nEntry; + eocd.nEntryTotal = (u16)p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc64(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, (int)nZip, zipfileFree); + } + } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); +} + + +/* +** Register the "zipfile" virtual table. +*/ +static int zipfileRegister(sqlite3 *db){ + static sqlite3_module zipfileModule = { + 1, /* iVersion */ + zipfileConnect, /* xCreate */ + zipfileConnect, /* xConnect */ + zipfileBestIndex, /* xBestIndex */ + zipfileDisconnect, /* xDisconnect */ + zipfileDisconnect, /* xDestroy */ + zipfileOpen, /* xOpen - open a cursor */ + zipfileClose, /* xClose - close a cursor */ + zipfileFilter, /* xFilter - configure scan constraints */ + zipfileNext, /* xNext - advance a cursor */ + zipfileEof, /* xEof - check for end of scan */ + zipfileColumn, /* xColumn - read data */ + 0, /* xRowid - read data */ + zipfileUpdate, /* xUpdate */ + zipfileBegin, /* xBegin */ + 0, /* xSync */ + zipfileCommit, /* xCommit */ + zipfileRollback, /* xRollback */ + zipfileFindFunction, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollback */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + + int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); + } + assert( sizeof(i64)==8 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(u8)==1 ); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define zipfileRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 + +#endif +int sqlite3_zipfile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return zipfileRegister(db); +} + +/************************* End ../ext/misc/zipfile.c ********************/ +/************************* Begin ../ext/misc/sqlar.c ******************/ +/* +** 2017-12-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Utility functions sqlar_compress() and sqlar_uncompress(). Useful +** for working with sqlar archives and used by the shell tool's built-in +** sqlar support. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Implementation of the "sqlar_compress(X)" SQL function. +** +** If the type of X is SQLITE_BLOB, and compressing that blob using +** zlib utility function compress() yields a smaller blob, return the +** compressed blob. Otherwise, return a copy of X. +** +** SQLar uses the "zlib format" for compressed content. The zlib format +** contains a two-byte identification header and a four-byte checksum at +** the end. This is different from ZIP which uses the raw deflate format. +** +** Future enhancements to SQLar might add support for new compression formats. +** If so, those new formats will be identified by alternative headers in the +** compressed data. +*/ +static void sqlarCompressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + const Bytef *pData = sqlite3_value_blob(argv[0]); + uLong nData = sqlite3_value_bytes(argv[0]); + uLongf nOut = compressBound(nData); + Bytef *pOut; + + pOut = (Bytef*)sqlite3_malloc(nOut); + if( pOut==0 ){ + sqlite3_result_error_nomem(context); + return; + }else{ + if( Z_OK!=compress(pOut, &nOut, pData, nData) ){ + sqlite3_result_error(context, "error in compress()", -1); + }else if( nOut +#include +#include + +#if !defined(SQLITE_AMALGAMATION) +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif +#endif /* !defined(SQLITE_AMALGAMATION) */ + + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* typedef sqlite3_int64 i64; */ +/* typedef sqlite3_uint64 u64; */ + +typedef struct IdxColumn IdxColumn; +typedef struct IdxConstraint IdxConstraint; +typedef struct IdxScan IdxScan; +typedef struct IdxStatement IdxStatement; +typedef struct IdxTable IdxTable; +typedef struct IdxWrite IdxWrite; + +#define STRLEN (int)strlen + +/* +** A temp table name that we assume no user database will actually use. +** If this assumption proves incorrect triggers on the table with the +** conflicting name will be ignored. +*/ +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776" + +/* +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or +** any other type of single-ended range constraint on a column). +** +** pLink: +** Used to temporarily link IdxConstraint objects into lists while +** creating candidate indexes. +*/ +struct IdxConstraint { + char *zColl; /* Collation sequence */ + int bRange; /* True for range, false for eq */ + int iCol; /* Constrained table column */ + int bFlag; /* Used by idxFindCompatible() */ + int bDesc; /* True if ORDER BY DESC */ + IdxConstraint *pNext; /* Next constraint in pEq or pRange list */ + IdxConstraint *pLink; /* See above */ +}; + +/* +** A single scan of a single table. +*/ +struct IdxScan { + IdxTable *pTab; /* Associated table object */ + int iDb; /* Database containing table zTable */ + i64 covering; /* Mask of columns required for cov. index */ + IdxConstraint *pOrder; /* ORDER BY columns */ + IdxConstraint *pEq; /* List of == constraints */ + IdxConstraint *pRange; /* List of < constraints */ + IdxScan *pNextScan; /* Next IdxScan object for same analysis */ +}; + +/* +** Information regarding a single database table. Extracted from +** "PRAGMA table_info" by function idxGetTableInfo(). +*/ +struct IdxColumn { + char *zName; + char *zColl; + int iPk; +}; +struct IdxTable { + int nCol; + char *zName; /* Table name */ + IdxColumn *aCol; + IdxTable *pNext; /* Next table in linked list of all tables */ +}; + +/* +** An object of the following type is created for each unique table/write-op +** seen. The objects are stored in a singly-linked list beginning at +** sqlite3expert.pWrite. +*/ +struct IdxWrite { + IdxTable *pTab; + int eOp; /* SQLITE_UPDATE, DELETE or INSERT */ + IdxWrite *pNext; +}; + +/* +** Each statement being analyzed is represented by an instance of this +** structure. +*/ +struct IdxStatement { + int iId; /* Statement number */ + char *zSql; /* SQL statement */ + char *zIdx; /* Indexes */ + char *zEQP; /* Plan */ + IdxStatement *pNext; +}; + + +/* +** A hash table for storing strings. With space for a payload string +** with each entry. Methods are: +** +** idxHashInit() +** idxHashClear() +** idxHashAdd() +** idxHashSearch() +*/ +#define IDX_HASH_SIZE 1023 +typedef struct IdxHashEntry IdxHashEntry; +typedef struct IdxHash IdxHash; +struct IdxHashEntry { + char *zKey; /* nul-terminated key */ + char *zVal; /* nul-terminated value string */ + char *zVal2; /* nul-terminated value string 2 */ + IdxHashEntry *pHashNext; /* Next entry in same hash bucket */ + IdxHashEntry *pNext; /* Next entry in hash */ +}; +struct IdxHash { + IdxHashEntry *pFirst; + IdxHashEntry *aHash[IDX_HASH_SIZE]; +}; + +/* +** sqlite3expert object. +*/ +struct sqlite3expert { + int iSample; /* Percentage of tables to sample for stat1 */ + sqlite3 *db; /* User database */ + sqlite3 *dbm; /* In-memory db for this analysis */ + sqlite3 *dbv; /* Vtab schema for this analysis */ + IdxTable *pTable; /* List of all IdxTable objects */ + IdxScan *pScan; /* List of scan objects */ + IdxWrite *pWrite; /* List of write objects */ + IdxStatement *pStatement; /* List of IdxStatement objects */ + int bRun; /* True once analysis has run */ + char **pzErrmsg; + int rc; /* Error code from whereinfo hook */ + IdxHash hIdx; /* Hash containing all candidate indexes */ + char *zCandidates; /* For EXPERT_REPORT_CANDIDATES */ +}; + + +/* +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. +*/ +static void *idxMalloc(int *pRc, int nByte){ + void *pRet; + assert( *pRc==SQLITE_OK ); + assert( nByte>0 ); + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; + } + return pRet; +} + +/* +** Initialize an IdxHash hash table. +*/ +static void idxHashInit(IdxHash *pHash){ + memset(pHash, 0, sizeof(IdxHash)); +} + +/* +** Reset an IdxHash hash table. +*/ +static void idxHashClear(IdxHash *pHash){ + int i; + for(i=0; iaHash[i]; pEntry; pEntry=pNext){ + pNext = pEntry->pHashNext; + sqlite3_free(pEntry->zVal2); + sqlite3_free(pEntry); + } + } + memset(pHash, 0, sizeof(IdxHash)); +} + +/* +** Return the index of the hash bucket that the string specified by the +** arguments to this function belongs. +*/ +static int idxHashString(const char *z, int n){ + unsigned int ret = 0; + int i; + for(i=0; i=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return 1; + } + } + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + if( pEntry ){ + pEntry->zKey = (char*)&pEntry[1]; + memcpy(pEntry->zKey, zKey, nKey); + if( zVal ){ + pEntry->zVal = &pEntry->zKey[nKey+1]; + memcpy(pEntry->zVal, zVal, nVal); + } + pEntry->pHashNext = pHash->aHash[iHash]; + pHash->aHash[iHash] = pEntry; + + pEntry->pNext = pHash->pFirst; + pHash->pFirst = pEntry; + } + return 0; +} + +/* +** If zKey/nKey is present in the hash table, return a pointer to the +** hash-entry object. +*/ +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){ + int iHash; + IdxHashEntry *pEntry; + if( nKey<0 ) nKey = STRLEN(zKey); + iHash = idxHashString(zKey, nKey); + assert( iHash>=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return pEntry; + } + } + return 0; +} + +/* +** If the hash table contains an entry with a key equal to the string +** passed as the final two arguments to this function, return a pointer +** to the payload string. Otherwise, if zKey/nKey is not present in the +** hash table, return NULL. +*/ +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){ + IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey); + if( pEntry ) return pEntry->zVal; + return 0; +} + +/* +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl +** variable to point to a copy of nul-terminated string zColl. +*/ +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ + IdxConstraint *pNew; + int nColl = STRLEN(zColl); + + assert( *pRc==SQLITE_OK ); + pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1); + if( pNew ){ + pNew->zColl = (char*)&pNew[1]; + memcpy(pNew->zColl, zColl, nColl+1); + } + return pNew; +} + +/* +** An error associated with database handle db has just occurred. Pass +** the error message to callback function xOut. +*/ +static void idxDatabaseError( + sqlite3 *db, /* Database handle */ + char **pzErrmsg /* Write error here */ +){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); +} + +/* +** Prepare an SQL statement. +*/ +static int idxPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zSql /* SQL statement to compile */ +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *ppStmt = 0; + idxDatabaseError(db, pzErrmsg); + } + return rc; +} + +/* +** Prepare an SQL statement using the results of a printf() formatting. +*/ +static int idxPrintfPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zFmt, /* printf() format of SQL statement */ + ... /* Trailing printf() arguments */ +){ + va_list ap; + int rc; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + va_end(ap); + return rc; +} + + +/************************************************************************* +** Beginning of virtual table implementation. +*/ +typedef struct ExpertVtab ExpertVtab; +struct ExpertVtab { + sqlite3_vtab base; + IdxTable *pTab; + sqlite3expert *pExpert; +}; + +typedef struct ExpertCsr ExpertCsr; +struct ExpertCsr { + sqlite3_vtab_cursor base; + sqlite3_stmt *pData; +}; + +static char *expertDequote(const char *zIn){ + int n = STRLEN(zIn); + char *zRet = sqlite3_malloc(n); + + assert( zIn[0]=='\'' ); + assert( zIn[n-1]=='\'' ); + + if( zRet ){ + int iOut = 0; + int iIn = 0; + for(iIn=1; iIn<(n-1); iIn++){ + if( zIn[iIn]=='\'' ){ + assert( zIn[iIn+1]=='\'' ); + iIn++; + } + zRet[iOut++] = zIn[iIn]; + } + zRet[iOut] = '\0'; + } + + return zRet; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int expertConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3expert *pExpert = (sqlite3expert*)pAux; + ExpertVtab *p = 0; + int rc; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("internal error!"); + rc = SQLITE_ERROR; + }else{ + char *zCreateTable = expertDequote(argv[3]); + if( zCreateTable ){ + rc = sqlite3_declare_vtab(db, zCreateTable); + if( rc==SQLITE_OK ){ + p = idxMalloc(&rc, sizeof(ExpertVtab)); + } + if( rc==SQLITE_OK ){ + p->pExpert = pExpert; + p->pTab = pExpert->pTable; + assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 ); + } + sqlite3_free(zCreateTable); + }else{ + rc = SQLITE_NOMEM; + } + } + + *ppVtab = (sqlite3_vtab*)p; + return rc; +} + +static int expertDisconnect(sqlite3_vtab *pVtab){ + ExpertVtab *p = (ExpertVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){ + ExpertVtab *p = (ExpertVtab*)pVtab; + int rc = SQLITE_OK; + int n = 0; + IdxScan *pScan; + const int opmask = + SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT | + SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE | + SQLITE_INDEX_CONSTRAINT_LE; + + pScan = idxMalloc(&rc, sizeof(IdxScan)); + if( pScan ){ + int i; + + /* Link the new scan object into the list */ + pScan->pTab = p->pTab; + pScan->pNextScan = p->pExpert->pScan; + p->pExpert->pScan = pScan; + + /* Add the constraints to the IdxScan object */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + if( pCons->usable + && pCons->iColumn>=0 + && p->pTab->aCol[pCons->iColumn].iPk==0 + && (pCons->op & opmask) + ){ + IdxConstraint *pNew; + const char *zColl = sqlite3_vtab_collation(pIdxInfo, i); + pNew = idxNewConstraint(&rc, zColl); + if( pNew ){ + pNew->iCol = pCons->iColumn; + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pNew->pNext = pScan->pEq; + pScan->pEq = pNew; + }else{ + pNew->bRange = 1; + pNew->pNext = pScan->pRange; + pScan->pRange = pNew; + } + } + n++; + pIdxInfo->aConstraintUsage[i].argvIndex = n; + } + } + + /* Add the ORDER BY to the IdxScan object */ + for(i=pIdxInfo->nOrderBy-1; i>=0; i--){ + int iCol = pIdxInfo->aOrderBy[i].iColumn; + if( iCol>=0 ){ + IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl); + if( pNew ){ + pNew->iCol = iCol; + pNew->bDesc = pIdxInfo->aOrderBy[i].desc; + pNew->pNext = pScan->pOrder; + pNew->pLink = pScan->pOrder; + pScan->pOrder = pNew; + n++; + } + } + } + } + + pIdxInfo->estimatedCost = 1000000.0 / (n+1); + return rc; +} + +static int expertUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid +){ + (void)pVtab; + (void)nData; + (void)azData; + (void)pRowid; + return SQLITE_OK; +} + +/* +** Virtual table module xOpen method. +*/ +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_OK; + ExpertCsr *pCsr; + (void)pVTab; + pCsr = idxMalloc(&rc, sizeof(ExpertCsr)); + *ppCursor = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +/* +** Virtual table module xClose method. +*/ +static int expertClose(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_finalize(pCsr->pData); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int expertEof(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + return pCsr->pData==0; +} + +/* +** Virtual table module xNext method. +*/ +static int expertNext(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + int rc = SQLITE_OK; + + assert( pCsr->pData ); + rc = sqlite3_step(pCsr->pData); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + }else{ + rc = SQLITE_OK; + } + + return rc; +} + +/* +** Virtual table module xRowid method. +*/ +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + (void)cur; + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Virtual table module xColumn method. +*/ +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_value *pVal; + pVal = sqlite3_column_value(pCsr->pData, i); + if( pVal ){ + sqlite3_result_value(ctx, pVal); + } + return SQLITE_OK; +} + +/* +** Virtual table module xFilter method. +*/ +static int expertFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab); + sqlite3expert *pExpert = pVtab->pExpert; + int rc; + + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + if( rc==SQLITE_OK ){ + rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName + ); + } + + if( rc==SQLITE_OK ){ + rc = expertNext(cur); + } + return rc; +} + +static int idxRegisterVtab(sqlite3expert *p){ + static sqlite3_module expertModule = { + 2, /* iVersion */ + expertConnect, /* xCreate - create a table */ + expertConnect, /* xConnect - connect to an existing table */ + expertBestIndex, /* xBestIndex - Determine search strategy */ + expertDisconnect, /* xDisconnect - Disconnect from a table */ + expertDisconnect, /* xDestroy - Drop a table */ + expertOpen, /* xOpen - open a cursor */ + expertClose, /* xClose - close a cursor */ + expertFilter, /* xFilter - configure scan constraints */ + expertNext, /* xNext - advance a cursor */ + expertEof, /* xEof */ + expertColumn, /* xColumn - read data */ + expertRowid, /* xRowid - read data */ + expertUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0, /* xIntegrity */ + }; + + return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); +} +/* +** End of virtual table implementation. +*************************************************************************/ +/* +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function +** is called, set it to the return value of sqlite3_finalize() before +** returning. Otherwise, discard the sqlite3_finalize() return value. +*/ +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +/* +** Attempt to allocate an IdxTable structure corresponding to table zTab +** in the main database of connection db. If successful, set (*ppOut) to +** point to the new object and return SQLITE_OK. Otherwise, return an +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be +** set to point to an error string. +** +** It is the responsibility of the caller to eventually free either the +** IdxTable object or error message using sqlite3_free(). +*/ +static int idxGetTableInfo( + sqlite3 *db, /* Database connection to read details from */ + const char *zTab, /* Table name */ + IdxTable **ppOut, /* OUT: New object (if successful) */ + char **pzErrmsg /* OUT: Error message (if not) */ +){ + sqlite3_stmt *p1 = 0; + int nCol = 0; + int nTab; + int nByte; + IdxTable *pNew = 0; + int rc, rc2; + char *pCsr = 0; + int nPk = 0; + + *ppOut = 0; + if( zTab==0 ) return SQLITE_ERROR; + nTab = STRLEN(zTab); + nByte = sizeof(IdxTable) + nTab + 1; + rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + if( zCol==0 ){ + rc = SQLITE_ERROR; + break; + } + nByte += 1 + STRLEN(zCol); + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( zColSeq==0 ) zColSeq = "binary"; + nByte += 1 + STRLEN(zColSeq); + nCol++; + nPk += (sqlite3_column_int(p1, 5)>0); + } + rc2 = sqlite3_reset(p1); + if( rc==SQLITE_OK ) rc = rc2; + + nByte += sizeof(IdxColumn) * nCol; + if( rc==SQLITE_OK ){ + pNew = idxMalloc(&rc, nByte); + } + if( rc==SQLITE_OK ){ + pNew->aCol = (IdxColumn*)&pNew[1]; + pNew->nCol = nCol; + pCsr = (char*)&pNew->aCol[nCol]; + } + + nCol = 0; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + int nCopy; + if( zCol==0 ) continue; + nCopy = STRLEN(zCol) + 1; + pNew->aCol[nCol].zName = pCsr; + pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1); + memcpy(pCsr, zCol, nCopy); + pCsr += nCopy; + + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + if( zColSeq==0 ) zColSeq = "binary"; + nCopy = STRLEN(zColSeq) + 1; + pNew->aCol[nCol].zColl = pCsr; + memcpy(pCsr, zColSeq, nCopy); + pCsr += nCopy; + } + + nCol++; + } + idxFinalize(&rc, p1); + + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + }else if( ALWAYS(pNew!=0) ){ + pNew->zName = pCsr; + if( ALWAYS(pNew->zName!=0) ) memcpy(pNew->zName, zTab, nTab+1); + } + + *ppOut = pNew; + return rc; +} + +/* +** This function is a no-op if *pRc is set to anything other than +** SQLITE_OK when it is called. +** +** If *pRc is initially set to SQLITE_OK, then the text specified by +** the printf() style arguments is appended to zIn and the result returned +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on +** zIn before returning. +*/ +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ + va_list ap; + char *zAppend = 0; + char *zRet = 0; + int nIn = zIn ? STRLEN(zIn) : 0; + int nAppend = 0; + va_start(ap, zFmt); + if( *pRc==SQLITE_OK ){ + zAppend = sqlite3_vmprintf(zFmt, ap); + if( zAppend ){ + nAppend = STRLEN(zAppend); + zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); + } + if( zAppend && zRet ){ + if( nIn ) memcpy(zRet, zIn, nIn); + memcpy(&zRet[nIn], zAppend, nAppend+1); + }else{ + sqlite3_free(zRet); + zRet = 0; + *pRc = SQLITE_NOMEM; + } + sqlite3_free(zAppend); + sqlite3_free(zIn); + } + va_end(ap); + return zRet; +} + +/* +** Return true if zId must be quoted in order to use it as an SQL +** identifier, or false otherwise. +*/ +static int idxIdentifierRequiresQuotes(const char *zId){ + int i; + int nId = STRLEN(zId); + + if( sqlite3_keyword_check(zId, nId) ) return 1; + + for(i=0; zId[i]; i++){ + if( !(zId[i]=='_') + && !(zId[i]>='0' && zId[i]<='9') + && !(zId[i]>='a' && zId[i]<='z') + && !(zId[i]>='A' && zId[i]<='Z') + ){ + return 1; + } + } + return 0; +} + +/* +** This function appends an index column definition suitable for constraint +** pCons to the string passed as zIn and returns the result. +*/ +static char *idxAppendColDefn( + int *pRc, /* IN/OUT: Error code */ + char *zIn, /* Column defn accumulated so far */ + IdxTable *pTab, /* Table index will be created on */ + IdxConstraint *pCons +){ + char *zRet = zIn; + IdxColumn *p = &pTab->aCol[pCons->iCol]; + if( zRet ) zRet = idxAppendText(pRc, zRet, ", "); + + if( idxIdentifierRequiresQuotes(p->zName) ){ + zRet = idxAppendText(pRc, zRet, "%Q", p->zName); + }else{ + zRet = idxAppendText(pRc, zRet, "%s", p->zName); + } + + if( sqlite3_stricmp(p->zColl, pCons->zColl) ){ + if( idxIdentifierRequiresQuotes(pCons->zColl) ){ + zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl); + }else{ + zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl); + } + } + + if( pCons->bDesc ){ + zRet = idxAppendText(pRc, zRet, " DESC"); + } + return zRet; +} + +/* +** Search database dbm for an index compatible with the one idxCreateFromCons() +** would create from arguments pScan, pEq and pTail. If no error occurs and +** such an index is found, return non-zero. Or, if no such index is found, +** return zero. +** +** If an error occurs, set *pRc to an SQLite error code and return zero. +*/ +static int idxFindCompatible( + int *pRc, /* OUT: Error code */ + sqlite3* dbm, /* Database to search */ + IdxScan *pScan, /* Scan for table to search for index on */ + IdxConstraint *pEq, /* List of == constraints */ + IdxConstraint *pTail /* List of range constraints */ +){ + const char *zTbl = pScan->pTab->zName; + sqlite3_stmt *pIdxList = 0; + IdxConstraint *pIter; + int nEq = 0; /* Number of elements in pEq */ + int rc; + + /* Count the elements in list pEq */ + for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++; + + rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl); + while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){ + int bMatch = 1; + IdxConstraint *pT = pTail; + sqlite3_stmt *pInfo = 0; + const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1); + if( zIdx==0 ) continue; + + /* Zero the IdxConstraint.bFlag values in the pEq list */ + for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0; + + rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx); + while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){ + int iIdx = sqlite3_column_int(pInfo, 0); + int iCol = sqlite3_column_int(pInfo, 1); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + + if( iIdxpLink){ + if( pIter->bFlag ) continue; + if( pIter->iCol!=iCol ) continue; + if( sqlite3_stricmp(pIter->zColl, zColl) ) continue; + pIter->bFlag = 1; + break; + } + if( pIter==0 ){ + bMatch = 0; + break; + } + }else{ + if( pT ){ + if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){ + bMatch = 0; + break; + } + pT = pT->pLink; + } + } + } + idxFinalize(&rc, pInfo); + + if( rc==SQLITE_OK && bMatch ){ + sqlite3_finalize(pIdxList); + return 1; + } + } + idxFinalize(&rc, pIdxList); + + *pRc = rc; + return 0; +} + +/* Callback for sqlite3_exec() with query with leading count(*) column. + * The first argument is expected to be an int*, referent to be incremented + * if that leading column is not exactly '0'. + */ +static int countNonzeros(void* pCount, int nc, + char* azResults[], char* azColumns[]){ + (void)azColumns; /* Suppress unused parameter warning */ + if( nc>0 && (azResults[0][0]!='0' || azResults[0][1]!=0) ){ + *((int *)pCount) += 1; + } + return 0; +} + +static int idxCreateFromCons( + sqlite3expert *p, + IdxScan *pScan, + IdxConstraint *pEq, + IdxConstraint *pTail +){ + sqlite3 *dbm = p->dbm; + int rc = SQLITE_OK; + if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ + IdxTable *pTab = pScan->pTab; + char *zCols = 0; + char *zIdx = 0; + IdxConstraint *pCons; + unsigned int h = 0; + const char *zFmt; + + for(pCons=pEq; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + for(pCons=pTail; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + + if( rc==SQLITE_OK ){ + /* Hash the list of columns to come up with a name for the index */ + const char *zTable = pScan->pTab->zName; + int quoteTable = idxIdentifierRequiresQuotes(zTable); + char *zName = 0; /* Index name */ + int collisions = 0; + do{ + int i; + char *zFind; + for(i=0; zCols[i]; i++){ + h += ((h<<3) + zCols[i]); + } + sqlite3_free(zName); + zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); + if( zName==0 ) break; + /* Is is unique among table, view and index names? */ + zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q" + " AND type in ('index','table','view')"; + zFind = sqlite3_mprintf(zFmt, zName); + i = 0; + rc = sqlite3_exec(dbm, zFind, countNonzeros, &i, 0); + assert(rc==SQLITE_OK); + sqlite3_free(zFind); + if( i==0 ){ + collisions = 0; + break; + } + ++collisions; + }while( collisions<50 && zName!=0 ); + if( collisions ){ + /* This return means "Gave up trying to find a unique index name." */ + rc = SQLITE_BUSY_TIMEOUT; + }else if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( quoteTable ){ + zFmt = "CREATE INDEX \"%w\" ON \"%w\"(%s)"; + }else{ + zFmt = "CREATE INDEX %s ON %s(%s)"; + } + zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols); + if( !zIdx ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg); + if( rc!=SQLITE_OK ){ + rc = SQLITE_BUSY_TIMEOUT; + }else{ + idxHashAdd(&rc, &p->hIdx, zName, zIdx); + } + } + sqlite3_free(zName); + sqlite3_free(zIdx); + } + } + + sqlite3_free(zCols); + } + return rc; +} + +/* +** Return true if list pList (linked by IdxConstraint.pLink) contains +** a constraint compatible with *p. Otherwise return false. +*/ +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ + IdxConstraint *pCmp; + for(pCmp=pList; pCmp; pCmp=pCmp->pLink){ + if( p->iCol==pCmp->iCol ) return 1; + } + return 0; +} + +static int idxCreateFromWhere( + sqlite3expert *p, + IdxScan *pScan, /* Create indexes for this scan */ + IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ +){ + IdxConstraint *p1 = 0; + IdxConstraint *pCon; + int rc; + + /* Gather up all the == constraints. */ + for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){ + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + pCon->pLink = p1; + p1 = pCon; + } + } + + /* Create an index using the == constraints collected above. And the + ** range constraint/ORDER BY terms passed in by the caller, if any. */ + rc = idxCreateFromCons(p, pScan, p1, pTail); + + /* If no range/ORDER BY passed by the caller, create a version of the + ** index for each range constraint. */ + if( pTail==0 ){ + for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ + assert( pCon->pLink==0 ); + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + rc = idxCreateFromCons(p, pScan, p1, pCon); + } + } + } + + return rc; +} + +/* +** Create candidate indexes in database [dbm] based on the data in +** linked-list pScan. +*/ +static int idxCreateCandidates(sqlite3expert *p){ + int rc = SQLITE_OK; + IdxScan *pIter; + + for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + rc = idxCreateFromWhere(p, pIter, 0); + if( rc==SQLITE_OK && pIter->pOrder ){ + rc = idxCreateFromWhere(p, pIter, pIter->pOrder); + } + } + + return rc; +} + +/* +** Free all elements of the linked list starting at pConstraint. +*/ +static void idxConstraintFree(IdxConstraint *pConstraint){ + IdxConstraint *pNext; + IdxConstraint *p; + + for(p=pConstraint; p; p=pNext){ + pNext = p->pNext; + sqlite3_free(p); + } +} + +/* +** Free all elements of the linked list starting from pScan up until pLast +** (pLast is not freed). +*/ +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ + IdxScan *p; + IdxScan *pNext; + for(p=pScan; p!=pLast; p=pNext){ + pNext = p->pNextScan; + idxConstraintFree(p->pOrder); + idxConstraintFree(p->pEq); + idxConstraintFree(p->pRange); + sqlite3_free(p); + } +} + +/* +** Free all elements of the linked list starting from pStatement up +** until pLast (pLast is not freed). +*/ +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ + IdxStatement *p; + IdxStatement *pNext; + for(p=pStatement; p!=pLast; p=pNext){ + pNext = p->pNext; + sqlite3_free(p->zEQP); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Free the linked list of IdxTable objects starting at pTab. +*/ +static void idxTableFree(IdxTable *pTab){ + IdxTable *pIter; + IdxTable *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + +/* +** Free the linked list of IdxWrite objects starting at pTab. +*/ +static void idxWriteFree(IdxWrite *pTab){ + IdxWrite *pIter; + IdxWrite *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + + + +/* +** This function is called after candidate indexes have been created. It +** runs all the queries to see which indexes they prefer, and populates +** IdxStatement.zIdx and IdxStatement.zEQP with the results. +*/ +static int idxFindIndexes( + sqlite3expert *p, + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + IdxStatement *pStmt; + sqlite3 *dbm = p->dbm; + int rc = SQLITE_OK; + + IdxHash hIdx; + idxHashInit(&hIdx); + + for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){ + IdxHashEntry *pEntry; + sqlite3_stmt *pExplain = 0; + idxHashClear(&hIdx); + rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr, + "EXPLAIN QUERY PLAN %s", pStmt->zSql + ); + while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ + /* int iId = sqlite3_column_int(pExplain, 0); */ + /* int iParent = sqlite3_column_int(pExplain, 1); */ + /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); + int nDetail; + int i; + + if( !zDetail ) continue; + nDetail = STRLEN(zDetail); + + for(i=0; ihIdx, zIdx, nIdx); + if( zSql ){ + idxHashAdd(&rc, &hIdx, zSql, 0); + if( rc ) goto find_indexes_out; + } + break; + } + } + + if( zDetail[0]!='-' ){ + pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail); + } + } + + for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey); + } + + idxFinalize(&rc, pExplain); + } + + find_indexes_out: + idxHashClear(&hIdx); + return rc; +} + +static int idxAuthCallback( + void *pCtx, + int eOp, + const char *z3, + const char *z4, + const char *zDb, + const char *zTrigger +){ + int rc = SQLITE_OK; + (void)z4; + (void)zTrigger; + if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){ + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3expert *p = (sqlite3expert*)pCtx; + IdxTable *pTab; + for(pTab=p->pTable; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_stricmp(z3, pTab->zName) ) break; + } + if( pTab ){ + IdxWrite *pWrite; + for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){ + if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break; + } + if( pWrite==0 ){ + pWrite = idxMalloc(&rc, sizeof(IdxWrite)); + if( rc==SQLITE_OK ){ + pWrite->pTab = pTab; + pWrite->eOp = eOp; + pWrite->pNext = p->pWrite; + p->pWrite = pWrite; + } + } + } + } + } + return rc; +} + +static int idxProcessOneTrigger( + sqlite3expert *p, + IdxWrite *pWrite, + char **pzErr +){ + static const char *zInt = UNIQUE_TABLE_NAME; + static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME; + IdxTable *pTab = pWrite->pTab; + const char *zTab = pTab->zName; + const char *zSql = + "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_schema " + "WHERE tbl_name = %Q AND type IN ('table', 'trigger') " + "ORDER BY type;"; + sqlite3_stmt *pSelect = 0; + int rc = SQLITE_OK; + char *zWrite = 0; + + /* Create the table and its triggers in the temp schema */ + rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0); + if( zCreate==0 ) continue; + rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr); + } + idxFinalize(&rc, pSelect); + + /* Rename the table in the temp schema to zInt */ + if( rc==SQLITE_OK ){ + char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt); + if( z==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr); + sqlite3_free(z); + } + } + + switch( pWrite->eOp ){ + case SQLITE_INSERT: { + int i; + zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", "); + } + zWrite = idxAppendText(&rc, zWrite, ")"); + break; + } + case SQLITE_UPDATE: { + int i; + zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", + pTab->aCol[i].zName + ); + } + break; + } + default: { + assert( pWrite->eOp==SQLITE_DELETE ); + if( rc==SQLITE_OK ){ + zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt); + if( zWrite==0 ) rc = SQLITE_NOMEM; + } + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pX = 0; + rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0); + idxFinalize(&rc, pX); + if( rc!=SQLITE_OK ){ + idxDatabaseError(p->dbv, pzErr); + } + } + sqlite3_free(zWrite); + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr); + } + + return rc; +} + +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + IdxWrite *pEnd = 0; + IdxWrite *pFirst = p->pWrite; + + while( rc==SQLITE_OK && pFirst!=pEnd ){ + IdxWrite *pIter; + for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){ + rc = idxProcessOneTrigger(p, pIter, pzErr); + } + pEnd = pFirst; + pFirst = p->pWrite; + } + + return rc; +} + +/* +** This function tests if the schema of the main database of database handle +** db contains an object named zTab. Assuming no error occurs, output parameter +** (*pbContains) is set to true if zTab exists, or false if it does not. +** +** Or, if an error occurs, an SQLite error code is returned. The final value +** of (*pbContains) is undefined in this case. +*/ +static int expertDbContainsObject( + sqlite3 *db, + const char *zTab, + int *pbContains /* OUT: True if object exists */ +){ + const char *zSql = "SELECT 1 FROM sqlite_schema WHERE name = ?"; + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + int ret = 0; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pSql, 1, zTab, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pSql) ){ + ret = 1; + } + rc = sqlite3_finalize(pSql); + } + + *pbContains = ret; + return rc; +} + +/* +** Execute SQL command zSql using database handle db. If no error occurs, +** set (*pzErr) to NULL and return SQLITE_OK. +** +** If an error does occur, return an SQLite error code and set (*pzErr) to +** point to a buffer containing an English language error message. Except, +** if the error message begins with "no such module:", then ignore the +** error and return as if the SQL statement had succeeded. +** +** This is used to copy as much of the database schema as possible while +** ignoring any errors related to missing virtual table modules. +*/ +static int expertSchemaSql(sqlite3 *db, const char *zSql, char **pzErr){ + int rc = SQLITE_OK; + char *zErr = 0; + + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK && zErr ){ + int nErr = STRLEN(zErr); + if( nErr>=15 && memcmp(zErr, "no such module:", 15)==0 ){ + sqlite3_free(zErr); + rc = SQLITE_OK; + zErr = 0; + } + } + + *pzErr = zErr; + return rc; +} + +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ + int rc = idxRegisterVtab(p); + sqlite3_stmt *pSchema = 0; + + /* For each table in the main db schema: + ** + ** 1) Add an entry to the p->pTable list, and + ** 2) Create the equivalent virtual table in dbv. + */ + rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg, + "SELECT type, name, sql, 1, " + " substr(sql,1,14)=='create virtual' COLLATE nocase " + "FROM sqlite_schema " + "WHERE type IN ('table','view') AND " + " substr(name,1,7)!='sqlite_' COLLATE nocase " + " UNION ALL " + "SELECT type, name, sql, 2, 0 FROM sqlite_schema " + "WHERE type = 'trigger'" + " AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') " + "ORDER BY 4, 5 DESC, 1" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){ + const char *zType = (const char*)sqlite3_column_text(pSchema, 0); + const char *zName = (const char*)sqlite3_column_text(pSchema, 1); + const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); + int bVirtual = sqlite3_column_int(pSchema, 4); + int bExists = 0; + + if( zType==0 || zName==0 ) continue; + rc = expertDbContainsObject(p->dbv, zName, &bExists); + if( rc || bExists ) continue; + + if( zType[0]=='v' || zType[1]=='r' || bVirtual ){ + /* A view. Or a trigger on a view. */ + if( zSql ) rc = expertSchemaSql(p->dbv, zSql, pzErrmsg); + }else{ + IdxTable *pTab; + rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); + if( rc==SQLITE_OK ){ + int i; + char *zInner = 0; + char *zOuter = 0; + pTab->pNext = p->pTable; + p->pTable = pTab; + + /* The statement the vtab will pass to sqlite3_declare_vtab() */ + zInner = idxAppendText(&rc, 0, "CREATE TABLE x("); + for(i=0; inCol; i++){ + zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", + (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl + ); + } + zInner = idxAppendText(&rc, zInner, ")"); + + /* The CVT statement to create the vtab */ + zOuter = idxAppendText(&rc, 0, + "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg); + } + sqlite3_free(zInner); + sqlite3_free(zOuter); + } + } + } + idxFinalize(&rc, pSchema); + return rc; +} + +struct IdxSampleCtx { + int iTarget; + double target; /* Target nRet/nRow value */ + double nRow; /* Number of rows seen */ + double nRet; /* Number of rows returned */ +}; + +static void idxSampleFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx); + int bRet; + + (void)argv; + assert( argc==0 ); + if( p->nRow==0.0 ){ + bRet = 1; + }else{ + bRet = (p->nRet / p->nRow) <= p->target; + if( bRet==0 ){ + unsigned short rnd; + sqlite3_randomness(2, (void*)&rnd); + bRet = ((int)rnd % 100) <= p->iTarget; + } + } + + sqlite3_result_int(pCtx, bRet); + p->nRow += 1.0; + p->nRet += (double)bRet; +} + +struct IdxRemCtx { + int nSlot; + struct IdxRemSlot { + int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ + i64 iVal; /* SQLITE_INTEGER value */ + double rVal; /* SQLITE_FLOAT value */ + int nByte; /* Bytes of space allocated at z */ + int n; /* Size of buffer z */ + char *z; /* SQLITE_TEXT/BLOB value */ + } aSlot[1]; +}; + +/* +** Implementation of scalar function sqlite_expert_rem(). +*/ +static void idxRemFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx); + struct IdxRemSlot *pSlot; + int iSlot; + assert( argc==2 ); + + iSlot = sqlite3_value_int(argv[0]); + assert( iSlotnSlot ); + pSlot = &p->aSlot[iSlot]; + + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; + + case SQLITE_INTEGER: + sqlite3_result_int64(pCtx, pSlot->iVal); + break; + + case SQLITE_FLOAT: + sqlite3_result_double(pCtx, pSlot->rVal); + break; + + case SQLITE_BLOB: + sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; + + case SQLITE_TEXT: + sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; + } + + pSlot->eType = sqlite3_value_type(argv[1]); + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; + + case SQLITE_INTEGER: + pSlot->iVal = sqlite3_value_int64(argv[1]); + break; + + case SQLITE_FLOAT: + pSlot->rVal = sqlite3_value_double(argv[1]); + break; + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int nByte = sqlite3_value_bytes(argv[1]); + const void *pData = 0; + if( nByte>pSlot->nByte ){ + char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + if( zNew==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + pSlot->nByte = nByte*2; + pSlot->z = zNew; + } + pSlot->n = nByte; + if( pSlot->eType==SQLITE_BLOB ){ + pData = sqlite3_value_blob(argv[1]); + if( pData ) memcpy(pSlot->z, pData, nByte); + }else{ + pData = sqlite3_value_text(argv[1]); + memcpy(pSlot->z, pData, nByte); + } + break; + } + } +} + +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){ + int rc = SQLITE_OK; + const char *zMax = + "SELECT max(i.seqno) FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l, " + " pragma_index_info(l.name) AS i " + "WHERE s.type = 'table'"; + sqlite3_stmt *pMax = 0; + + *pnMax = 0; + rc = idxPrepareStmt(db, &pMax, pzErr, zMax); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + *pnMax = sqlite3_column_int(pMax, 0) + 1; + } + idxFinalize(&rc, pMax); + + return rc; +} + +static int idxPopulateOneStat1( + sqlite3expert *p, + sqlite3_stmt *pIndexXInfo, + sqlite3_stmt *pWriteStat, + const char *zTab, + const char *zIdx, + char **pzErr +){ + char *zCols = 0; + char *zOrder = 0; + char *zQuery = 0; + int nCol = 0; + int i; + sqlite3_stmt *pQuery = 0; + int *aStat = 0; + int rc = SQLITE_OK; + + assert( p->iSample>0 ); + + /* Formulate the query text */ + sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC); + while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){ + const char *zComma = zCols==0 ? "" : ", "; + const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); + const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); + if( zName==0 ){ + /* This index contains an expression. Ignore it. */ + sqlite3_free(zCols); + sqlite3_free(zOrder); + return sqlite3_reset(pIndexXInfo); + } + zCols = idxAppendText(&rc, zCols, + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl + ); + zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); + } + sqlite3_reset(pIndexXInfo); + if( rc==SQLITE_OK ){ + if( p->iSample==100 ){ + zQuery = sqlite3_mprintf( + "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder + ); + }else{ + zQuery = sqlite3_mprintf( + "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder + ); + } + } + sqlite3_free(zCols); + sqlite3_free(zOrder); + + /* Formulate the query text */ + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery); + } + sqlite3_free(zQuery); + + if( rc==SQLITE_OK ){ + aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); + } + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + IdxHashEntry *pEntry; + char *zStat = 0; + for(i=0; i<=nCol; i++) aStat[i] = 1; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + aStat[0]++; + for(i=0; ihIdx, zIdx, STRLEN(zIdx)); + if( pEntry ){ + assert( pEntry->zVal2==0 ); + pEntry->zVal2 = zStat; + }else{ + sqlite3_free(zStat); + } + } + sqlite3_free(aStat); + idxFinalize(&rc, pQuery); + + return rc; +} + +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){ + int rc; + char *zSql; + + rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + if( rc!=SQLITE_OK ) return rc; + + zSql = sqlite3_mprintf( + "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab + ); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0); + sqlite3_free(zSql); + + return rc; +} + +/* +** This function is called as part of sqlite3_expert_analyze(). Candidate +** indexes have already been created in database sqlite3expert.dbm, this +** function populates sqlite_stat1 table in the same database. +** +** The stat1 data is generated by querying the +*/ +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + int nMax =0; + struct IdxRemCtx *pCtx = 0; + struct IdxSampleCtx samplectx; + int i; + i64 iPrev = -100000; + sqlite3_stmt *pAllIndex = 0; + sqlite3_stmt *pIndexXInfo = 0; + sqlite3_stmt *pWrite = 0; + + const char *zAllIndex = + "SELECT s.rowid, s.name, l.name FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l " + "WHERE s.type = 'table'"; + const char *zIndexXInfo = + "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key"; + const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)"; + + /* If iSample==0, no sqlite_stat1 data is required. */ + if( p->iSample==0 ) return SQLITE_OK; + + rc = idxLargestIndex(p->dbm, &nMax, pzErr); + if( nMax<=0 || rc!=SQLITE_OK ) return rc; + + rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); + + if( rc==SQLITE_OK ){ + int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); + } + + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + pCtx->nSlot = nMax+1; + rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){ + i64 iRowid = sqlite3_column_int64(pAllIndex, 0); + const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1); + const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2); + if( zTab==0 || zIdx==0 ) continue; + if( p->iSample<100 && iPrev!=iRowid ){ + samplectx.target = (double)p->iSample / 100.0; + samplectx.iTarget = p->iSample; + samplectx.nRow = 0.0; + samplectx.nRet = 0.0; + rc = idxBuildSampleTable(p, zTab); + if( rc!=SQLITE_OK ) break; + } + rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr); + iPrev = iRowid; + } + if( rc==SQLITE_OK && p->iSample<100 ){ + rc = sqlite3_exec(p->dbv, + "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0 + ); + } + + idxFinalize(&rc, pAllIndex); + idxFinalize(&rc, pIndexXInfo); + idxFinalize(&rc, pWrite); + + if( pCtx ){ + for(i=0; inSlot; i++){ + sqlite3_free(pCtx->aSlot[i].z); + } + sqlite3_free(pCtx); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); + } + + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + return rc; +} + +/* +** Define and possibly pretend to use a useless collation sequence. +** This pretense allows expert to accept SQL using custom collations. +*/ +int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ + (void)up1; + (void)up2; + (void)up3; + (void)up4; + (void)up5; + assert(0); /* VDBE should never be run. */ + return 0; +} +/* And a callback to register above upon actual need */ +void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ + (void)up1; + sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0); +} + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) +/* +** dummy functions for no-op implementation of UDFs during expert's work +*/ +void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ + (void)up1; + (void)up2; + (void)up3; + assert(0); /* VDBE should never be run. */ +} +void dummyUDFvalue(sqlite3_context *up1){ + (void)up1; + assert(0); /* VDBE should never be run. */ +} + +/* +** Register UDFs from user database with another. +*/ +int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ + sqlite3_stmt *pStmt; + int rc = sqlite3_prepare_v2(dbSrc, + "SELECT name,type,enc,narg,flags " + "FROM pragma_function_list() " + "WHERE builtin==0", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + int nargs = sqlite3_column_int(pStmt,3); + int flags = sqlite3_column_int(pStmt,4); + const char *name = (char*)sqlite3_column_text(pStmt,0); + const char *type = (char*)sqlite3_column_text(pStmt,1); + const char *enc = (char*)sqlite3_column_text(pStmt,2); + if( name==0 || type==0 || enc==0 ){ + /* no-op. Only happens on OOM */ + }else{ + int ienc = SQLITE_UTF8; + int rcf = SQLITE_ERROR; + if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE; + else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE; + ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY)); + if( strcmp(type,"w")==0 ){ + rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0, + dummyUDF,dummyUDFvalue,0,0,0); + }else if( strcmp(type,"a")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + 0,dummyUDF,dummyUDFvalue); + }else if( strcmp(type,"s")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + dummyUDF,0,0); + } + if( rcf!=SQLITE_OK ){ + rc = rcf; + break; + } + } + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + return rc; +} +#endif + +/* +** Allocate a new sqlite3expert object. +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ + int rc = SQLITE_OK; + sqlite3expert *pNew; + + pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert)); + + /* Open two in-memory databases to work with. The "vtab database" (dbv) + ** will contain a virtual table corresponding to each real table in + ** the user database schema, and a copy of each view. It is used to + ** collect information regarding the WHERE, ORDER BY and other clauses + ** of the user's query. + */ + if( rc==SQLITE_OK ){ + pNew->db = db; + pNew->iSample = 100; + rc = sqlite3_open(":memory:", &pNew->dbv); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_open(":memory:", &pNew->dbm); + if( rc==SQLITE_OK ){ + sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); + } + } + + /* Allow custom collations to be dealt with through prepare. */ + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS); + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS); + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + /* Register UDFs from database [db] with [dbm] and [dbv]. */ + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbm); + } + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbv); + } +#endif + + /* Copy the entire schema of database [db] into [dbm]. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSql = 0; + rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, + "SELECT sql, name, substr(sql,1,14)=='create virtual' COLLATE nocase" + " FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' COLLATE nocase" + " ORDER BY 3 DESC, rowid" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + const char *zName = (const char*)sqlite3_column_text(pSql, 1); + int bExists = 0; + rc = expertDbContainsObject(pNew->dbm, zName, &bExists); + if( rc==SQLITE_OK && zSql && bExists==0 ){ + rc = expertSchemaSql(pNew->dbm, zSql, pzErrmsg); + } + } + idxFinalize(&rc, pSql); + } + + /* Create the vtab schema */ + if( rc==SQLITE_OK ){ + rc = idxCreateVtabSchema(pNew, pzErrmsg); + } + + /* Register the auth callback with dbv */ + if( rc==SQLITE_OK ){ + sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); + } + + /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + ** return the new sqlite3expert handle. */ + if( rc!=SQLITE_OK ){ + sqlite3_expert_destroy(pNew); + pNew = 0; + } + return pNew; +} + +/* +** Configure an sqlite3expert object. +*/ +int sqlite3_expert_config(sqlite3expert *p, int op, ...){ + int rc = SQLITE_OK; + va_list ap; + va_start(ap, op); + switch( op ){ + case EXPERT_CONFIG_SAMPLE: { + int iVal = va_arg(ap, int); + if( iVal<0 ) iVal = 0; + if( iVal>100 ) iVal = 100; + p->iSample = iVal; + break; + } + default: + rc = SQLITE_NOTFOUND; + break; + } + + va_end(ap); + return rc; +} + +/* +** Add an SQL statement to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From sqlite3_expert_new() */ + const char *zSql, /* SQL statement to add */ + char **pzErr /* OUT: Error message (if any) */ +){ + IdxScan *pScanOrig = p->pScan; + IdxStatement *pStmtOrig = p->pStatement; + int rc = SQLITE_OK; + const char *zStmt = zSql; + + if( p->bRun ) return SQLITE_MISUSE; + + while( rc==SQLITE_OK && zStmt && zStmt[0] ){ + sqlite3_stmt *pStmt = 0; + /* Ensure that the provided statement compiles against user's DB. */ + rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt); + if( rc!=SQLITE_OK ) break; + sqlite3_finalize(pStmt); + rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); + if( rc==SQLITE_OK ){ + if( pStmt ){ + IdxStatement *pNew; + const char *z = sqlite3_sql(pStmt); + int n = STRLEN(z); + pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); + if( rc==SQLITE_OK ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, z, n+1); + pNew->pNext = p->pStatement; + if( p->pStatement ) pNew->iId = p->pStatement->iId+1; + p->pStatement = pNew; + } + sqlite3_finalize(pStmt); + } + }else{ + idxDatabaseError(p->dbv, pzErr); + } + } + + if( rc!=SQLITE_OK ){ + idxScanFree(p->pScan, pScanOrig); + idxStatementFree(p->pStatement, pStmtOrig); + p->pScan = pScanOrig; + p->pStatement = pStmtOrig; + } + + return rc; +} + +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ + int rc; + IdxHashEntry *pEntry; + + /* Do trigger processing to collect any extra IdxScan structures */ + rc = idxProcessTriggers(p, pzErr); + + /* Create candidate indexes within the in-memory database file */ + if( rc==SQLITE_OK ){ + rc = idxCreateCandidates(p); + }else if ( rc==SQLITE_BUSY_TIMEOUT ){ + if( pzErr ) + *pzErr = sqlite3_mprintf("Cannot find a unique index name to propose."); + return rc; + } + + /* Generate the stat1 data */ + if( rc==SQLITE_OK ){ + rc = idxPopulateStat1(p, pzErr); + } + + /* Formulate the EXPERT_REPORT_CANDIDATES text */ + for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + p->zCandidates = idxAppendText(&rc, p->zCandidates, + "%s;%s%s\n", pEntry->zVal, + pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2 + ); + } + + /* Figure out which of the candidate indexes are preferred by the query + ** planner and report the results to the user. */ + if( rc==SQLITE_OK ){ + rc = idxFindIndexes(p, pzErr); + } + + if( rc==SQLITE_OK ){ + p->bRun = 1; + } + return rc; +} + +/* +** Return the total number of statements that have been added to this +** sqlite3expert using sqlite3_expert_sql(). +*/ +int sqlite3_expert_count(sqlite3expert *p){ + int nRet = 0; + if( p->pStatement ) nRet = p->pStatement->iId+1; + return nRet; +} + +/* +** Return a component of the report. +*/ +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ + const char *zRet = 0; + IdxStatement *pStmt; + + if( p->bRun==0 ) return 0; + for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext); + switch( eReport ){ + case EXPERT_REPORT_SQL: + if( pStmt ) zRet = pStmt->zSql; + break; + case EXPERT_REPORT_INDEXES: + if( pStmt ) zRet = pStmt->zIdx; + break; + case EXPERT_REPORT_PLAN: + if( pStmt ) zRet = pStmt->zEQP; + break; + case EXPERT_REPORT_CANDIDATES: + zRet = p->zCandidates; + break; + } + return zRet; +} + +/* +** Free an sqlite3expert object. +*/ +void sqlite3_expert_destroy(sqlite3expert *p){ + if( p ){ + sqlite3_close(p->dbm); + sqlite3_close(p->dbv); + idxScanFree(p->pScan, 0); + idxStatementFree(p->pStatement, 0); + idxTableFree(p->pTable); + idxWriteFree(p->pWrite); + idxHashClear(&p->hIdx); + sqlite3_free(p->zCandidates); + sqlite3_free(p); + } +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/expert/sqlite3expert.c ********************/ +/************************* Begin ../ext/intck/sqlite3intck.h ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ + +/************************* End ../ext/intck/sqlite3intck.h ********************/ +/************************* Begin ../ext/intck/sqlite3intck.c ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* #include "sqlite3intck.h" */ +#include +#include + +#include +#include + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} + +/************************* End ../ext/intck/sqlite3intck.c ********************/ +/************************* Begin ../ext/misc/stmtrand.c ******************/ +/* +** 2024-05-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** An SQL function that return pseudo-random non-negative integers. +** +** SELECT stmtrand(123); +** +** A special feature of this function is that the same sequence of random +** integers is returned for each invocation of the statement. This makes +** the results repeatable, and hence useful for testing. The argument is +** an integer which is the seed for the random number sequence. The seed +** is used by the first invocation of this function only and is ignored +** for all subsequent calls within the same statement. +** +** Resetting a statement (sqlite3_reset()) also resets the random number +** sequence. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* State of the pseudo-random number generator */ +typedef struct Stmtrand { + unsigned int x, y; +} Stmtrand; + +/* auxdata key */ +#define STMTRAND_KEY (-4418371) + +/* +** Function: stmtrand(SEED) +** +** Return a pseudo-random number. +*/ +static void stmtrandFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stmtrand *p; + + p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); + if( p==0 ){ + unsigned int seed; + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + if( argc>=1 ){ + seed = (unsigned int)sqlite3_value_int(argv[0]); + }else{ + seed = 0; + } + p->x = seed | 1; + p->y = seed; + sqlite3_set_auxdata(context, STMTRAND_KEY, p, sqlite3_free); + p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + p->x = (p->x>>1) ^ ((1+~(p->x&1)) & 0xd0000001); + p->y = p->y*1103515245 + 12345; + sqlite3_result_int(context, (int)((p->x ^ p->y)&0x7fffffff)); +} + +#ifdef _WIN32 + +#endif +int sqlite3_stmtrand_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "stmtrand", 1, SQLITE_UTF8, 0, + stmtrandFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "stmtrand", 0, SQLITE_UTF8, 0, + stmtrandFunc, 0, 0); + } + return rc; +} + +/************************* End ../ext/misc/stmtrand.c ********************/ +/************************* Begin ../ext/misc/vfstrace.c ******************/ +/* +** 2011 March 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code implements a VFS shim that writes diagnostic +** output for each VFS call, similar to "strace". +** +** USAGE: +** +** This source file exports a single symbol which is the name of a +** function: +** +** int vfstrace_register( +** const char *zTraceName, // Name of the newly constructed VFS +** const char *zOldVfsName, // Name of the underlying VFS +** int (*xOut)(const char*,void*), // Output routine. ex: fputs +** void *pOutArg, // 2nd argument to xOut. ex: stderr +** int makeDefault // Make the new VFS the default +** ); +** +** Applications that want to trace their VFS usage must provide a callback +** function with this prototype: +** +** int traceOutput(const char *zMessage, void *pAppData); +** +** This function will "output" the trace messages, where "output" can +** mean different things to different applications. The traceOutput function +** for the command-line shell (see shell.c) is "fputs" from the standard +** library, which means that all trace output is written on the stream +** specified by the second argument. In the case of the command-line shell +** the second argument is stderr. Other applications might choose to output +** trace information to a file, over a socket, or write it into a buffer. +** +** The vfstrace_register() function creates a new "shim" VFS named by +** the zTraceName parameter. A "shim" VFS is an SQLite backend that does +** not really perform the duties of a true backend, but simply filters or +** interprets VFS calls before passing them off to another VFS which does +** the actual work. In this case the other VFS - the one that does the +** real work - is identified by the second parameter, zOldVfsName. If +** the 2nd parameter is NULL then the default VFS is used. The common +** case is for the 2nd parameter to be NULL. +** +** The third and fourth parameters are the pointer to the output function +** and the second argument to the output function. For the SQLite +** command-line shell, when the -vfstrace option is used, these parameters +** are fputs and stderr, respectively. +** +** The fifth argument is true (non-zero) to cause the newly created VFS +** to become the default VFS. The common case is for the fifth parameter +** to be true. +** +** The call to vfstrace_register() simply creates the shim VFS that does +** tracing. The application must also arrange to use the new VFS for +** all database connections that are created and for which tracing is +** desired. This can be done by specifying the trace VFS using URI filename +** notation, or by specifying the trace VFS as the 4th parameter to +** sqlite3_open_v2() or by making the trace VFS be the default (by setting +** the 5th parameter of vfstrace_register() to 1). +** +** +** ENABLING VFSTRACE IN A COMMAND-LINE SHELL +** +** The SQLite command line shell implemented by the shell.c source file +** can be used with this module. To compile in -vfstrace support, first +** gather this file (test_vfstrace.c), the shell source file (shell.c), +** and the SQLite amalgamation source files (sqlite3.c, sqlite3.h) into +** the working directory. Then compile using a command like the following: +** +** gcc -o sqlite3 -Os -I. -DSQLITE_ENABLE_VFSTRACE \ +** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ +** -DHAVE_READLINE -DHAVE_USLEEP=1 \ +** shell.c test_vfstrace.c sqlite3.c -ldl -lreadline -lncurses +** +** The gcc command above works on Linux and provides (in addition to the +** -vfstrace option) support for FTS3 and FTS4, RTREE, and command-line +** editing using the readline library. The command-line shell does not +** use threads so we added -DSQLITE_THREADSAFE=0 just to make the code +** run a little faster. For compiling on a Mac, you'll probably need +** to omit the -DHAVE_READLINE, the -lreadline, and the -lncurses options. +** The compilation could be simplified to just this: +** +** gcc -DSQLITE_ENABLE_VFSTRACE \ +** shell.c test_vfstrace.c sqlite3.c -ldl -lpthread +** +** In this second example, all unnecessary options have been removed +** Note that since the code is now threadsafe, we had to add the -lpthread +** option to pull in the pthreads library. +** +** To cross-compile for windows using MinGW, a command like this might +** work: +** +** /opt/mingw/bin/i386-mingw32msvc-gcc -o sqlite3.exe -Os -I \ +** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_VFSTRACE \ +** shell.c test_vfstrace.c sqlite3.c +** +** Similar compiler commands will work on different systems. The key +** invariants are (1) you must have -DSQLITE_ENABLE_VFSTRACE so that +** the shell.c source file will know to include the -vfstrace command-line +** option and (2) you must compile and link the three source files +** shell,c, test_vfstrace.c, and sqlite3.c. +*/ +#include +#include +/* #include "sqlite3.h" */ + +/* +** An instance of this structure is attached to the each trace VFS to +** provide auxiliary information. +*/ +typedef struct vfstrace_info vfstrace_info; +struct vfstrace_info { + sqlite3_vfs *pRootVfs; /* The underlying real VFS */ + int (*xOut)(const char*, void*); /* Send output here */ + void *pOutArg; /* First argument to xOut */ + const char *zVfsName; /* Name of this trace-VFS */ + sqlite3_vfs *pTraceVfs; /* Pointer back to the trace VFS */ +}; + +/* +** The sqlite3_file object for the trace VFS +*/ +typedef struct vfstrace_file vfstrace_file; +struct vfstrace_file { + sqlite3_file base; /* Base class. Must be first */ + vfstrace_info *pInfo; /* The trace-VFS to which this file belongs */ + const char *zFName; /* Base name of the file */ + sqlite3_file *pReal; /* The real underlying file */ +}; + +/* +** Method declarations for vfstrace_file. +*/ +static int vfstraceClose(sqlite3_file*); +static int vfstraceRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vfstraceWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int vfstraceTruncate(sqlite3_file*, sqlite3_int64 size); +static int vfstraceSync(sqlite3_file*, int flags); +static int vfstraceFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vfstraceLock(sqlite3_file*, int); +static int vfstraceUnlock(sqlite3_file*, int); +static int vfstraceCheckReservedLock(sqlite3_file*, int *); +static int vfstraceFileControl(sqlite3_file*, int op, void *pArg); +static int vfstraceSectorSize(sqlite3_file*); +static int vfstraceDeviceCharacteristics(sqlite3_file*); +static int vfstraceShmLock(sqlite3_file*,int,int,int); +static int vfstraceShmMap(sqlite3_file*,int,int,int, void volatile **); +static void vfstraceShmBarrier(sqlite3_file*); +static int vfstraceShmUnmap(sqlite3_file*,int); + +/* +** Method declarations for vfstrace_vfs. +*/ +static int vfstraceOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int vfstraceDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int vfstraceAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int vfstraceFullPathname(sqlite3_vfs*, const char *zName, int, char *); +static void *vfstraceDlOpen(sqlite3_vfs*, const char *zFilename); +static void vfstraceDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*vfstraceDlSym(sqlite3_vfs*,void*, const char *zSymbol))(void); +static void vfstraceDlClose(sqlite3_vfs*, void*); +static int vfstraceRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int vfstraceSleep(sqlite3_vfs*, int microseconds); +static int vfstraceCurrentTime(sqlite3_vfs*, double*); +static int vfstraceGetLastError(sqlite3_vfs*, int, char*); +static int vfstraceCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int vfstraceSetSystemCall(sqlite3_vfs*,const char*, sqlite3_syscall_ptr); +static sqlite3_syscall_ptr vfstraceGetSystemCall(sqlite3_vfs*, const char *); +static const char *vfstraceNextSystemCall(sqlite3_vfs*, const char *zName); + +/* +** Return a pointer to the tail of the pathname. Examples: +** +** /home/drh/xyzzy.txt -> xyzzy.txt +** xyzzy.txt -> xyzzy.txt +*/ +static const char *fileTail(const char *z){ + size_t i; + if( z==0 ) return 0; + i = strlen(z)-1; + while( i>0 && z[i-1]!='/' ){ i--; } + return &z[i]; +} + +/* +** Send trace output defined by zFormat and subsequent arguments. +*/ +static void vfstrace_printf( + vfstrace_info *pInfo, + const char *zFormat, + ... +){ + va_list ap; + char *zMsg; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + pInfo->xOut(zMsg, pInfo->pOutArg); + sqlite3_free(zMsg); +} + +/* +** Try to convert an error code into a symbolic name for that error code. +*/ +static const char *vfstrace_errcode_name(int rc ){ + const char *zVal = 0; + switch( rc ){ + case SQLITE_OK: zVal = "SQLITE_OK"; break; + case SQLITE_INTERNAL: zVal = "SQLITE_INTERNAL"; break; + case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break; + case SQLITE_PERM: zVal = "SQLITE_PERM"; break; + case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zVal = "SQLITE_LOCKED"; break; + case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zVal = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zVal = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zVal = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break; + case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break; + case SQLITE_IOERR_READ: zVal = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zVal = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zVal = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zVal = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zVal = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zVal = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zVal = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zVal = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zVal = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zVal = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zVal = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zVal = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zVal = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zVal = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zVal = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zVal = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zVal = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zVal = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zVal = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zVal = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zVal = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zVal = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_GETTEMPPATH: zVal = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zVal = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_READONLY_DBMOVED: zVal = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zVal = "SQLITE_LOCKED_SHAREDCACHE"; break; + case SQLITE_BUSY_RECOVERY: zVal = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zVal = "SQLITE_CANTOPEN_NOTEMPDIR"; break; + } + return zVal; +} + +/* +** Convert value rc into a string and print it using zFormat. zFormat +** should have exactly one %s +*/ +static void vfstrace_print_errcode( + vfstrace_info *pInfo, + const char *zFormat, + int rc +){ + const char *zVal; + char zBuf[50]; + zVal = vfstrace_errcode_name(rc); + if( zVal==0 ){ + zVal = vfstrace_errcode_name(rc&0xff); + if( zVal ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%s | 0x%x", zVal, rc&0xffff00); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d (0x%x)", rc, rc); + } + zVal = zBuf; + } + vfstrace_printf(pInfo, zFormat, zVal); +} + +/* +** Append to a buffer. +*/ +static void strappend(char *z, int *pI, const char *zAppend){ + int i = *pI; + while( zAppend[0] ){ z[i++] = *(zAppend++); } + z[i] = 0; + *pI = i; +} + +/* +** Close an vfstrace-file. +*/ +static int vfstraceClose(sqlite3_file *pFile){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xClose(%s)", pInfo->zVfsName, p->zFName); + rc = p->pReal->pMethods->xClose(p->pReal); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + if( rc==SQLITE_OK ){ + sqlite3_free((void*)p->base.pMethods); + p->base.pMethods = 0; + } + return rc; +} + +/* +** Read data from an vfstrace-file. +*/ +static int vfstraceRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xRead(%s,n=%d,ofst=%lld)", + pInfo->zVfsName, p->zFName, iAmt, iOfst); + rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + +/* +** Write data to an vfstrace-file. +*/ +static int vfstraceWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xWrite(%s,n=%d,ofst=%lld)", + pInfo->zVfsName, p->zFName, iAmt, iOfst); + rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + +/* +** Truncate an vfstrace-file. +*/ +static int vfstraceTruncate(sqlite3_file *pFile, sqlite_int64 size){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xTruncate(%s,%lld)", pInfo->zVfsName, p->zFName, + size); + rc = p->pReal->pMethods->xTruncate(p->pReal, size); + vfstrace_printf(pInfo, " -> %d\n", rc); + return rc; +} + +/* +** Sync an vfstrace-file. +*/ +static int vfstraceSync(sqlite3_file *pFile, int flags){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + int i; + char zBuf[100]; + memcpy(zBuf, "|0", 3); + i = 0; + if( flags & SQLITE_SYNC_FULL ) strappend(zBuf, &i, "|FULL"); + else if( flags & SQLITE_SYNC_NORMAL ) strappend(zBuf, &i, "|NORMAL"); + if( flags & SQLITE_SYNC_DATAONLY ) strappend(zBuf, &i, "|DATAONLY"); + if( flags & ~(SQLITE_SYNC_FULL|SQLITE_SYNC_DATAONLY) ){ + sqlite3_snprintf(sizeof(zBuf)-i, &zBuf[i], "|0x%x", flags); + } + vfstrace_printf(pInfo, "%s.xSync(%s,%s)", pInfo->zVfsName, p->zFName, + &zBuf[1]); + rc = p->pReal->pMethods->xSync(p->pReal, flags); + vfstrace_printf(pInfo, " -> %d\n", rc); + return rc; +} + +/* +** Return the current file-size of an vfstrace-file. +*/ +static int vfstraceFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xFileSize(%s)", pInfo->zVfsName, p->zFName); + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + vfstrace_print_errcode(pInfo, " -> %s,", rc); + vfstrace_printf(pInfo, " size=%lld\n", *pSize); + return rc; +} + +/* +** Return the name of a lock. +*/ +static const char *lockName(int eLock){ + const char *azLockNames[] = { + "NONE", "SHARED", "RESERVED", "PENDING", "EXCLUSIVE" + }; + if( eLock<0 || eLock>=(int)(sizeof(azLockNames)/sizeof(azLockNames[0])) ){ + return "???"; + }else{ + return azLockNames[eLock]; + } +} + +/* +** Lock an vfstrace-file. +*/ +static int vfstraceLock(sqlite3_file *pFile, int eLock){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xLock(%s,%s)", pInfo->zVfsName, p->zFName, + lockName(eLock)); + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + +/* +** Unlock an vfstrace-file. +*/ +static int vfstraceUnlock(sqlite3_file *pFile, int eLock){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xUnlock(%s,%s)", pInfo->zVfsName, p->zFName, + lockName(eLock)); + rc = p->pReal->pMethods->xUnlock(p->pReal, eLock); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + +/* +** Check if another file-handle holds a RESERVED lock on an vfstrace-file. +*/ +static int vfstraceCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xCheckReservedLock(%s,%d)", + pInfo->zVfsName, p->zFName); + rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", out=%d\n", *pResOut); + return rc; +} + +/* +** File control method. For custom operations on an vfstrace-file. +*/ +static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + char zBuf[100]; + char zBuf2[100]; + char *zOp; + char *zRVal = 0; + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break; + case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break; + case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break; + case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break; + case SQLITE_FCNTL_SIZE_HINT: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "SIZE_HINT,%lld", + *(sqlite3_int64*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "CHUNK_SIZE,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; + case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "PERSIST_WAL,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; + case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: zOp = "POWERSAFE_OVERWRITE"; break; + case SQLITE_FCNTL_PRAGMA: { + const char *const* a = (const char*const*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_BUSYHANDLER: zOp = "BUSYHANDLER"; break; + case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_int64 iMMap = *(sqlite3_int64*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "MMAP_SIZE,%lld",iMMap); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_TRACE: zOp = "TRACE"; break; + case SQLITE_FCNTL_HAS_MOVED: zOp = "HAS_MOVED"; break; + case SQLITE_FCNTL_SYNC: zOp = "SYNC"; break; + case SQLITE_FCNTL_COMMIT_PHASETWO: zOp = "COMMIT_PHASETWO"; break; + case SQLITE_FCNTL_WIN32_SET_HANDLE: zOp = "WIN32_SET_HANDLE"; break; + case SQLITE_FCNTL_WAL_BLOCK: zOp = "WAL_BLOCK"; break; + case SQLITE_FCNTL_ZIPVFS: zOp = "ZIPVFS"; break; + case SQLITE_FCNTL_RBU: zOp = "RBU"; break; + case SQLITE_FCNTL_VFS_POINTER: zOp = "VFS_POINTER"; break; + case SQLITE_FCNTL_JOURNAL_POINTER: zOp = "JOURNAL_POINTER"; break; + case SQLITE_FCNTL_WIN32_GET_HANDLE: zOp = "WIN32_GET_HANDLE"; break; + case SQLITE_FCNTL_PDB: zOp = "PDB"; break; + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: zOp = "BEGIN_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: zOp = "COMMIT_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + zOp = "ROLLBACK_ATOMIC_WRITE"; + break; + } + case SQLITE_FCNTL_LOCK_TIMEOUT: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "LOCK_TIMEOUT,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_DATA_VERSION: zOp = "DATA_VERSION"; break; + case SQLITE_FCNTL_SIZE_LIMIT: zOp = "SIZE_LIMIT"; break; + case SQLITE_FCNTL_CKPT_DONE: zOp = "CKPT_DONE"; break; + case SQLITE_FCNTL_RESERVE_BYTES: zOp = "RESERVED_BYTES"; break; + case SQLITE_FCNTL_CKPT_START: zOp = "CKPT_START"; break; + case SQLITE_FCNTL_EXTERNAL_READER: zOp = "EXTERNAL_READER"; break; + case SQLITE_FCNTL_CKSM_FILE: zOp = "CKSM_FILE"; break; + case SQLITE_FCNTL_RESET_CACHE: zOp = "RESET_CACHE"; break; + case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + default: { + sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); + zOp = zBuf; + break; + } + } + vfstrace_printf(pInfo, "%s.xFileControl(%s,%s)", + pInfo->zVfsName, p->zFName, zOp); + rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + if( rc==SQLITE_OK ){ + switch( op ){ + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", + pInfo->zVfsName, *(char**)pArg); + zRVal = *(char**)pArg; + break; + } + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%lld", *(sqlite3_int64*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_HAS_MOVED: + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_PRAGMA: + case SQLITE_FCNTL_TEMPFILENAME: { + zRVal = *(char**)pArg; + break; + } + } + } + if( zRVal ){ + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", %s\n", zRVal); + }else{ + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + } + return rc; +} + +/* +** Return the sector-size in bytes for an vfstrace-file. +*/ +static int vfstraceSectorSize(sqlite3_file *pFile){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xSectorSize(%s)", pInfo->zVfsName, p->zFName); + rc = p->pReal->pMethods->xSectorSize(p->pReal); + vfstrace_printf(pInfo, " -> %d\n", rc); + return rc; +} + +/* +** Return the device characteristic flags supported by an vfstrace-file. +*/ +static int vfstraceDeviceCharacteristics(sqlite3_file *pFile){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xDeviceCharacteristics(%s)", + pInfo->zVfsName, p->zFName); + rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal); + vfstrace_printf(pInfo, " -> 0x%08x\n", rc); + return rc; +} + +/* +** Shared-memory operations. +*/ +static int vfstraceShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + char zLck[100]; + int i = 0; + memcpy(zLck, "|0", 3); + if( flags & SQLITE_SHM_UNLOCK ) strappend(zLck, &i, "|UNLOCK"); + if( flags & SQLITE_SHM_LOCK ) strappend(zLck, &i, "|LOCK"); + if( flags & SQLITE_SHM_SHARED ) strappend(zLck, &i, "|SHARED"); + if( flags & SQLITE_SHM_EXCLUSIVE ) strappend(zLck, &i, "|EXCLUSIVE"); + if( flags & ~(0xf) ){ + sqlite3_snprintf(sizeof(zLck)-i, &zLck[i], "|0x%x", flags); + } + vfstrace_printf(pInfo, "%s.xShmLock(%s,ofst=%d,n=%d,%s)", + pInfo->zVfsName, p->zFName, ofst, n, &zLck[1]); + rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} +static int vfstraceShmMap( + sqlite3_file *pFile, + int iRegion, + int szRegion, + int isWrite, + void volatile **pp +){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xShmMap(%s,iRegion=%d,szRegion=%d,isWrite=%d,*)", + pInfo->zVfsName, p->zFName, iRegion, szRegion, isWrite); + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} +static void vfstraceShmBarrier(sqlite3_file *pFile){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + vfstrace_printf(pInfo, "%s.xShmBarrier(%s)\n", pInfo->zVfsName, p->zFName); + p->pReal->pMethods->xShmBarrier(p->pReal); +} +static int vfstraceShmUnmap(sqlite3_file *pFile, int delFlag){ + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = p->pInfo; + int rc; + vfstrace_printf(pInfo, "%s.xShmUnmap(%s,delFlag=%d)", + pInfo->zVfsName, p->zFName, delFlag); + rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + + + +/* +** Open an vfstrace file handle. +*/ +static int vfstraceOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + int rc; + vfstrace_file *p = (vfstrace_file *)pFile; + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + p->pInfo = pInfo; + p->zFName = zName ? fileTail(zName) : ""; + p->pReal = (sqlite3_file *)&p[1]; + rc = pRoot->xOpen(pRoot, zName, p->pReal, flags, pOutFlags); + vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)", + pInfo->zVfsName, p->zFName, flags); + if( p->pReal->pMethods ){ + sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) ); + const sqlite3_io_methods *pSub = p->pReal->pMethods; + memset(pNew, 0, sizeof(*pNew)); + pNew->iVersion = pSub->iVersion; + pNew->xClose = vfstraceClose; + pNew->xRead = vfstraceRead; + pNew->xWrite = vfstraceWrite; + pNew->xTruncate = vfstraceTruncate; + pNew->xSync = vfstraceSync; + pNew->xFileSize = vfstraceFileSize; + pNew->xLock = vfstraceLock; + pNew->xUnlock = vfstraceUnlock; + pNew->xCheckReservedLock = vfstraceCheckReservedLock; + pNew->xFileControl = vfstraceFileControl; + pNew->xSectorSize = vfstraceSectorSize; + pNew->xDeviceCharacteristics = vfstraceDeviceCharacteristics; + if( pNew->iVersion>=2 ){ + pNew->xShmMap = pSub->xShmMap ? vfstraceShmMap : 0; + pNew->xShmLock = pSub->xShmLock ? vfstraceShmLock : 0; + pNew->xShmBarrier = pSub->xShmBarrier ? vfstraceShmBarrier : 0; + pNew->xShmUnmap = pSub->xShmUnmap ? vfstraceShmUnmap : 0; + } + pFile->pMethods = pNew; + } + vfstrace_print_errcode(pInfo, " -> %s", rc); + if( pOutFlags ){ + vfstrace_printf(pInfo, ", outFlags=0x%x\n", *pOutFlags); + }else{ + vfstrace_printf(pInfo, "\n"); + } + return rc; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int vfstraceDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + int rc; + vfstrace_printf(pInfo, "%s.xDelete(\"%s\",%d)", + pInfo->zVfsName, zPath, dirSync); + rc = pRoot->xDelete(pRoot, zPath, dirSync); + vfstrace_print_errcode(pInfo, " -> %s\n", rc); + return rc; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int vfstraceAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + int rc; + vfstrace_printf(pInfo, "%s.xAccess(\"%s\",%d)", + pInfo->zVfsName, zPath, flags); + rc = pRoot->xAccess(pRoot, zPath, flags, pResOut); + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", out=%d\n", *pResOut); + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (DEVSYM_MAX_PATHNAME+1) bytes. +*/ +static int vfstraceFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + int rc; + vfstrace_printf(pInfo, "%s.xFullPathname(\"%s\")", + pInfo->zVfsName, zPath); + rc = pRoot->xFullPathname(pRoot, zPath, nOut, zOut); + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", out=\"%.*s\"\n", nOut, zOut); + return rc; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *vfstraceDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + vfstrace_printf(pInfo, "%s.xDlOpen(\"%s\")\n", pInfo->zVfsName, zPath); + return pRoot->xDlOpen(pRoot, zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void vfstraceDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + vfstrace_printf(pInfo, "%s.xDlError(%d)", pInfo->zVfsName, nByte); + pRoot->xDlError(pRoot, nByte, zErrMsg); + vfstrace_printf(pInfo, " -> \"%s\"", zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*vfstraceDlSym(sqlite3_vfs *pVfs,void *p,const char *zSym))(void){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + vfstrace_printf(pInfo, "%s.xDlSym(\"%s\")\n", pInfo->zVfsName, zSym); + return pRoot->xDlSym(pRoot, p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void vfstraceDlClose(sqlite3_vfs *pVfs, void *pHandle){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + vfstrace_printf(pInfo, "%s.xDlOpen()\n", pInfo->zVfsName); + pRoot->xDlClose(pRoot, pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int vfstraceRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + vfstrace_printf(pInfo, "%s.xRandomness(%d)\n", pInfo->zVfsName, nByte); + return pRoot->xRandomness(pRoot, nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int vfstraceSleep(sqlite3_vfs *pVfs, int nMicro){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xSleep(pRoot, nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int vfstraceCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xCurrentTime(pRoot, pTimeOut); +} +static int vfstraceCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xCurrentTimeInt64(pRoot, pTimeOut); +} + +/* +** Return th3 most recent error code and message +*/ +static int vfstraceGetLastError(sqlite3_vfs *pVfs, int iErr, char *zErr){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xGetLastError(pRoot, iErr, zErr); +} + +/* +** Override system calls. +*/ +static int vfstraceSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pFunc +){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xSetSystemCall(pRoot, zName, pFunc); +} +static sqlite3_syscall_ptr vfstraceGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xGetSystemCall(pRoot, zName); +} +static const char *vfstraceNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; + sqlite3_vfs *pRoot = pInfo->pRootVfs; + return pRoot->xNextSystemCall(pRoot, zName); +} + + +/* +** Clients invoke this routine to construct a new trace-vfs shim. +** +** Return SQLITE_OK on success. +** +** SQLITE_NOMEM is returned in the case of a memory allocation error. +** SQLITE_NOTFOUND is returned if zOldVfsName does not exist. +*/ +int vfstrace_register( + const char *zTraceName, /* Name of the newly constructed VFS */ + const char *zOldVfsName, /* Name of the underlying VFS */ + int (*xOut)(const char*,void*), /* Output routine. ex: fputs */ + void *pOutArg, /* 2nd argument to xOut. ex: stderr */ + int makeDefault /* True to make the new VFS the default */ +){ + sqlite3_vfs *pNew; + sqlite3_vfs *pRoot; + vfstrace_info *pInfo; + size_t nName; + size_t nByte; + + pRoot = sqlite3_vfs_find(zOldVfsName); + if( pRoot==0 ) return SQLITE_NOTFOUND; + nName = strlen(zTraceName); + nByte = sizeof(*pNew) + sizeof(*pInfo) + nName + 1; + pNew = sqlite3_malloc64( nByte ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, nByte); + pInfo = (vfstrace_info*)&pNew[1]; + pNew->iVersion = pRoot->iVersion; + pNew->szOsFile = pRoot->szOsFile + sizeof(vfstrace_file); + pNew->mxPathname = pRoot->mxPathname; + pNew->zName = (char*)&pInfo[1]; + memcpy((char*)&pInfo[1], zTraceName, nName+1); + pNew->pAppData = pInfo; + pNew->xOpen = vfstraceOpen; + pNew->xDelete = vfstraceDelete; + pNew->xAccess = vfstraceAccess; + pNew->xFullPathname = vfstraceFullPathname; + pNew->xDlOpen = pRoot->xDlOpen==0 ? 0 : vfstraceDlOpen; + pNew->xDlError = pRoot->xDlError==0 ? 0 : vfstraceDlError; + pNew->xDlSym = pRoot->xDlSym==0 ? 0 : vfstraceDlSym; + pNew->xDlClose = pRoot->xDlClose==0 ? 0 : vfstraceDlClose; + pNew->xRandomness = vfstraceRandomness; + pNew->xSleep = vfstraceSleep; + pNew->xCurrentTime = vfstraceCurrentTime; + pNew->xGetLastError = pRoot->xGetLastError==0 ? 0 : vfstraceGetLastError; + if( pNew->iVersion>=2 ){ + pNew->xCurrentTimeInt64 = pRoot->xCurrentTimeInt64==0 ? 0 : + vfstraceCurrentTimeInt64; + if( pNew->iVersion>=3 ){ + pNew->xSetSystemCall = pRoot->xSetSystemCall==0 ? 0 : + vfstraceSetSystemCall; + pNew->xGetSystemCall = pRoot->xGetSystemCall==0 ? 0 : + vfstraceGetSystemCall; + pNew->xNextSystemCall = pRoot->xNextSystemCall==0 ? 0 : + vfstraceNextSystemCall; + } + } + pInfo->pRootVfs = pRoot; + pInfo->xOut = xOut; + pInfo->pOutArg = pOutArg; + pInfo->zVfsName = pNew->zName; + pInfo->pTraceVfs = pNew; + vfstrace_printf(pInfo, "%s.enabled_for(\"%s\")\n", + pInfo->zVfsName, pRoot->zName); + return sqlite3_vfs_register(pNew, makeDefault); +} + +/* +** Look for the named VFS. If it is a TRACEVFS, then unregister it +** and delete it. +*/ +void vfstrace_unregister(const char *zTraceName){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zTraceName); + if( pVfs==0 ) return; + if( pVfs->xOpen!=vfstraceOpen ) return; + sqlite3_vfs_unregister(pVfs); + sqlite3_free(pVfs); +} + +/************************* End ../ext/misc/vfstrace.c ********************/ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#define SQLITE_SHELL_HAVE_RECOVER 1 +#else +#define SQLITE_SHELL_HAVE_RECOVER 0 +#endif +#if SQLITE_SHELL_HAVE_RECOVER +/************************* Begin ../ext/recover/sqlite3recover.h ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface to the "recover" extension - +** an SQLite extension designed to recover data from corrupted database +** files. +*/ + +/* +** OVERVIEW: +** +** To use the API to recover data from a corrupted database, an +** application: +** +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). +** +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on +** the handle until it returns something other than SQLITE_OK. If it +** returns SQLITE_DONE, then the recovery operation completed without +** error. If it returns some other non-SQLITE_OK value, then an error +** has occurred. +** +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. +** +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). +** +** The application may abandon the recovery operation at any point +** before it is finished by passing the sqlite3_recover handle to +** sqlite3_recover_finish(). This is not an error, but the final state +** of the output database, or the results of running the partial script +** delivered to the SQL callback, are undefined. +*/ + +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H + +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An instance of the sqlite3_recover object represents a recovery +** operation in progress. +** +** Constructors: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** Destructor: +** +** sqlite3_recover_finish() +** +** Methods: +** +** sqlite3_recover_config() +** sqlite3_recover_errcode() +** sqlite3_recover_errmsg() +** sqlite3_recover_run() +** sqlite3_recover_step() +*/ +typedef struct sqlite3_recover sqlite3_recover; + +/* +** These two APIs attempt to create and return a new sqlite3_recover object. +** In both cases the first two arguments identify the (possibly +** corrupt) database to recover data from. The first argument is an open +** database handle and the second the name of a database attached to that +** handle (i.e. "main", "temp" or the name of an attached database). +** +** If sqlite3_recover_init() is used to create the new sqlite3_recover +** handle, then data is recovered into a new database, identified by +** string parameter zUri. zUri may be an absolute or relative file path, +** or may be an SQLite URI. If the identified database file already exists, +** it is overwritten. +** +** If sqlite3_recover_init_sql() is invoked, then any recovered data will +** be returned to the user as a series of SQL statements. Executing these +** SQL statements results in the same database as would have been created +** had sqlite3_recover_init() been used. For each SQL statement in the +** output, the callback function passed as the third argument (xSql) is +** invoked once. The first parameter is a passed a copy of the fourth argument +** to this function (pCtx) as its first parameter, and a pointer to a +** nul-terminated buffer containing the SQL statement formated as UTF-8 as +** the second. If the xSql callback returns any value other than SQLITE_OK, +** then processing is immediately abandoned and the value returned used as +** the recover handle error code (see below). +** +** If an out-of-memory error occurs, NULL may be returned instead of +** a valid handle. In all other cases, it is the responsibility of the +** application to avoid resource leaks by ensuring that +** sqlite3_recover_finish() is called on all allocated handles. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pCtx +); + +/* +** Configure an sqlite3_recover object that has just been created using +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function +** may only be called before the first call to sqlite3_recover_step() +** or sqlite3_recover_run() on the object. +** +** The second argument passed to this function must be one of the +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument +** depend on the specific SQLITE_RECOVER_* symbol in use. +** +** SQLITE_OK is returned if the configuration operation was successful, +** or an SQLite error code otherwise. +*/ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +/* +** SQLITE_RECOVER_LOST_AND_FOUND: +** The pArg argument points to a string buffer containing the name +** of a "lost-and-found" table in the output database, or NULL. If +** the argument is non-NULL and the database contains seemingly +** valid pages that cannot be associated with any table in the +** recovered part of the schema, data is extracted from these +** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1) and a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. The default +** value is 0 (clear). +** +** SQLITE_RECOVER_ROWIDS: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1), then an attempt is made to recover rowid values +** that are not also INTEGER PRIMARY KEY values. If this option is +** clear, then new rowids are assigned to all recovered rows. The +** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. +*/ +#define SQLITE_RECOVER_LOST_AND_FOUND 1 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2 +#define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 + +/* +** Perform a unit of work towards the recovery operation. This function +** must normally be called multiple times to complete database recovery. +** +** If no error occurs but the recovery operation is not completed, this +** function returns SQLITE_OK. If recovery has been completed successfully +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not +** considered an error if some or all of the data cannot be recovered +** due to database corruption. +** +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, +** all further such calls on the same recover handle are no-ops that return +** the same non-SQLITE_OK value. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +/* +** Run the recovery operation to completion. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. Calling this function is the same +** as executing: +** +** while( SQLITE_OK==sqlite3_recover_step(p) ); +** return sqlite3_recover_errcode(p); +*/ +int sqlite3_recover_run(sqlite3_recover*); + +/* +** If an error has been encountered during a prior call to +** sqlite3_recover_step(), then this function attempts to return a +** pointer to a buffer containing an English language explanation of +** the error. If no error message is available, or if an out-of memory +** error occurs while attempting to allocate a buffer in which to format +** the error message, NULL is returned. +** +** The returned buffer remains valid until the sqlite3_recover handle is +** destroyed using sqlite3_recover_finish(). +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +/* +** If this function is called on an sqlite3_recover handle after +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3_recover_errcode(sqlite3_recover*); + +/* +** Clean up a recovery object created by a call to sqlite3_recover_init(). +** The results of using a recovery object with any API after it has been +** passed to this function are undefined. +** +** This function returns the same value as sqlite3_recover_errcode(). +*/ +int sqlite3_recover_finish(sqlite3_recover*); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ + +/************************* End ../ext/recover/sqlite3recover.h ********************/ +# ifndef SQLITE_HAVE_SQLITE3R +/************************* Begin ../ext/recover/dbdata.c ******************/ +/* +** 2019-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of two eponymous virtual tables, +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the +** "sqlite_dbpage" eponymous virtual table be available. +** +** SQLITE_DBDATA: +** sqlite_dbdata is used to extract data directly from a database b-tree +** page and its associated overflow pages, bypassing the b-tree layer. +** The table schema is equivalent to: +** +** CREATE TABLE sqlite_dbdata( +** pgno INTEGER, +** cell INTEGER, +** field INTEGER, +** value ANY, +** schema TEXT HIDDEN +** ); +** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND +** "schema". +** +** Each page of the database is inspected. If it cannot be interpreted as +** a b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the +** table contains one row for each field in the record associated with +** each cell on the page. For intkey b-trees, the key value is stored in +** field -1. +** +** For example, for the database: +** +** CREATE TABLE t1(a, b); -- root page is page 2 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: +** +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES +** (2, 0, -1, 5 ), +** (2, 0, 0, 'v' ), +** (2, 0, 1, 'five'), +** (2, 1, -1, 10 ), +** (2, 1, 0, 'x' ), +** (2, 1, 1, 'ten' ); +** +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. +** +** SQLITE_DBPTR: +** The sqlite_dbptr table has the following schema: +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); +** +** It contains one entry for each b-tree pointer between a parent and +** child page in the database. +*/ + +#if !defined(SQLITEINT_H) +/* #include "sqlite3.h" */ + +/* typedef unsigned char u8; */ +/* typedef unsigned int u32; */ + +#endif +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +#define DBDATA_PADDING_BYTES 100 + +typedef struct DbdataTable DbdataTable; +typedef struct DbdataCursor DbdataCursor; +typedef struct DbdataBuffer DbdataBuffer; + +/* +** Buffer type. +*/ +struct DbdataBuffer { + u8 *aBuf; + sqlite3_int64 nBuf; +}; + +/* Cursor object */ +struct DbdataCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + + int iPgno; /* Current page number */ + u8 *aPage; /* Buffer containing page */ + int nPage; /* Size of aPage[] in bytes */ + int nCell; /* Number of cells on aPage[] */ + int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + int szDb; + sqlite3_int64 iRowid; + + /* Only for the sqlite_dbdata table */ + DbdataBuffer rec; + sqlite3_int64 nRec; /* Size of pRec[] in bytes */ + sqlite3_int64 nHdr; /* Size of header in bytes */ + int iField; /* Current field number */ + u8 *pHdrPtr; + u8 *pPtr; + u32 enc; /* Text encoding */ + + sqlite3_int64 iIntkey; /* Integer key value */ +}; + +/* Table object */ +struct DbdataTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + int bPtr; /* True for sqlite3_dbptr table */ +}; + +/* Column and schema definitions for sqlite_dbdata */ +#define DBDATA_COLUMN_PGNO 0 +#define DBDATA_COLUMN_CELL 1 +#define DBDATA_COLUMN_FIELD 2 +#define DBDATA_COLUMN_VALUE 3 +#define DBDATA_COLUMN_SCHEMA 4 +#define DBDATA_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " cell INTEGER," \ + " field INTEGER," \ + " value ANY," \ + " schema TEXT HIDDEN" \ + ")" + +/* Column and schema definitions for sqlite_dbptr */ +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" + +/* +** Ensure the buffer passed as the first argument is at least nMin bytes +** in size. If an error occurs while attempting to resize the buffer, +** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ + if( nMin>pBuf->nBuf ){ + sqlite3_int64 nNew = nMin+16384; + u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); + + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->aBuf = aNew; + pBuf->nBuf = nNew; + } + return SQLITE_OK; +} + +/* +** Release the allocation managed by buffer pBuf. +*/ +static void dbdataBufferFree(DbdataBuffer *pBuf){ + sqlite3_free(pBuf->aBuf); + memset(pBuf, 0, sizeof(*pBuf)); +} + +/* +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual +** table. +*/ +static int dbdataConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbdataTable *pTab = 0; + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); + + (void)argc; + (void)argv; + (void)pzErr; + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); + if( rc==SQLITE_OK ){ + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(DbdataTable)); + pTab->db = db; + pTab->bPtr = (pAux!=0); + } + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. +*/ +static int dbdataDisconnect(sqlite3_vtab *pVtab){ + DbdataTable *pTab = (DbdataTable*)pVtab; + if( pTab ){ + sqlite3_finalize(pTab->pStmt); + sqlite3_free(pVtab); + } + return SQLITE_OK; +} + +/* +** This function interprets two types of constraints: +** +** schema=? +** pgno=? +** +** If neither are present, idxNum is set to 0. If schema=? is present, +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit +** in idxNum is set. +** +** If both parameters are present, schema is in position 0 and pgno in +** position 1. +*/ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ + DbdataTable *pTab = (DbdataTable*)tab; + int i; + int iSchema = -1; + int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( p->iColumn==colSchema ){ + if( p->usable==0 ) return SQLITE_CONSTRAINT; + iSchema = i; + } + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ + iPgno = i; + } + } + } + + if( iSchema>=0 ){ + pIdx->aConstraintUsage[iSchema].argvIndex = 1; + pIdx->aConstraintUsage[iSchema].omit = 1; + } + if( iPgno>=0 ){ + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdx->aConstraintUsage[iPgno].omit = 1; + pIdx->estimatedCost = 100; + pIdx->estimatedRows = 50; + + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ + int iCol = pIdx->aOrderBy[0].iColumn; + if( pIdx->nOrderBy==1 ){ + pIdx->orderByConsumed = (iCol==0 || iCol==1); + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); + } + } + + }else{ + pIdx->estimatedCost = 100000000; + pIdx->estimatedRows = 1000000000; + } + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + return SQLITE_OK; +} + +/* +** Open a new sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbdataCursor *pCsr; + + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(DbdataCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Restore a cursor object to the state it was in when first allocated +** by dbdataOpen(). +*/ +static void dbdataResetCursor(DbdataCursor *pCsr){ + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); + if( pTab->pStmt==0 ){ + pTab->pStmt = pCsr->pStmt; + }else{ + sqlite3_finalize(pCsr->pStmt); + } + pCsr->pStmt = 0; + pCsr->iPgno = 1; + pCsr->iCell = 0; + pCsr->iField = 0; + pCsr->bOnePage = 0; + sqlite3_free(pCsr->aPage); + dbdataBufferFree(&pCsr->rec); + pCsr->aPage = 0; + pCsr->nRec = 0; +} + +/* +** Close an sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataClose(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + dbdataResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Utility methods to decode 16 and 32-bit big-endian unsigned integers. +*/ +static u32 get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); +} + +/* +** Load page pgno from the database via the sqlite_dbpage virtual table. +** If successful, set (*ppPage) to point to a buffer containing the page +** data, (*pnPage) to the size of that buffer in bytes and return +** SQLITE_OK. In this case it is the responsibility of the caller to +** eventually free the buffer using sqlite3_free(). +** +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and +** return an SQLite error code. +*/ +static int dbdataLoadPage( + DbdataCursor *pCsr, /* Cursor object */ + u32 pgno, /* Page number of page to load */ + u8 **ppPage, /* OUT: pointer to page buffer */ + int *pnPage /* OUT: Size of (*ppPage) in bytes */ +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = pCsr->pStmt; + + *ppPage = 0; + *pnPage = 0; + if( pgno>0 ){ + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + } + *ppPage = pPage; + *pnPage = nCopy; + } + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } + } + u = (u<<8) + (z[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; +} + +/* +** Like dbdataGetVarint(), but set the output to 0 if it is less than 0 +** or greater than 0xFFFFFFFF. This can be used for all varints in an +** SQLite database except for key values in intkey tables. +*/ +static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 val; + int nRet = dbdataGetVarint(z, &val); + if( val<0 || val>0xFFFFFFFF ) val = 0; + *pVal = val; + return nRet; +} + +/* +** Return the number of bytes of space used by an SQLite value of type +** eType. +*/ +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + if( eType>0 ){ + return ((eType-12) / 2); + } + return 0; + } +} + +/* +** Load a value of type eType from buffer pData and use it to set the +** result of context object pCtx. +*/ +static void dbdataValue( + sqlite3_context *pCtx, + u32 enc, + int eType, + u8 *pData, + sqlite3_int64 nData +){ + if( eType>=0 ){ + if( dbdataValueBytes(eType)<=nData ){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + switch( enc ){ + #ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + #endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } + }else{ + if( eType==7 ){ + sqlite3_result_double(pCtx, 0.0); + }else if( eType<7 ){ + sqlite3_result_int(pCtx, 0); + }else if( eType%2 ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + }else{ + sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); + } + } + } +} + +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + +/* +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. +*/ +static int dbdataNext(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + + pCsr->iRowid++; + while( 1 ){ + int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); + int bNextPage = 0; + + if( pCsr->aPage==0 ){ + while( 1 ){ + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + if( pCsr->aPage && pCsr->nPage>=256 ) break; + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + } + + assert( iOff+3+2<=pCsr->nPage ); + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } + } + + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; + } + pCsr->iCell++; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ + return SQLITE_OK; + } + }else{ + /* If there is no record loaded, load it now. */ + assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); + if( pCsr->nRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } + + if( pCsr->iCell>=pCsr->nCell ){ + bNextPage = 1; + }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; + + if( iCellPtr>pCsr->nPage ){ + bNextPage = 1; + }else{ + iOff = get_uint16(&pCsr->aPage[iCellPtr]); + } + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ + bNextPage = 1; + }else{ + iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); + if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; + if( nPayload==0 ) nPayload = 1; + } + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid && !bNextPage && iOffnPage ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Figure out how much data to read from the local page */ + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; + }else{ + X = ((U-12)*64/255)-23; + } + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } + } + + if( bNextPage || nLocal+iOff>pCsr->nPage ){ + bNextPage = 1; + }else{ + + /* Allocate space for payload. And a bit more to catch small buffer + ** overruns caused by attempting to read a varint or similar from + ** near the end of a corrupt record. */ + rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); + if( rc!=SQLITE_OK ) return rc; + assert( pCsr->rec.aBuf!=0 ); + assert( nPayload!=0 ); + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + if( aOvfl==0 ) break; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + pgnoOvfl = get_uint32(aOvfl); + sqlite3_free(aOvfl); + } + nPayload -= nRem; + } + memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); + pCsr->nRec = nPayload; + + iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); + if( nHdr>nPayload ) nHdr = 0; + pCsr->nHdr = nHdr; + pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; + pCsr->iField = (bHasRowid ? -1 : 0); + } + } + }else{ + pCsr->iField++; + if( pCsr->iField>0 ){ + sqlite3_int64 iType; + if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ + bNextPage = 1; + }else{ + int szField = 0; + pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); + szField = dbdataValueBytes(iType); + if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))pPtr = &pCsr->rec.aBuf[pCsr->nRec]; + }else{ + pCsr->pPtr += szField; + } + } + } + } + + if( bNextPage ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + pCsr->nRec = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ + return SQLITE_OK; + } + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + pCsr->nRec = 0; + pCsr->iCell++; + } + } + } + + assert( !"can't get here" ); + return SQLITE_OK; +} + +/* +** Return true if the cursor is at EOF. +*/ +static int dbdataEof(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + return pCsr->aPage==0; +} + +/* +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. +*/ +static int dbdataIsFunction(const char *zSchema){ + size_t n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return (int)n-2; + } + return 0; +} + +/* +** Determine the size in pages of database zSchema (where zSchema is +** "main", "temp" or the name of an attached database) and set +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, +** an SQLite error code. +*/ +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; + char *zSql = 0; + int rc, rc2; + int nFunc = 0; + sqlite3_stmt *pStmt = 0; + + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + } + if( zSql==0 ) return SQLITE_NOMEM; + + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + pCsr->szDb = sqlite3_column_int(pStmt, 0); + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +/* +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. +*/ +static int dbdataGetEncoding(DbdataCursor *pCsr){ + int rc = SQLITE_OK; + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + if( rc==SQLITE_OK && nPg1>=(56+4) ){ + pCsr->enc = get_uint32(&aPg1[56]); + } + sqlite3_free(aPg1); + return rc; +} + + +/* +** xFilter method for sqlite_dbdata and sqlite_dbptr. +*/ +static int dbdataFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + int rc = SQLITE_OK; + const char *zSchema = "main"; + (void)idxStr; + (void)argc; + + dbdataResetCursor(pCsr); + assert( pCsr->iPgno==1 ); + if( idxNum & 0x01 ){ + zSchema = (const char*)sqlite3_value_text(argv[0]); + if( zSchema==0 ) zSchema = ""; + } + if( idxNum & 0x02 ){ + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; + }else{ + rc = dbdataDbsize(pCsr, zSchema); + } + + if( rc==SQLITE_OK ){ + int nFunc = 0; + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + } + + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ + if( rc==SQLITE_OK ){ + rc = dbdataGetEncoding(pCsr); + } + + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + + if( rc==SQLITE_OK ){ + rc = dbdataNext(pCursor); + } + return rc; +} + +/* +** Return a column for the sqlite_dbdata or sqlite_dbptr table. +*/ +static int dbdataColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + if( iOff>pCsr->nPage ) return SQLITE_OK; + iOff = get_uint16(&pCsr->aPage[iOff]); + } + if( iOff<=pCsr->nPage ){ + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + } + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ + sqlite3_int64 iType; + dbdataGetVarintU32(pCsr->pHdrPtr, &iType); + dbdataValue( + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr + ); + } + break; + } + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +*/ +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + + +/* +** Invoke this routine to register the "sqlite_dbdata" virtual table module +*/ +static int sqlite3DbdataRegister(sqlite3 *db){ + static sqlite3_module dbdata_module = { + 0, /* iVersion */ + 0, /* xCreate */ + dbdataConnect, /* xConnect */ + dbdataBestIndex, /* xBestIndex */ + dbdataDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + dbdataOpen, /* xOpen - open a cursor */ + dbdataClose, /* xClose - close a cursor */ + dbdataFilter, /* xFilter - configure scan constraints */ + dbdataNext, /* xNext - advance a cursor */ + dbdataEof, /* xEof - check for end of scan */ + dbdataColumn, /* xColumn - read data */ + dbdataRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); + } + return rc; +} + +int sqlite3_dbdata_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + (void)pzErrMsg; + return sqlite3DbdataRegister(db); +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/recover/dbdata.c ********************/ +/************************* Begin ../ext/recover/sqlite3recover.c ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +/* #include "sqlite3recover.h" */ +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* typedef unsigned int u32; */ +/* typedef unsigned char u8; */ +/* typedef sqlite3_int64 i64; */ + +typedef struct RecoverTable RecoverTable; +typedef struct RecoverColumn RecoverColumn; + +/* +** When recovering rows of data that can be associated with table +** definitions recovered from the sqlite_schema table, each table is +** represented by an instance of the following object. +** +** iRoot: +** The root page in the original database. Not necessarily (and usually +** not) the same in the recovered database. +** +** zTab: +** Name of the table. +** +** nCol/aCol[]: +** aCol[] is an array of nCol columns. In the order in which they appear +** in the table. +** +** bIntkey: +** Set to true for intkey tables, false for WITHOUT ROWID. +** +** iRowidBind: +** Each column in the aCol[] array has associated with it the index of +** the bind parameter its values will be bound to in the INSERT statement +** used to construct the output database. If the table does has a rowid +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the +** index of the bind paramater to which the rowid value should be bound. +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY +** KEY column, then the rowid value should be bound to the index associated +** with the column. +** +** pNext: +** All RecoverTable objects used by the recovery operation are allocated +** and populated as part of creating the recovered database schema in +** the output database, before any non-schema data are recovered. They +** are then stored in a singly-linked list linked by this variable beginning +** at sqlite3_recover.pTblList. +*/ +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ + RecoverTable *pNext; +}; + +/* +** Each database column is represented by an instance of the following object +** stored in the RecoverTable.aCol[] array of the associated table. +** +** iField: +** The index of the associated field within database records. Or -1 if +** there is no associated field (e.g. for virtual generated columns). +** +** iBind: +** The bind index of the INSERT statement to bind this columns values +** to. Or 0 if there is no such index (iff (iField<0)). +** +** bIPK: +** True if this is the INTEGER PRIMARY KEY column. +** +** zCol: +** Name of column. +** +** eHidden: +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). +*/ +struct RecoverColumn { + int iField; /* Field in record on disk */ + int iBind; /* Binding to use in INSERT */ + int bIPK; /* True for IPK column */ + char *zCol; + int eHidden; +}; + +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */ +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */ +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */ +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */ + +/* +** Bitmap object used to track pages in the input database. Allocated +** and manipulated only by the following functions: +** +** recoverBitmapAlloc() +** recoverBitmapFree() +** recoverBitmapSet() +** recoverBitmapQuery() +** +** nPg: +** Largest page number that may be stored in the bitmap. The range +** of valid keys is 1 to nPg, inclusive. +** +** aElem[]: +** Array large enough to contain a bit for each key. For key value +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. +** In other words, the following is true if bit iKey is set, or +** false if it is clear: +** +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 +*/ +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[1]; /* Array of 32-bit bitmasks */ +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data for tables identified in the recovered schema (state +** RECOVER_STATE_WRITING). +*/ +typedef struct RecoverStateW1 RecoverStateW1; +struct RecoverStateW1 { + sqlite3_stmt *pTbls; + sqlite3_stmt *pSel; + sqlite3_stmt *pInsert; + int nInsert; + + RecoverTable *pTab; /* Table currently being written */ + int nMax; /* Max column count in any schema table */ + sqlite3_value **apVal; /* Array of nMax values */ + int nVal; /* Number of valid entries in apVal[] */ + int bHaveRowid; + i64 iRowid; + i64 iPrevPage; + int iPrevCell; +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data destined for the lost and found table (states +** RECOVER_STATE_LOSTANDFOUND[123]). +*/ +typedef struct RecoverStateLAF RecoverStateLAF; +struct RecoverStateLAF { + RecoverBitmap *pUsed; + i64 nPg; /* Size of db in pages */ + sqlite3_stmt *pAllAndParent; + sqlite3_stmt *pMapInsert; + sqlite3_stmt *pMaxField; + sqlite3_stmt *pUsedPages; + sqlite3_stmt *pFindRoot; + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ + sqlite3_stmt *pAllPage; + sqlite3_stmt *pPageData; + sqlite3_value **apVal; + int nMaxField; +}; + +/* +** Main recover handle structure. +*/ +struct sqlite3_recover { + /* Copies of sqlite3_recover_init[_sql]() parameters */ + sqlite3 *dbIn; /* Input database */ + char *zDb; /* Name of input db ("main" etc.) */ + char *zUri; /* URI for output database */ + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ + + /* Values configured by sqlite3_recover_config() */ + char *zStateDb; /* State database to use (or NULL) */ + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ + + int pgsz; + int detected_pgsz; + int nReserve; + u8 *pPage1Disk; + u8 *pPage1Cache; + + /* Error code and error message */ + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + int eState; + int bCloseTransaction; + + /* Variables used with eState==RECOVER_STATE_WRITING */ + RecoverStateW1 w1; + + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ + RecoverStateLAF laf; + + /* Fields used within sqlite3_recover_run() */ + sqlite3 *dbOut; /* Output database */ + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ + RecoverTable *pTblList; /* List of tables recovered from schema */ +}; + +/* +** The various states in which an sqlite3_recover object may exist: +** +** RECOVER_STATE_INIT: +** The object is initially created in this state. sqlite3_recover_step() +** has yet to be called. This is the only state in which it is permitted +** to call sqlite3_recover_config(). +** +** RECOVER_STATE_WRITING: +** +** RECOVER_STATE_LOSTANDFOUND1: +** State to populate the bitmap of pages used by other tables or the +** database freelist. +** +** RECOVER_STATE_LOSTANDFOUND2: +** Populate the recovery.map table - used to figure out a "root" page +** for each lost page from in the database from which records are +** extracted. +** +** RECOVER_STATE_LOSTANDFOUND3: +** Populate the lost-and-found table itself. +*/ +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND1 2 +#define RECOVER_STATE_LOSTANDFOUND2 3 +#define RECOVER_STATE_LOSTANDFOUND3 4 +#define RECOVER_STATE_SCHEMA2 5 +#define RECOVER_STATE_DONE 6 + + +/* +** Global variables used by this extension. +*/ +typedef struct RecoverGlobal RecoverGlobal; +struct RecoverGlobal { + const sqlite3_io_methods *pMethods; + sqlite3_recover *p; +}; +static RecoverGlobal recover_g; + +/* +** Use this static SQLite mutex to protect the globals during the +** first call to sqlite3_recover_step(). +*/ +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 + + +/* +** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). +*/ +#define RECOVER_ROWID_DEFAULT 1 + +/* +** Mutex handling: +** +** recoverEnterMutex() - Enter the recovery mutex +** recoverLeaveMutex() - Leave the recovery mutex +** recoverAssertMutexHeld() - Assert that the recovery mutex is held +*/ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 +# define recoverEnterMutex() +# define recoverLeaveMutex() +#else +static void recoverEnterMutex(void){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +static void recoverLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +#endif +#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG) +static void recoverAssertMutexHeld(void){ + assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); +} +#else +# define recoverAssertMutexHeld() +#endif + + +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + if( zStr==0 ) return 0; + return (int)(strlen(zStr)&0x7fffffff); +} + +/* +** This function is a no-op if the recover handle passed as the first +** argument already contains an error (if p->errCode!=SQLITE_OK). +** +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte +** bytes in size. If successful, a pointer to the new buffer is returned. Or, +** if an OOM error occurs, NULL is returned and the handle error code +** (p->errCode) set to SQLITE_NOMEM. +*/ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + p->errCode = SQLITE_NOMEM; + } + } + return pRet; +} + +/* +** Set the error code and error message for the recover handle passed as +** the first argument. The error code is set to the value of parameter +** errCode. +** +** Parameter zFmt must be a printf() style formatting string. The handle +** error message is set to the result of using any trailing arguments for +** parameter substitutions in the formatting string. +** +** For example: +** +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); +*/ +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + char *z = 0; + va_list ap; + va_start(ap, zFmt); + if( zFmt ){ + z = sqlite3_vmprintf(zFmt, ap); + } + va_end(ap); + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} + + +/* +** This function is a no-op if p->errCode is initially other than SQLITE_OK. +** In this case it returns NULL. +** +** Otherwise, an attempt is made to allocate and return a bitmap object +** large enough to store a bit for all page numbers between 1 and nPg, +** inclusive. The bitmap is initially zeroed. +*/ +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); + + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; +} + +/* +** Free a bitmap object allocated by recoverBitmapAlloc(). +*/ +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} + +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg && iPg>0 ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. +** Or, if an error occurs, leave an error code and message in the recover +** handle and return a copy of the error code. +*/ +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, db); + } + } + return p->errCode; +} + +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; + } + return z; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt ){ + sqlite3_step(pStmt); + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + +/* +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +** +** SELECT read_i32(, ) +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + + if( (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.laf.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + assert( nArg==1 ); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the argument +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); + return; + }else{ + if( p->pGetPage==0 ){ + pStmt = p->pGetPage = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else if( p->errCode==SQLITE_OK ){ + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const u8 *aPg; + int nPg; + assert( p->errCode==SQLITE_OK ); + aPg = sqlite3_column_blob(pStmt, 0); + nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; + } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + } + recoverReset(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *recoverUnusedString( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Implementation of scalar SQL function "escape_crlf". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. +*/ +static void recoverEscapeCrlf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + (void)argc; + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. +*/ +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); + } + } +} + +/* +** Transfer the following settings from the input database to the output +** database: +** +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and +*/ +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } + + for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){ + const char *zPrag = aPragma[ii]; + sqlite3_stmt *p1 = 0; + p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + recoverFinalize(p, p1); + } + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); + + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } + } + + sqlite3_close(db2); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. +*/ +static int recoverOpenOutput(sqlite3_recover *p){ + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crlf", 1, recoverEscapeCrlf }, + }; + + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ + + assert( p->dbOut==0 ); + + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); + } + + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ + if( p->errCode==SQLITE_OK ){ + p->errCode = sqlite3_dbdata_init(db, 0, 0); + } + + /* Register the custom user-functions with the output handle. */ + for(ii=0; + p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); + ii++){ + p->errCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); + } + + p->dbOut = db; + return p->errCode; +} + +/* +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. +*/ +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + recoverExec(p, p->dbOut, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); +} + + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. +** +** Parameter iRoot must be the root page of table zName in the INPUT +** database. +*/ +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + int iPk = -1; + int iBind = 1; + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + int iField = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int iPKF = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + pNew->bIntkey = 1; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; + } + recoverFinalize(p, pStmt); + + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } + } + } +} + +/* +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: +** +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. +*/ +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND (sql LIKE '%unique%' OR ?1))" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE tbl OR isIndex" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; + + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + if( bTable && !bVirtual ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + sqlite3_free(zFree); + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} + +/* +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: +** +** * views, +** * triggers, +** * non-UNIQUE indexes. +** +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. +*/ +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. +** +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. +** +** For example, if table pTab is: +** +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); +** +** And nField is 4, then the SQL statement prepared and returned is: +** +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); +** +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. +** +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: +** +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; +** +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). +*/ +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + sqlite3_stmt *pRet = 0; + const char *zSep = ""; + const char *zSqlSep = ""; + char *zSql = 0; + char *zFinal = 0; + char *zBind = 0; + int ii; + int bSql = p->xSql ? 1 : 0; + + if( nField<=0 ) return 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + } + zSqlSep = "||', '||"; + zSep = ", "; + } + + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + + if( bSql ){ + zBind = recoverMPrintf(p, + "%z%sescape_crlf(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + } + zSep = ", "; + } + } + + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + } + + pRet = recoverPrepare(p, p->dbOut, zFinal); + sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); + + return pRet; +} + + +/* +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. +*/ +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +/* +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). +** +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. +*/ +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ +){ + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; + + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); + } + + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; + } + recoverReset(p, pProbe); + } + + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); + + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + recoverSqlCallback(p, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); + } + + return zTbl; +} + +/* +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. +*/ +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField +){ + int nTotal = nField + 4; + int ii; + char *zBind = 0; + sqlite3_stmt *pRet = 0; + + if( p->xSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); + } + + sqlite3_free(zBind); + return pRet; +} + +/* +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. +*/ +static int recoverLostAndFoundFindRoot( + sqlite3_recover *p, + i64 iPg, + i64 *piRoot +){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); + } + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; + } + recoverReset(p, pLaf->pFindRoot); + } + return p->errCode; +} + +/* +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. +*/ +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; + + int nVal = -1; + int iPrevCell = 0; + i64 iRoot = 0; + int bHaveRowid = 0; + i64 iRowid = 0; + int ii = 0; + + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); + + if( iPrevCell!=iCell && nVal>=0 ){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ + } + for(ii=0; iinMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); + apVal[iField] = sqlite3_value_dup(pVal); + assert( iField==nVal || (nVal==-1 && iField==0) ); + nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + + iPrevCell = iCell; + } + recoverReset(p, pPageData); + + for(ii=0; iilaf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } + } + } + } + return SQLITE_OK; +} + +/* +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ + + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); + + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" + ); + + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) + ); + } +} + +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + } + + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; + + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + p1->pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0" + ); + + return p->errCode; +} + +/* +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). +*/ +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); + } + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; + + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); + + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; + + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; + + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } + + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; + } + } + assert( p->errCode!=SQLITE_OK || p1->pTab ); + + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; + + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); + + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; + } + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); + } + } + } + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); + } + if( SQLITE_ROW==sqlite3_step(pInsert) ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; + } + + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; + } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; + } + + return p->errCode; +} + +/* +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; + + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" + ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; + } + } + return rc; +} + +/* +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg + ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); + } + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; + } + } + return p->errCode; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +*/ +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls. +*/ +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + + recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + + { +#ifndef NDEBUG + int res = +#endif + sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); + } + p->dbOut = 0; +} + +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} + +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} + +/* +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. +*/ +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } + } + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; +} + +/* +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. +*/ +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; + int ii = 0; + + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n || nByte<4 ) return 0; + if( iNext && iNextiContent ) return 0; + for(ii=0; iin ){ + return 0; + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); + } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); + + if( nPayloadn ){ + return 0; + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; + } + } + + nActual = 0; + for(ii=0; iipMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); +} + +/* +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. +*/ +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} + +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ +static void recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; +} + +/* +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. +*/ +static int recoverVfsDetectPagesize( + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ + u32 nReserve, /* Possible nReserve value */ + i64 nSz /* Size of database file in bytes */ +){ + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + int nBlk = 0; + + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + do { + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOff(u32)p->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); + + p->detected_pgsz = pgsz; + sqlite3_free(aPg); + return rc; +} + +/* +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. +*/ +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + the wal-mode flags (16-bits at offset 18) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[108] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30, + + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 + }; + u8 *a = (u8*)aBuf; + + u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + sqlite3_recover *p = recover_g.p; + + if( pgsz==0x01 ) pgsz = 65536; + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); + + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); + } + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; + nReserve = p->nReserve; + } + + if( pgsz ){ + dbsz = dbFileSize / pgsz; + } + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; + } + + sqlite3_free(p->pPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; + + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + aHdr[18] = a[18]; + aHdr[19] = a[19]; + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz-nReserve); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; + for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){ + memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4); + } + memcpy(aBuf, aHdr, sizeof(aHdr)); + memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr)); + + memcpy(p->pPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; + } + + } + pFd->pMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + } + return rc; +} + +/* +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; + +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); + } +} +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} + +static int recoverVfsFetch( + sqlite3_file *pFd, + sqlite3_int64 iOff, + int iAmt, + void **pp +){ + (void)pFd; + (void)iOff; + (void)iAmt; + *pp = 0; + return SQLITE_OK; +} +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ + (void)pFd; + (void)iOff; + (void)p; + return SQLITE_OK; +} + +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. +*/ +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + assert( pFd==0 || pFd->pMethods!=&recover_methods ); + if( pFd && pFd->pMethods ){ + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); + recover_g.pMethods = pFd->pMethods; + recover_g.p = p; + recover_methods.iVersion = iVersion; + pFd->pMethods = &recover_methods; + } +} + +/* +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. +*/ +static void recoverUninstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + if( pFd && pFd->pMethods ){ + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + recover_g.p = 0; + } +} + +/* +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. +*/ +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + + recoverEnterMutex(); + recoverInstallWrapper(p); + + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); + recoverCacheSchema(p); + + recoverUninstallWrapper(p); + recoverLeaveMutex(); + + recoverExec(p, p->dbOut, "BEGIN"); + + recoverWriteSchema1(p); + p->eState = RECOVER_STATE_WRITING; + break; + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND1; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; + } + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); + } + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); + } + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } + break; + } + + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; + + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; + + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; + } +} + + +/* +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ +sqlite3_recover *recoverInit( + sqlite3* db, + const char *zDb, + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; + } + + return pRet; +} + +/* +** Initialize a recovery handle that creates a new database containing +** the recovered data. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + return recoverInit(db, zDb, zUri, 0, 0); +} + +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, 0, xSql, pSqlCtx); +} + +/* +** Return the handle error message, if any. +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; +} + +/* +** Return the handle error code. +*/ +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +/* +** Configure the handle. +*/ +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + /* This undocumented magic configuration option is used to set the + ** name of the auxiliary database that is ATTACH-ed to the database + ** connection and used to hold state information during the + ** recovery process. This option is for debugging use only and + ** is subject to change or removal at any time. */ + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); + break; + + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } + break; + } + + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; + break; + + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; + break; + + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } + } + + return rc; +} + +/* +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; +} + +/* +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. +*/ +int sqlite3_recover_run(sqlite3_recover *p){ + while( SQLITE_OK==sqlite3_recover_step(p) ); + return sqlite3_recover_errcode(p); +} + + +/* +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. +*/ +int sqlite3_recover_finish(sqlite3_recover *p){ + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); + sqlite3_free(p); + } + return rc; +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/recover/sqlite3recover.c ********************/ +# endif /* SQLITE_HAVE_SQLITE3R */ +#endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif + +#if defined(SQLITE_ENABLE_SESSION) +/* +** State information for a single open session +*/ +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ +}; +#endif + +typedef struct ExpertInfo ExpertInfo; +struct ExpertInfo { + sqlite3expert *pExpert; + int bVerbose; +}; + +/* A single line in the EQP output */ +typedef struct EQPGraphRow EQPGraphRow; +struct EQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + EQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct EQPGraph EQPGraph; +struct EQPGraph { + EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + EQPGraphRow *pLast; /* Last element of the pRow list */ + char zPrefix[100]; /* Graph prefix */ +}; + +/* Parameters affecting columnar mode result display (defaulting together) */ +typedef struct ColModeOpts { + int iWrap; /* In columnar modes, wrap lines reaching this limit */ + u8 bQuote; /* Quote results for .mode box and table */ + u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ +} ColModeOpts; +#define ColModeOpts_default { 60, 0, 0 } +#define ColModeOpts_default_qbox { 60, 1, 0 } + +/* +** State information about the database connection is contained in an +** instance of the following structure. +*/ +typedef struct ShellState ShellState; +struct ShellState { + sqlite3 *db; /* The database */ + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtest; /* autoEQP is in test mode */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ + u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ + u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ + u8 bSafeMode; /* True to prohibit unsafe operations */ + u8 bSafeModePersist; /* The long-term value of bSafeMode */ + u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ + ColModeOpts cmOpts; /* Option values affecting columnar mode output */ + unsigned statsOn; /* True to display memory stats before each finalize */ + unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + int inputNesting; /* Track nesting level of .read and other redirects */ + int outCount; /* Revert to stdout when reaching zero */ + int cnt; /* Number of records displayed so far */ + int lineno; /* Line number of last line read from in */ + int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ + FILE *in; /* Read commands from this stream */ + FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite3_trace() */ + int nErr; /* Number of errors seen */ + int mode; /* An output mode setting */ + int modePrior; /* Saved mode */ + int cMode; /* temporary output mode for the current query */ + int normalMode; /* Output mode before ".explain on" */ + int writableSchema; /* True if PRAGMA writable_schema=ON */ + int showHeader; /* True to show column names in List or Column mode */ + int nCheck; /* Number of ".check" commands run */ + unsigned nProgress; /* Number of progress callbacks encountered */ + unsigned mxProgress; /* Maximum progress callbacks before failing */ + unsigned flgProgress; /* Flags for the progress callback */ + unsigned shellFlgs; /* Various flags */ + unsigned priorShFlgs; /* Saved copy of flags */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ + char *zDestTable; /* Name of destination table when MODE_Insert */ + char *zTempFile; /* Temporary file that might need deleting */ + char zTestcase[30]; /* Name of current test case */ + char colSeparator[20]; /* Column separator character for several modes */ + char rowSeparator[20]; /* Row separator character for MODE_Ascii */ + char colSepPrior[20]; /* Saved column separator */ + char rowSepPrior[20]; /* Saved row separator */ + int *colWidth; /* Requested width of each column in columnar modes */ + int *actualWidth; /* Actual width of each column */ + int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ + char nullValue[20]; /* The text to print when a NULL comes back from + ** the database */ + char outfile[FILENAME_MAX]; /* Filename for *out */ + sqlite3_stmt *pStmt; /* Current statement if any. */ + FILE *pLog; /* Write log output here */ + struct AuxDb { /* Storage space for auxiliary database connections */ + sqlite3 *db; /* Connection pointer */ + const char *zDbFilename; /* Filename used to open the connection */ + char *zFreeOnClose; /* Free this memory allocation on close */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif + } aAuxDb[5], /* Array of all database connections */ + *pAuxDb; /* Currently active database connection */ + int *aiIndent; /* Array of indents used in MODE_Explain */ + int nIndent; /* Size of array aiIndent[] */ + int iIndent; /* Index of current op in aiIndent[] */ + char *zNonce; /* Nonce for temporary safe-mode escapes */ + EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ + ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#ifdef SQLITE_SHELL_FIDDLE + struct { + const char * zInput; /* Input string from wasm/JS proxy */ + const char * zPos; /* Cursor pos into zInput */ + const char * zDefaultDbName; /* Default name for db file */ + } wasm; +#endif +}; + +#ifdef SQLITE_SHELL_FIDDLE +static ShellState shellState; +#endif + + +/* Allowed values for ShellState.autoEQP +*/ +#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ +#define AUTOEQP_on 1 /* Automatic EQP is on */ +#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ +#define AUTOEQP_full 3 /* Show full EXPLAIN */ + +/* Allowed values for ShellState.openMode +*/ +#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ +#define SHELL_OPEN_NORMAL 1 /* Normal database file */ +#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ +#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ +#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ + +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + +/* Bits in the ShellState.flgProgress variable */ +#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress + ** callback limit is reached, and for each + ** top-level SQL statement */ +#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ + +/* +** These are the allowed shellFlgs values +*/ +#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ +#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ +#define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ +#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ +#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ + +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) + +/* +** These are the allowed modes. +*/ +#define MODE_Line 0 /* One column per line. Blank line between records */ +#define MODE_Column 1 /* One record per line in neat columns */ +#define MODE_List 2 /* One record per line with a separator */ +#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ +#define MODE_Html 4 /* Generate an XHTML table */ +#define MODE_Insert 5 /* Generate SQL "insert" statements */ +#define MODE_Quote 6 /* Quote values as for SQL */ +#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ +#define MODE_Csv 8 /* Quote strings, numbers are plain */ +#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ +#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Pretty 11 /* Pretty-print schemas */ +#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ +#define MODE_Json 13 /* Output JSON */ +#define MODE_Markdown 14 /* Markdown formatting */ +#define MODE_Table 15 /* MySQL-style table formatting */ +#define MODE_Box 16 /* Unicode box-drawing characters */ +#define MODE_Count 17 /* Output only a count of the rows of output */ +#define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ +#define MODE_Www 20 /* Full web-page output */ + +static const char *modeDescr[] = { + "line", + "column", + "list", + "semi", + "html", + "insert", + "quote", + "tcl", + "csv", + "explain", + "ascii", + "prettyprint", + "eqp", + "json", + "markdown", + "table", + "box", + "count", + "off", + "scanexp", + "www", +}; + +/* +** These are the column/row/line separators used by the various +** import/export modes. +*/ +#define SEP_Column "|" +#define SEP_Row "\n" +#define SEP_Tab "\t" +#define SEP_Space " " +#define SEP_Comma "," +#define SEP_CrLf "\r\n" +#define SEP_Unit "\x1F" +#define SEP_Record "\x1E" + +/* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + +/* +** A callback for the sqlite3_log() interface. +*/ +static void shellLog(void *pArg, int iErrCode, const char *zMsg){ + ShellState *p = (ShellState*)pArg; + if( p->pLog==0 ) return; + sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + fflush(p->pLog); +} + +/* +** SQL function: shell_putsnl(X) +** +** Write the text X to the screen (or whatever output is being directed) +** adding a newline at the end, and then return X. +*/ +static void shellPutsFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + (void)nVal; + sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + sqlite3_result_value(pCtx, apVal[0]); +} + +/* +** If in safe mode, print an error message described by the arguments +** and exit immediately. +*/ +static void failIfSafeMode( + ShellState *p, + const char *zErrMsg, + ... +){ + if( p->bSafeMode ){ + va_list ap; + char *zMsg; + va_start(ap, zErrMsg); + zMsg = sqlite3_vmprintf(zErrMsg, ap); + va_end(ap); + sqlite3_fprintf(stderr, "line %d: %s\n", p->lineno, zMsg); + exit(1); + } +} + +/* +** SQL function: edit(VALUE) +** edit(VALUE,EDITOR) +** +** These steps: +** +** (1) Write VALUE into a temporary file. +** (2) Run program EDITOR on that temporary file. +** (3) Read the temporary file back and return its content as the result. +** (4) Delete the temporary file +** +** If the EDITOR argument is omitted, use the value in the VISUAL +** environment variable. If still there is no EDITOR, through an error. +** +** Also throw an error if the EDITOR program returns a non-zero exit code. +*/ +#ifndef SQLITE_NOHAVE_SYSTEM +static void editFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zEditor; + char *zTempFile = 0; + sqlite3 *db; + char *zCmd = 0; + int bBin; + int rc; + int hasCRLF = 0; + FILE *f = 0; + sqlite3_int64 sz; + sqlite3_int64 x; + unsigned char *p = 0; + + if( argc==2 ){ + zEditor = (const char*)sqlite3_value_text(argv[1]); + }else{ + zEditor = getenv("VISUAL"); + } + if( zEditor==0 ){ + sqlite3_result_error(context, "no editor for edit()", -1); + return; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + sqlite3_result_error(context, "NULL input to edit()", -1); + return; + } + db = sqlite3_context_db_handle(context); + zTempFile = 0; + sqlite3_file_control(db, 0, SQLITE_FCNTL_TEMPFILENAME, &zTempFile); + if( zTempFile==0 ){ + sqlite3_uint64 r = 0; + sqlite3_randomness(sizeof(r), &r); + zTempFile = sqlite3_mprintf("temp%llx", r); + if( zTempFile==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB; + /* When writing the file to be edited, do \n to \r\n conversions on systems + ** that want \r\n line endings */ + f = sqlite3_fopen(zTempFile, bBin ? "wb" : "w"); + if( f==0 ){ + sqlite3_result_error(context, "edit() cannot open temp file", -1); + goto edit_func_end; + } + sz = sqlite3_value_bytes(argv[0]); + if( bBin ){ + x = fwrite(sqlite3_value_blob(argv[0]), 1, (size_t)sz, f); + }else{ + const char *z = (const char*)sqlite3_value_text(argv[0]); + /* Remember whether or not the value originally contained \r\n */ + if( z && strstr(z,"\r\n")!=0 ) hasCRLF = 1; + x = fwrite(sqlite3_value_text(argv[0]), 1, (size_t)sz, f); + } + fclose(f); + f = 0; + if( x!=sz ){ + sqlite3_result_error(context, "edit() could not write the whole file", -1); + goto edit_func_end; + } + zCmd = sqlite3_mprintf("%s \"%s\"", zEditor, zTempFile); + if( zCmd==0 ){ + sqlite3_result_error_nomem(context); + goto edit_func_end; + } + rc = system(zCmd); + sqlite3_free(zCmd); + if( rc ){ + sqlite3_result_error(context, "EDITOR returned non-zero", -1); + goto edit_func_end; + } + f = sqlite3_fopen(zTempFile, "rb"); + if( f==0 ){ + sqlite3_result_error(context, + "edit() cannot reopen temp file after edit", -1); + goto edit_func_end; + } + fseek(f, 0, SEEK_END); + sz = ftell(f); + rewind(f); + p = sqlite3_malloc64( sz+1 ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + goto edit_func_end; + } + x = fread(p, 1, (size_t)sz, f); + fclose(f); + f = 0; + if( x!=sz ){ + sqlite3_result_error(context, "could not read back the whole file", -1); + goto edit_func_end; + } + if( bBin ){ + sqlite3_result_blob64(context, p, sz, sqlite3_free); + }else{ + sqlite3_int64 i, j; + if( hasCRLF ){ + /* If the original contains \r\n then do no conversions back to \n */ + }else{ + /* If the file did not originally contain \r\n then convert any new + ** \r\n back into \n */ + p[sz] = 0; + for(i=j=0; imodePrior = p->mode; + p->priorShFlgs = p->shellFlgs; + memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); + memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); +} +static void outputModePop(ShellState *p){ + p->mode = p->modePrior; + p->shellFlgs = p->priorShFlgs; + memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); + memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); +} + +/* +** Set output mode to text or binary for Windows. +*/ +static void setCrlfMode(ShellState *p){ +#ifdef _WIN32 + if( p->crlfMode ){ + sqlite3_fsetmode(p->out, _O_TEXT); + }else{ + sqlite3_fsetmode(p->out, _O_BINARY); + } +#else + UNUSED_PARAMETER(p); +#endif +} + +/* +** Output the given string as a hex-encoded blob (eg. X'1234' ) +*/ +static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ + int i; + unsigned char *aBlob = (unsigned char*)pBlob; + + char *zStr = sqlite3_malloc(nBlob*2 + 1); + shell_check_oom(zStr); + + for(i=0; i> 4) ]; + zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; + } + zStr[i*2] = '\0'; + + sqlite3_fprintf(out, "X'%s'", zStr); + sqlite3_free(zStr); +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *unused_string( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** +** See also: output_quoted_escaped_string() +*/ +static void output_quoted_string(ShellState *p, const char *z){ + int i; + char c; + FILE *out = p->out; + sqlite3_fsetmode(out, _O_BINARY); + if( z==0 ) return; + for(i=0; (c = z[i])!=0 && c!='\''; i++){} + if( c==0 ){ + sqlite3_fprintf(out, "'%s'",z); + }else{ + sqlite3_fputs("'", out); + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + sqlite3_fprintf(out, "%.*s", i, z); + z += i; + } + if( c=='\'' ){ + sqlite3_fputs("'", out); + continue; + } + if( c==0 ){ + break; + } + z++; + } + sqlite3_fputs("'", out); + } + setCrlfMode(p); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** Additionallly , escape the "\n" and "\r" characters so that they do not +** get corrupted by end-of-line translation facilities in some operating +** systems. +** +** This is like output_quoted_string() but with the addition of the \r\n +** escape mechanism. +*/ +static void output_quoted_escaped_string(ShellState *p, const char *z){ + int i; + char c; + FILE *out = p->out; + sqlite3_fsetmode(out, _O_BINARY); + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + sqlite3_fprintf(out, "'%s'",z); + }else{ + const char *zNL = 0; + const char *zCR = 0; + int nNL = 0; + int nCR = 0; + char zBuf1[20], zBuf2[20]; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) nNL++; + if( z[i]=='\r' ) nCR++; + } + if( nNL ){ + sqlite3_fputs("replace(", out); + zNL = unused_string(z, "\\n", "\\012", zBuf1); + } + if( nCR ){ + sqlite3_fputs("replace(", out); + zCR = unused_string(z, "\\r", "\\015", zBuf2); + } + sqlite3_fputs("'", out); + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + sqlite3_fprintf(out, "%.*s", i, z); + z += i; + } + if( c=='\'' ){ + sqlite3_fputs("'", out); + continue; + } + if( c==0 ){ + break; + } + z++; + if( c=='\n' ){ + sqlite3_fputs(zNL, out); + continue; + } + sqlite3_fputs(zCR, out); + } + sqlite3_fputs("'", out); + if( nCR ){ + sqlite3_fprintf(out, ",'%s',char(13))", zCR); + } + if( nNL ){ + sqlite3_fprintf(out, ",'%s',char(10))", zNL); + } + } + setCrlfMode(p); +} + +/* +** Find earliest of chars within s specified in zAny. +** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. +*/ +static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ + const char *pcFirst = 0; + if( ns == ~(size_t)0 ) ns = strlen(s); + while(*zAny){ + const char *pc = (const char*)memchr(s, *zAny&0xff, ns); + if( pc ){ + pcFirst = pc; + ns = pcFirst - s; + } + ++zAny; + } + return pcFirst; +} + +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +const char *zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; + } + } + return z; +} + + +/* +** Output the given string as a quoted according to C or TCL quoting rules. +*/ +static void output_c_string(FILE *out, const char *z){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ + char ace[3] = "\\?"; + char cbsSay; + sqlite3_fputs(zq, out); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; + if( pcEnd > z ){ + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + } + if( (c = *pcEnd)==0 ) break; + ++pcEnd; + switch( c ){ + case '\\': case '"': + cbsSay = (char)c; + break; + case '\t': cbsSay = 't'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\f': cbsSay = 'f'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + sqlite3_fputs(ace, out); + }else if( !isprint(c&0xff) ){ + sqlite3_fprintf(out, "\\%03o", c&0xff); + }else{ + ace[1] = (char)c; + sqlite3_fputs(ace+1, out); + } + z = pcEnd; + } + sqlite3_fputs(zq, out); +} + +/* +** Output the given string as a quoted according to JSON quoting rules. +*/ +static void output_json_string(FILE *out, const char *z, i64 n){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBS = "\"\\"; + const char *pcLimit; + char ace[3] = "\\?"; + char cbsSay; + + if( z==0 ) z = ""; + pcLimit = z + ((n<0)? strlen(z) : (size_t)n); + sqlite3_fputs(zq, out); + while( z < pcLimit ){ + const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); + const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); + const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + if( pcEnd > z ){ + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + z = pcEnd; + } + if( z >= pcLimit ) break; + c = *(z++); + switch( c ){ + case '"': case '\\': + cbsSay = (char)c; + break; + case '\b': cbsSay = 'b'; break; + case '\f': cbsSay = 'f'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\t': cbsSay = 't'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + sqlite3_fputs(ace, out); + }else if( c<=0x1f ){ + sqlite3_fprintf(out, "u%04x", c); + }else{ + ace[1] = (char)c; + sqlite3_fputs(ace+1, out); + } + } + sqlite3_fputs(zq, out); +} + +/* +** Output the given string with characters that are special to +** HTML escaped. +*/ +static void output_html_string(FILE *out, const char *z){ + int i; + if( z==0 ) z = ""; + while( *z ){ + for(i=0; z[i] + && z[i]!='<' + && z[i]!='&' + && z[i]!='>' + && z[i]!='\"' + && z[i]!='\''; + i++){} + if( i>0 ){ + sqlite3_fprintf(out, "%.*s",i,z); + } + if( z[i]=='<' ){ + sqlite3_fputs("<", out); + }else if( z[i]=='&' ){ + sqlite3_fputs("&", out); + }else if( z[i]=='>' ){ + sqlite3_fputs(">", out); + }else if( z[i]=='\"' ){ + sqlite3_fputs(""", out); + }else if( z[i]=='\'' ){ + sqlite3_fputs("'", out); + }else{ + break; + } + z += i + 1; + } +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char needCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Output a single term of CSV. Actually, p->colSeparator is used for +** the separator, which may or may not be a comma. p->nullValue is +** the null value. Strings are quoted if necessary. The separator +** is only issued if bSep is true. +*/ +static void output_csv(ShellState *p, const char *z, int bSep){ + if( z==0 ){ + sqlite3_fprintf(p->out, "%s",p->nullValue); + }else{ + unsigned i; + for(i=0; z[i]; i++){ + if( needCsvQuote[((unsigned char*)z)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(z, p->colSeparator)!=0 ){ + char *zQuoted = sqlite3_mprintf("\"%w\"", z); + shell_check_oom(zQuoted); + sqlite3_fputs(zQuoted, p->out); + sqlite3_free(zQuoted); + }else{ + sqlite3_fputs(z, p->out); + } + } + if( bSep ){ + sqlite3_fputs(p->colSeparator, p->out); + } +} + +/* +** This routine runs when the user presses Ctrl-C +*/ +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( ++seenInterrupt>1 ) exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); +} + +#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) +/* +** This routine runs for console events (e.g. Ctrl-C) on Win32 +*/ +static BOOL WINAPI ConsoleCtrlHandler( + DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ +){ + if( dwCtrlType==CTRL_C_EVENT ){ + interrupt_handler(0); + return TRUE; + } + return FALSE; +} +#endif + +#ifndef SQLITE_OMIT_AUTHORIZATION +/* +** This authorizer runs in safe mode. +*/ +static int safeModeAuth( + void *pClientData, + int op, + const char *zA1, + const char *zA2, + const char *zA3, + const char *zA4 +){ + ShellState *p = (ShellState*)pClientData; + static const char *azProhibitedFunctions[] = { + "edit", + "fts3_tokenizer", + "load_extension", + "readfile", + "writefile", + "zipfile", + "zipfile_cds", + }; + UNUSED_PARAMETER(zA1); + UNUSED_PARAMETER(zA3); + UNUSED_PARAMETER(zA4); + switch( op ){ + case SQLITE_ATTACH: { +#ifndef SQLITE_SHELL_FIDDLE + /* In WASM builds the filesystem is a virtual sandbox, so + ** there's no harm in using ATTACH. */ + failIfSafeMode(p, "cannot run ATTACH in safe mode"); +#endif + break; + } + case SQLITE_FUNCTION: { + int i; + for(i=0; iout, "authorizer: %s", azAction[op]); + for(i=0; i<4; i++){ + sqlite3_fputs(" ", p->out); + if( az[i] ){ + output_c_string(p->out, az[i]); + }else{ + sqlite3_fputs("NULL", p->out); + } + } + sqlite3_fputs("\n", p->out); + if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); + return SQLITE_OK; +} +#endif + +/* +** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** +** This routine converts some CREATE TABLE statements for shadow tables +** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +** +** If the schema statement in z[] contains a start-of-comment and if +** sqlite3_complete() returns false, try to terminate the comment before +** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c +*/ +static void printSchemaLine(FILE *out, const char *z, const char *zTail){ + char *zToFree = 0; + if( z==0 ) return; + if( zTail==0 ) return; + if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){ + const char *zOrig = z; + static const char *azTerm[] = { "", "*/", "\n" }; + int i; + for(i=0; iautoEQPtest ){ + sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + } + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + shell_check_oom(pNew); + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->sGraph.pLast ){ + p->sGraph.pLast->pNext = pNew; + }else{ + p->sGraph.pRow = pNew; + } + p->sGraph.pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->sGraph. +*/ +static void eqp_reset(ShellState *p){ + EQPGraphRow *pRow, *pNext; + for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + memset(&p->sGraph, 0, sizeof(p->sGraph)); +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ + EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void eqp_render_level(ShellState *p, int iEqpId){ + EQPGraphRow *pRow, *pNext; + i64 n = strlen(p->sGraph.zPrefix); + char *z; + for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = eqp_next_row(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ + memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); + eqp_render_level(p, pRow->iEqpId); + p->sGraph.zPrefix[n] = 0; + } + } +} + +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void eqp_render(ShellState *p, i64 nCycle){ + EQPGraphRow *pRow = p->sGraph.pRow; + if( pRow ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + eqp_reset(p); + return; + } + sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); + p->sGraph.pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); + }else{ + sqlite3_fputs("QUERY PLAN\n", p->out); + } + p->sGraph.zPrefix[0] = 0; + eqp_render_level(p, 0); + eqp_reset(p); + } +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** Progress handler callback. +*/ +static int progress_handler(void *pClientData) { + ShellState *p = (ShellState*)pClientData; + p->nProgress++; + if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ + sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); + if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; + if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; + return 1; + } + if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ + sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); + } + return 0; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + +/* +** Print N dashes +*/ +static void print_dashes(FILE *out, int N){ + const char zDash[] = "--------------------------------------------------"; + const int nDash = sizeof(zDash) - 1; + while( N>nDash ){ + sqlite3_fputs(zDash, out); + N -= nDash; + } + sqlite3_fprintf(out, "%.*s", N, zDash); +} + +/* +** Print a markdown or table-style row separator using ascii-art +*/ +static void print_row_separator( + ShellState *p, + int nArg, + const char *zSep +){ + int i; + if( nArg>0 ){ + sqlite3_fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[0]+2); + for(i=1; iout); + print_dashes(p->out, p->actualWidth[i]+2); + } + sqlite3_fputs(zSep, p->out); + } + sqlite3_fputs("\n", p->out); +} + +/* +** This is the callback routine that the shell +** invokes for each row of a query result. +*/ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types. Might be NULL */ +){ + int i; + ShellState *p = (ShellState*)pArg; + + if( azArg==0 ) return 0; + switch( p->cMode ){ + case MODE_Count: + case MODE_Off: { + break; + } + case MODE_Line: { + int w = 5; + if( azArg==0 ) break; + for(i=0; iw ) w = len; + } + if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); + for(i=0; iout, "%*s = %s%s", w, azCol[i], + azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + } + break; + } + case MODE_ScanExp: + case MODE_Explain: { + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; + } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ + if( p->cnt++==0 ){ + for(i=0; iout, aWidth[i], azCol[ aMap[i] ]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); + } + for(i=0; iout, aWidth[i]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); + } + } + + /* If there is no data, exit early. */ + if( azArg==0 ) break; + + for(i=0; iw ){ + w = strlenChar(zVal); + zSep = " "; + } + if( i==iIndent && p->aiIndent && p->pStmt ){ + if( p->iIndentnIndent ){ + sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + } + p->iIndent++; + } + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); + } + break; + } + case MODE_Semi: { /* .schema and .fullschema output */ + printSchemaLine(p->out, azArg[0], ";\n"); + break; + } + case MODE_Pretty: { /* .schema and .fullschema with --indent */ + char *z; + int j; + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + assert( nArg==1 ); + if( azArg[0]==0 ) break; + if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 + ){ + sqlite3_fprintf(p->out, "%s;\n", azArg[0]); + break; + } + z = sqlite3_mprintf("%s", azArg[0]); + shell_check_oom(z); + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 ){ + printSchemaLineN(p->out, z, j, "\n"); + j = 0; + } + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + ){ + if( c=='\n' ) j--; + printSchemaLineN(p->out, z, j, "\n "); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } + } + } + z[j] = 0; + } + printSchemaLine(p->out, z, ";\n"); + sqlite3_free(z); + break; + } + case MODE_List: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout, "%s%s", azCol[i], + i==nArg-1 ? p->rowSeparator : p->colSeparator); + } + } + if( azArg==0 ) break; + for(i=0; inullValue; + sqlite3_fputs(z, p->out); + sqlite3_fputs((icolSeparator : p->rowSeparator, p->out); + } + break; + } + case MODE_Www: + case MODE_Html: { + if( p->cnt==0 && p->cMode==MODE_Www ){ + sqlite3_fputs( + "\n" + "\n" + ,p->out + ); + } + if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ + sqlite3_fputs("", p->out); + for(i=0; i", p->out); + output_html_string(p->out, azCol[i]); + sqlite3_fputs("\n", p->out); + } + sqlite3_fputs("\n", p->out); + } + p->cnt++; + if( azArg==0 ) break; + sqlite3_fputs("", p->out); + for(i=0; i", p->out); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + sqlite3_fputs("\n", p->out); + } + sqlite3_fputs("\n", p->out); + break; + } + case MODE_Tcl: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout, azCol[i] ? azCol[i] : ""); + if(icolSeparator, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + if( azArg==0 ) break; + for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); + if(icolSeparator, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_Csv: { + sqlite3_fsetmode(p->out, _O_BINARY); + if( p->cnt++==0 && p->showHeader ){ + for(i=0; irowSeparator, p->out); + } + if( nArg>0 ){ + for(i=0; irowSeparator, p->out); + } + setCrlfMode(p); + break; + } + case MODE_Insert: { + if( azArg==0 ) break; + sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + sqlite3_fputs("(", p->out); + for(i=0; i0 ) sqlite3_fputs(",", p->out); + if( quoteChar(azCol[i]) ){ + char *z = sqlite3_mprintf("\"%w\"", azCol[i]); + shell_check_oom(z); + sqlite3_fputs(z, p->out); + sqlite3_free(z); + }else{ + sqlite3_fprintf(p->out, "%s", azCol[i]); + } + } + sqlite3_fputs(")", p->out); + } + p->cnt++; + for(i=0; i0 ? "," : " VALUES(", p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("NULL", p->out); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p, azArg[i]); + }else{ + output_quoted_escaped_string(p, azArg[i]); + } + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + sqlite3_fputs(azArg[i], p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + sqlite3_fputs("9.0e+999", p->out); + }else if( ur==0xfff0000000000000LL ){ + sqlite3_fputs("-9.0e+999", p->out); + }else{ + sqlite3_int64 ir = (sqlite3_int64)r; + if( r==(double)ir ){ + sqlite3_snprintf(50,z,"%lld.0", ir); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + } + sqlite3_fputs(z, p->out); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + sqlite3_fputs(azArg[i], p->out); + }else if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p, azArg[i]); + }else{ + output_quoted_escaped_string(p, azArg[i]); + } + } + sqlite3_fputs(");\n", p->out); + break; + } + case MODE_Json: { + if( azArg==0 ) break; + if( p->cnt==0 ){ + sqlite3_fputs("[{", p->out); + }else{ + sqlite3_fputs(",\n{", p->out); + } + p->cnt++; + for(i=0; iout, azCol[i], -1); + sqlite3_fputs(":", p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("null", p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + sqlite3_fputs("9.0e+999", p->out); + }else if( ur==0xfff0000000000000LL ){ + sqlite3_fputs("-9.0e+999", p->out); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + sqlite3_fputs(z, p->out); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_json_string(p->out, pBlob, nBlob); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_json_string(p->out, azArg[i], -1); + }else{ + sqlite3_fputs(azArg[i], p->out); + } + if( iout); + } + } + sqlite3_fputs("}", p->out); + break; + } + case MODE_Quote: { + if( azArg==0 ) break; + if( p->cnt==0 && p->showHeader ){ + for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); + output_quoted_string(p, azCol[i]); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + p->cnt++; + for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("NULL", p->out); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_quoted_string(p, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + sqlite3_fputs(azArg[i], p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + sqlite3_fputs(z, p->out); + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + sqlite3_fputs(azArg[i], p->out); + }else{ + output_quoted_string(p, azArg[i]); + } + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_Ascii: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + if( azArg==0 ) break; + for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_EQP: { + eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); + break; + } + } + return 0; +} + +/* +** This is the callback routine that the SQLite library +** invokes for each row of a query result. +*/ +static int callback(void *pArg, int nArg, char **azArg, char **azCol){ + /* since we don't have type info, call the shell_callback with a NULL value */ + return shell_callback(pArg, nArg, azArg, azCol, NULL); +} + +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + UNUSED_PARAMETER(az); + if( azArg==0 ) return 0; + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_schema\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO [_shell$self]\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); +} + + +/* +** Set the destination table field of the ShellState structure to +** the name of the table given. Escape any quote characters in the +** table name. +*/ +static void set_table_name(ShellState *p, const char *zName){ + int i, n; + char cQuote; + char *z; + + if( p->zDestTable ){ + free(p->zDestTable); + p->zDestTable = 0; + } + if( zName==0 ) return; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += n+2; + z = p->zDestTable = malloc( n+1 ); + shell_check_oom(z); + n = 0; + if( cQuote ) z[n++] = cQuote; + for(i=0; zName[i]; i++){ + z[n++] = zName[i]; + if( zName[i]==cQuote ) z[n++] = cQuote; + } + if( cQuote ) z[n++] = cQuote; + z[n] = 0; +} + +/* +** Maybe construct two lines of text that point out the position of a +** syntax error. Return a pointer to the text, in memory obtained from +** sqlite3_malloc(). Or, if the most recent error does not involve a +** specific token that we can point to, return an empty string. +** +** In all cases, the memory returned is obtained from sqlite3_malloc64() +** and should be released by the caller invoking sqlite3_free(). +*/ +static char *shell_error_context(const char *zSql, sqlite3 *db){ + int iOffset; + size_t len; + char *zCode; + char *zMsg; + int i; + if( db==0 + || zSql==0 + || (iOffset = sqlite3_error_offset(db))<0 + || iOffset>=(int)strlen(zSql) + ){ + return sqlite3_mprintf(""); + } + while( iOffset>50 ){ + iOffset--; + zSql++; + while( (zSql[0]&0xc0)==0x80 ){ zSql++; iOffset--; } + } + len = strlen(zSql); + if( len>78 ){ + len = 78; + while( len>0 && (zSql[len]&0xc0)==0x80 ) len--; + } + zCode = sqlite3_mprintf("%.*s", len, zSql); + shell_check_oom(zCode); + for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; } + if( iOffset<25 ){ + zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode,iOffset,""); + }else{ + zMsg = sqlite3_mprintf("\n %z\n %*serror here ---^", zCode,iOffset-14,""); + } + return zMsg; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static int run_table_dump_query( + ShellState *p, /* Query context */ + const char *zSelect /* SELECT statement to extract content */ +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); + if( rc!=SQLITE_OK || !pSelect ){ + char *zContext = shell_error_context(zSelect, p->db); + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + rc, sqlite3_errmsg(p->db), zContext); + sqlite3_free(zContext); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + return rc; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + sqlite3_fprintf(p->out, "%s", z); + for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + sqlite3_fputs("\n;\n", p->out); + }else{ + sqlite3_fputs(";\n", p->out); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + rc, sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } + return rc; +} + +/* +** Allocate space and save off string indicating current error. +*/ +static char *save_err_msg( + sqlite3 *db, /* Database to query */ + const char *zPhase, /* When the error occurs */ + int rc, /* Error code returned from API */ + const char *zSql /* SQL string, or NULL */ +){ + char *zErr; + char *zContext; + sqlite3_str *pStr = sqlite3_str_new(0); + sqlite3_str_appendf(pStr, "%s, %s", zPhase, sqlite3_errmsg(db)); + if( rc>1 ){ + sqlite3_str_appendf(pStr, " (%d)", rc); + } + zContext = shell_error_context(zSql, db); + if( zContext ){ + sqlite3_str_appendall(pStr, zContext); + sqlite3_free(zContext); + } + zErr = sqlite3_str_finish(pStr); + shell_check_oom(zErr); + return zErr; +} + +#ifdef __linux__ +/* +** Attempt to display I/O stats on Linux using /proc/PID/io +*/ +static void displayLinuxIoStats(FILE *out){ + FILE *in; + char z[200]; + sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); + in = sqlite3_fopen(z, "rb"); + if( in==0 ) return; + while( sqlite3_fgets(z, sizeof(z), in)!=0 ){ + static const struct { + const char *zPattern; + const char *zDesc; + } aTrans[] = { + { "rchar: ", "Bytes received by read():" }, + { "wchar: ", "Bytes sent to write():" }, + { "syscr: ", "Read() system calls:" }, + { "syscw: ", "Write() system calls:" }, + { "read_bytes: ", "Bytes read from storage:" }, + { "write_bytes: ", "Bytes written to storage:" }, + { "cancelled_write_bytes: ", "Cancelled write bytes:" }, + }; + int i; + for(i=0; i1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + sqlite3_fprintf(out, "%-36s %s\n", zLabel, zLine); +} + +/* +** Display memory stats. +*/ +static int display_stats( + sqlite3 *db, /* Database to query */ + ShellState *pArg, /* Pointer to ShellState */ + int bReset /* True to reset the stats */ +){ + int iCur; + int iHiwtr; + FILE *out; + if( pArg==0 || pArg->out==0 ) return 0; + out = pArg->out; + + if( pArg->pStmt && pArg->statsOn==2 ){ + int nCol, i, x; + sqlite3_stmt *pStmt = pArg->pStmt; + char z[100]; + nCol = sqlite3_column_count(pStmt); + sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); + for(i=0; istatsOn==3 ){ + if( pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); + sqlite3_fprintf(out, "VM-steps: %d\n", iCur); + } + return 0; + } + + displayStatLine(out, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(out, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); + if( pArg->shellFlgs & SHFLG_Pagecache ){ + displayStatLine(out, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); + } + displayStatLine(out, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); + displayStatLine(out, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(out, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); +#ifdef YYTRACKMAXSTACKDEPTH + displayStatLine(out, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); +#endif + + if( db ){ + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Successful lookaside attempts: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Lookaside failures due to size: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Lookaside failures due to OOM: %d\n", iHiwtr); + } + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Pager Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); + sqlite3_fprintf(out, + "Page cache hits: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); + sqlite3_fprintf(out, + "Page cache misses: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + sqlite3_fprintf(out, + "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); + sqlite3_fprintf(out, + "Page cache spills: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Schema Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); + sqlite3_fprintf(out, + "Statement Heap/Lookaside Usage: %d bytes\n", iCur); + } + + if( pArg->pStmt ){ + int iHit, iMiss; + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); + sqlite3_fprintf(out, + "Fullscan Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); + sqlite3_fprintf(out, + "Sort Operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); + sqlite3_fprintf(out, + "Autoindex Inserts: %d\n", iCur); + iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, + bReset); + iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, + bReset); + if( iHit || iMiss ){ + sqlite3_fprintf(out, + "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); + } + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + sqlite3_fprintf(out, + "Virtual Machine Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); + sqlite3_fprintf(out, + "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + sqlite3_fprintf(out, + "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + sqlite3_fprintf(out, + "Memory used by prepared stmt: %d\n", iCur); + } + +#ifdef __linux__ + displayLinuxIoStats(pArg->out); +#endif + + /* Do not remove this machine readable comment: extra-stats-output-here */ + + return 0; +} + + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *p = pArg->pStmt; + int ii = 0; + i64 nTotal = 0; + int nWidth = 0; + eqp_reset(pArg); + + for(ii=0; 1; ii++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + scanStatsHeight(p, ii)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 4; + + sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(ii=0; 1; ii++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + char *zText = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + zText = sqlite3_mprintf("%s", zo); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + char *z = 0; + if( nCycle>=0 && nTotal>0 ){ + z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, + nCycle, ((nCycle*100)+nTotal/2) / nTotal + ); + } + if( nLoop>=0 ){ + z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); + } + if( nRow>=0 ){ + z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); + } + + if( zName && pArg->scanstatsOn>1 ){ + double rpl = (double)nRow / (double)nLoop; + z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); + } + + zText = sqlite3_mprintf( + "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z + ); + } + + eqp_append(pArg, iId, iPid, zText); + sqlite3_free(zText); + } + + eqp_render(pArg, nTotal); +} +#endif + + +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int str_in_array(const char *zStr, const char **azArray){ + int i; + for(i=0; azArray[i]; i++){ + if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** If compiled statement pSql appears to be an EXPLAIN statement, allocate +** and populate the ShellState.aiIndent[] array with the number of +** spaces each opcode should be indented before it is output. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ + int *abYield = 0; /* True if op is an OP_Yield */ + int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ + int iOp; /* Index of operation in p->aiIndent[] */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ + int i; + int iAddr = sqlite3_column_int(pSql, 0); + const char *zOp = (const char*)sqlite3_column_text(pSql, 1); + int p1 = sqlite3_column_int(pSql, 2); + int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the p->aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + shell_check_oom(p->aiIndent); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + shell_check_oom(abYield); + } + + abYield[iOp] = str_in_array(zOp, azYield); + p->aiIndent[iOp] = 0; + p->nIndent = iOp+1; + if( str_in_array(zOp, azNext) && p2op>0 ){ + for(i=p2op; iaiIndent[i] += 2; + } + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; + } + } + + p->iIndent = 0; + sqlite3_free(abYield); + sqlite3_reset(pSql); +} + +/* +** Free the array allocated by explain_data_prepare(). +*/ +static void explain_data_delete(ShellState *p){ + sqlite3_free(p->aiIndent); + p->aiIndent = 0; + p->nIndent = 0; + p->iIndent = 0; +} + +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + +/* +** Disable and restore .wheretrace and .treetrace/.selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +} +static void restore_debug_trace_modes(void){ + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +} + +/* Create the TEMP table used to store parameter bindings */ +static void bind_table_init(ShellState *p){ + int wrSchema = 0; + int defensiveMode = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &defensiveMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, + "CREATE TABLE IF NOT EXISTS temp.sqlite_parameters(\n" + " key TEXT PRIMARY KEY,\n" + " value\n" + ") WITHOUT ROWID;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, defensiveMode, 0); +} + +/* +** Bind parameters on a prepared statement. +** +** Parameter bindings are taken from a TEMP table of the form: +** +** CREATE TEMP TABLE sqlite_parameters(key TEXT PRIMARY KEY, value) +** WITHOUT ROWID; +** +** No bindings occur if this table does not exist. The name of the table +** begins with "sqlite_" so that it will not collide with ordinary application +** tables. The table must be in the TEMP schema. +*/ +static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ + int nVar; + int i; + int rc; + sqlite3_stmt *pQ = 0; + + nVar = sqlite3_bind_parameter_count(pStmt); + if( nVar==0 ) return; /* Nothing to do */ + if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", + "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ + rc = SQLITE_NOTFOUND; + pQ = 0; + }else{ + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); + } + for(i=1; i<=nVar; i++){ + char zNum[30]; + const char *zVar = sqlite3_bind_parameter_name(pStmt, i); + if( zVar==0 ){ + sqlite3_snprintf(sizeof(zNum),zNum,"?%d",i); + zVar = zNum; + } + sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){ + sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); +#ifdef NAN + }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, NAN); +#endif +#ifdef INFINITY + }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, INFINITY); +#endif + }else if( strncmp(zVar, "$int_", 5)==0 ){ + sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); + }else if( strncmp(zVar, "$text_", 6)==0 ){ + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); + if( zBuf ){ + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); + } + }else{ + sqlite3_bind_null(pStmt, i); + } + sqlite3_reset(pQ); + } + sqlite3_finalize(pQ); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void print_box_line(FILE *out, int N){ + const char zDash[] = + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; + const int nDash = sizeof(zDash) - 1; + N *= 3; + while( N>nDash ){ + sqlite3_fputs(zDash, out); + N -= nDash; + } + sqlite3_fprintf(out, "%.*s", N, zDash); +} + +/* +** Draw a horizontal separator for a MODE_Box table. +*/ +static void print_box_row_separator( + ShellState *p, + int nArg, + const char *zSep1, + const char *zSep2, + const char *zSep3 +){ + int i; + if( nArg>0 ){ + sqlite3_fputs(zSep1, p->out); + print_box_line(p->out, p->actualWidth[0]+2); + for(i=1; iout); + print_box_line(p->out, p->actualWidth[i]+2); + } + sqlite3_fputs(zSep3, p->out); + } + sqlite3_fputs("\n", p->out); +} + +/* +** z[] is a line of text that is to be displayed the .mode box or table or +** similar tabular formats. z[] might contain control characters such +** as \n, \t, \f, or \r. +** +** Compute characters to display on the first line of z[]. Stop at the +** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained +** from malloc()) of that first line, which caller should free sometime. +** Write anything to display on the next line into *pzTail. If this is +** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) +*/ +static char *translateForDisplayAndDup( + const unsigned char *z, /* Input text to be transformed */ + const unsigned char **pzTail, /* OUT: Tail of the input for next line */ + int mxWidth, /* Max width. 0 means no limit */ + u8 bWordWrap /* If true, avoid breaking mid-word */ +){ + int i; /* Input bytes consumed */ + int j; /* Output bytes generated */ + int k; /* Input bytes to be displayed */ + int n; /* Output column number */ + unsigned char *zOut; /* Output text */ + + if( z==0 ){ + *pzTail = 0; + return 0; + } + if( mxWidth<0 ) mxWidth = -mxWidth; + if( mxWidth==0 ) mxWidth = 1000000; + i = j = n = 0; + while( n=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + i += len; + j += len; + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ + n++; + i++; + j++; + continue; + } + if( c=='\t' ){ + do{ + n++; + j++; + }while( (n&7)!=0 && n=mxWidth && bWordWrap ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i; k>i/2; k--){ + if( isspace(z[k-1]) ) break; + } + if( k<=i/2 ){ + for(k=i; k>i/2; k--){ + if( isalnum(z[k-1])!=isalnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k<=i/2 ){ + k = i; + }else{ + i = k; + while( z[i]==' ' ) i++; + } + }else{ + k = i; + } + if( n>=mxWidth && z[i]>=' ' ){ + *pzTail = &z[i]; + }else if( z[i]=='\r' && z[i+1]=='\n' ){ + *pzTail = z[i+2] ? &z[i+2] : 0; + }else if( z[i]==0 || z[i+1]==0 ){ + *pzTail = 0; + }else{ + *pzTail = &z[i+1]; + } + zOut = malloc( j+1 ); + shell_check_oom(zOut); + i = j = n = 0; + while( i=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + do{ zOut[j++] = z[i++]; }while( (--len)>0 ); + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ + n++; + zOut[j++] = z[i++]; + continue; + } + if( z[i]=='\t' ){ + do{ + n++; + zOut[j++] = ' '; + }while( (n&7)!=0 && ncmOpts.bWordWrap; + const char *zEmpty = ""; + const char *zShowNull = p->nullValue; + + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ) return; + nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; + nAlloc = nColumn*4; + if( nAlloc<=0 ) nAlloc = 1; + azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); + shell_check_oom(azData); + azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); + shell_check_oom(azNextLine); + memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); + if( p->cmOpts.bQuote ){ + azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); + shell_check_oom(azQuoted); + memset(azQuoted, 0, nColumn*sizeof(char*) ); + } + abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); + shell_check_oom(abRowDiv); + if( nColumn>p->nWidth ){ + p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); + shell_check_oom(p->colWidth); + for(i=p->nWidth; icolWidth[i] = 0; + p->nWidth = nColumn; + p->actualWidth = &p->colWidth[nColumn]; + } + memset(p->actualWidth, 0, nColumn*sizeof(int)); + for(i=0; icolWidth[i]; + if( w<0 ) w = -w; + p->actualWidth[i] = w; + } + for(i=0; icolWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + uz = (const unsigned char*)sqlite3_column_name(pStmt,i); + if( uz==0 ) uz = (u8*)""; + azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); + } + do{ + int useNextLine = bNextLine; + bNextLine = 0; + if( (nRow+2)*nColumn >= nAlloc ){ + nAlloc *= 2; + azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); + shell_check_oom(azData); + abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); + shell_check_oom(abRowDiv); + } + abRowDiv[nRow] = 1; + nRow++; + for(i=0; icolWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + if( useNextLine ){ + uz = azNextLine[i]; + if( uz==0 ) uz = (u8*)zEmpty; + }else if( p->cmOpts.bQuote ){ + sqlite3_free(azQuoted[i]); + azQuoted[i] = quoted_column(pStmt,i); + uz = (const unsigned char*)azQuoted[i]; + }else{ + uz = (const unsigned char*)sqlite3_column_text(pStmt,i); + if( uz==0 ) uz = (u8*)zShowNull; + } + azData[nRow*nColumn + i] + = translateForDisplayAndDup(uz, &azNextLine[i], wx, bw); + if( azNextLine[i] ){ + bNextLine = 1; + abRowDiv[nRow-1] = 0; + bMultiLineRowExists = 1; + } + } + }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); + nTotal = nColumn*(nRow+1); + for(i=0; ip->actualWidth[j] ) p->actualWidth[j] = n; + } + if( seenInterrupt ) goto columnar_end; + switch( p->cMode ){ + case MODE_Column: { + colSep = " "; + rowSep = "\n"; + if( p->showHeader ){ + for(i=0; iactualWidth[i]; + if( p->colWidth[i]<0 ) w = -w; + utf8_width_print(p->out, w, azData[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); + } + for(i=0; iout, p->actualWidth[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); + } + } + break; + } + case MODE_Table: { + colSep = " | "; + rowSep = " |\n"; + print_row_separator(p, nColumn, "+"); + sqlite3_fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "+"); + break; + } + case MODE_Markdown: { + colSep = " | "; + rowSep = " |\n"; + sqlite3_fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "|"); + break; + } + case MODE_Box: { + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); + sqlite3_fputs(BOX_13 " ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + } + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + break; + } + } + for(i=nColumn, j=0; icMode!=MODE_Column ){ + sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); + } + z = azData[i]; + if( z==0 ) z = p->nullValue; + w = p->actualWidth[j]; + if( p->colWidth[j]<0 ) w = -w; + utf8_width_print(p->out, w, z); + if( j==nColumn-1 ){ + sqlite3_fputs(rowSep, p->out); + if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + }else if( p->cMode==MODE_Column ){ + sqlite3_fputs("\n", p->out); + } + } + j = -1; + if( seenInterrupt ) goto columnar_end; + }else{ + sqlite3_fputs(colSep, p->out); + } + } + if( p->cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); + } +columnar_end: + if( seenInterrupt ){ + sqlite3_fputs("Interrupt\n", p->out); + } + nData = (nRow+1)*nColumn; + for(i=0; icMode==MODE_Column + || pArg->cMode==MODE_Table + || pArg->cMode==MODE_Box + || pArg->cMode==MODE_Markdown + ){ + exec_prepared_stmt_columnar(pArg, pStmt); + return; + } + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + shell_out_of_memory(); + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; icMode==MODE_Insert || pArg->cMode==MODE_Quote) + ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + if( pArg->cMode==MODE_Json ){ + sqlite3_fputs("]\n", pArg->out); + }else if( pArg->cMode==MODE_Www ){ + sqlite3_fputs("
\n
\n", pArg->out);
+      }else if( pArg->cMode==MODE_Count ){
+        char zBuf[200];
+        sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n",
+                         nRow, nRow!=1 ? "s" : "");
+        printf("%s", zBuf);
+      }
+    }
+  }
+}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+/*
+** This function is called to process SQL if the previous shell command
+** was ".expert". It passes the SQL in the second argument directly to
+** the sqlite3expert object.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertHandleSQL(
+  ShellState *pState,
+  const char *zSql,
+  char **pzErr
+){
+  assert( pState->expert.pExpert );
+  assert( pzErr==0 || *pzErr==0 );
+  return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr);
+}
+
+/*
+** This function is called either to silently clean up the object
+** created by the ".expert" command (if bCancel==1), or to generate a
+** report from it and then clean it up (if bCancel==0).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertFinish(
+  ShellState *pState,
+  int bCancel,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  sqlite3expert *p = pState->expert.pExpert;
+  FILE *out = pState->out;
+  assert( p );
+  assert( bCancel || pzErr==0 || *pzErr==0 );
+  if( bCancel==0 ){
+    int bVerbose = pState->expert.bVerbose;
+
+    rc = sqlite3_expert_analyze(p, pzErr);
+    if( rc==SQLITE_OK ){
+      int nQuery = sqlite3_expert_count(p);
+      int i;
+
+      if( bVerbose ){
+        const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+        sqlite3_fputs("-- Candidates -----------------------------\n", out);
+        sqlite3_fprintf(out, "%s\n", zCand);
+      }
+      for(i=0; iexpert.pExpert = 0;
+  return rc;
+}
+
+/*
+** Implementation of ".expert" dot command.
+*/
+static int expertDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  int i;
+  int iSample = 0;
+
+  assert( pState->expert.pExpert==0 );
+  memset(&pState->expert, 0, sizeof(ExpertInfo));
+
+  for(i=1; rc==SQLITE_OK && i=2 && 0==cli_strncmp(z, "-verbose", n) ){
+      pState->expert.bVerbose = 1;
+    }
+    else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){
+      if( i==(nArg-1) ){
+        sqlite3_fprintf(stderr, "option requires an argument: %s\n", z);
+        rc = SQLITE_ERROR;
+      }else{
+        iSample = (int)integerValue(azArg[++i]);
+        if( iSample<0 || iSample>100 ){
+          sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+    }
+    else{
+      sqlite3_fprintf(stderr,"unknown option: %s\n", z);
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
+    if( pState->expert.pExpert==0 ){
+      sqlite3_fprintf(stderr,
+          "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory");
+      rc = SQLITE_ERROR;
+    }else{
+      sqlite3_expert_config(
+          pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample
+      );
+    }
+  }
+  sqlite3_free(zErr);
+
+  return rc;
+}
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+/*
+** Execute a statement or set of statements.  Print
+** any result rows/columns depending on the current mode
+** set via the supplied callback.
+**
+** This is very similar to SQLite's built-in sqlite3_exec()
+** function except it takes a slightly different callback
+** and callback data argument.
+*/
+static int shell_exec(
+  ShellState *pArg,                         /* Pointer to ShellState */
+  const char *zSql,                         /* SQL to be evaluated */
+  char **pzErrMsg                           /* Error msg written here */
+){
+  sqlite3_stmt *pStmt = NULL;     /* Statement to execute. */
+  int rc = SQLITE_OK;             /* Return Code */
+  int rc2;
+  const char *zLeftover;          /* Tail of unprocessed SQL */
+  sqlite3 *db = pArg->db;
+
+  if( pzErrMsg ){
+    *pzErrMsg = NULL;
+  }
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( pArg->expert.pExpert ){
+    rc = expertHandleSQL(pArg, zSql, pzErrMsg);
+    return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg);
+  }
+#endif
+
+  while( zSql[0] && (SQLITE_OK == rc) ){
+    static const char *zStmtSql;
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
+    if( SQLITE_OK != rc ){
+      if( pzErrMsg ){
+        *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql);
+      }
+    }else{
+      if( !pStmt ){
+        /* this happens for a comment or white-space */
+        zSql = zLeftover;
+        while( IsSpace(zSql[0]) ) zSql++;
+        continue;
+      }
+      zStmtSql = sqlite3_sql(pStmt);
+      if( zStmtSql==0 ) zStmtSql = "";
+      while( IsSpace(zStmtSql[0]) ) zStmtSql++;
+
+      /* save off the prepared statement handle and reset row count */
+      if( pArg ){
+        pArg->pStmt = pStmt;
+        pArg->cnt = 0;
+      }
+
+      /* Show the EXPLAIN QUERY PLAN if .eqp is on */
+      if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){
+        sqlite3_stmt *pExplain;
+        int triggerEQP = 0;
+        disable_debug_trace_modes();
+        sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
+        if( pArg->autoEQP>=AUTOEQP_trigger ){
+          sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
+        }
+        pExplain = pStmt;
+        sqlite3_reset(pExplain);
+        rc = sqlite3_stmt_explain(pExplain, 2);
+        if( rc==SQLITE_OK ){
+          bind_prepared_stmt(pArg, pExplain);
+          while( sqlite3_step(pExplain)==SQLITE_ROW ){
+            const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
+            int iEqpId = sqlite3_column_int(pExplain, 0);
+            int iParentId = sqlite3_column_int(pExplain, 1);
+            if( zEQPLine==0 ) zEQPLine = "";
+            if( zEQPLine[0]=='-' ) eqp_render(pArg, 0);
+            eqp_append(pArg, iEqpId, iParentId, zEQPLine);
+          }
+          eqp_render(pArg, 0);
+        }
+        if( pArg->autoEQP>=AUTOEQP_full ){
+          /* Also do an EXPLAIN for ".eqp full" mode */
+          sqlite3_reset(pExplain);
+          rc = sqlite3_stmt_explain(pExplain, 1);
+          if( rc==SQLITE_OK ){
+            pArg->cMode = MODE_Explain;
+            assert( sqlite3_stmt_isexplain(pExplain)==1 );
+            bind_prepared_stmt(pArg, pExplain);
+            explain_data_prepare(pArg, pExplain);
+            exec_prepared_stmt(pArg, pExplain);
+            explain_data_delete(pArg);
+          }
+        }
+        if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
+          sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0);
+        }
+        sqlite3_reset(pStmt);
+        sqlite3_stmt_explain(pStmt, 0);
+        restore_debug_trace_modes();
+      }
+
+      if( pArg ){
+        int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1);
+        pArg->cMode = pArg->mode;
+        if( pArg->autoExplain ){
+          if( bIsExplain ){
+            pArg->cMode = MODE_Explain;
+          }
+          if( sqlite3_stmt_isexplain(pStmt)==2 ){
+            pArg->cMode = MODE_EQP;
+          }
+        }
+
+        /* If the shell is currently in ".explain" mode, gather the extra
+        ** data required to add indents to the output.*/
+        if( pArg->cMode==MODE_Explain && bIsExplain ){
+          explain_data_prepare(pArg, pStmt);
+        }
+      }
+
+      bind_prepared_stmt(pArg, pStmt);
+      exec_prepared_stmt(pArg, pStmt);
+      explain_data_delete(pArg);
+      eqp_render(pArg, 0);
+
+      /* print usage stats if stats on */
+      if( pArg && pArg->statsOn ){
+        display_stats(db, pArg, 0);
+      }
+
+      /* print loop-counters if required */
+      if( pArg && pArg->scanstatsOn ){
+        display_scanstats(db, pArg);
+      }
+
+      /* Finalize the statement just executed. If this fails, save a
+      ** copy of the error message. Otherwise, set zSql to point to the
+      ** next statement to execute. */
+      rc2 = sqlite3_finalize(pStmt);
+      if( rc!=SQLITE_NOMEM ) rc = rc2;
+      if( rc==SQLITE_OK ){
+        zSql = zLeftover;
+        while( IsSpace(zSql[0]) ) zSql++;
+      }else if( pzErrMsg ){
+        *pzErrMsg = save_err_msg(db, "stepping", rc, 0);
+      }
+
+      /* clear saved stmt handle */
+      if( pArg ){
+        pArg->pStmt = NULL;
+      }
+    }
+  } /* end while */
+
+  return rc;
+}
+
+/*
+** Release memory previously allocated by tableColumnList().
+*/
+static void freeColumnList(char **azCol){
+  int i;
+  for(i=1; azCol[i]; i++){
+    sqlite3_free(azCol[i]);
+  }
+  /* azCol[0] is a static string */
+  sqlite3_free(azCol);
+}
+
+/*
+** Return a list of pointers to strings which are the names of all
+** columns in table zTab.   The memory to hold the names is dynamically
+** allocated and must be released by the caller using a subsequent call
+** to freeColumnList().
+**
+** The azCol[0] entry is usually NULL.  However, if zTab contains a rowid
+** value that needs to be preserved, then azCol[0] is filled in with the
+** name of the rowid column.
+**
+** The first regular column in the table is azCol[1].  The list is terminated
+** by an entry with azCol[i]==0.
+*/
+static char **tableColumnList(ShellState *p, const char *zTab){
+  char **azCol = 0;
+  sqlite3_stmt *pStmt;
+  char *zSql;
+  int nCol = 0;
+  int nAlloc = 0;
+  int nPK = 0;       /* Number of PRIMARY KEY columns seen */
+  int isIPK = 0;     /* True if one PRIMARY KEY column of type INTEGER */
+  int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid);
+  int rc;
+
+  zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab);
+  shell_check_oom(zSql);
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc ) return 0;
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    if( nCol>=nAlloc-2 ){
+      nAlloc = nAlloc*2 + nCol + 10;
+      azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
+      shell_check_oom(azCol);
+    }
+    azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+    shell_check_oom(azCol[nCol]);
+    if( sqlite3_column_int(pStmt, 5) ){
+      nPK++;
+      if( nPK==1
+       && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
+                          "INTEGER")==0
+      ){
+        isIPK = 1;
+      }else{
+        isIPK = 0;
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+  if( azCol==0 ) return 0;
+  azCol[0] = 0;
+  azCol[nCol+1] = 0;
+
+  /* The decision of whether or not a rowid really needs to be preserved
+  ** is tricky.  We never need to preserve a rowid for a WITHOUT ROWID table
+  ** or a table with an INTEGER PRIMARY KEY.  We are unable to preserve
+  ** rowids on tables where the rowid is inaccessible because there are other
+  ** columns in the table named "rowid", "_rowid_", and "oid".
+  */
+  if( preserveRowid && isIPK ){
+    /* If a single PRIMARY KEY column with type INTEGER was seen, then it
+    ** might be an alias for the ROWID.  But it might also be a WITHOUT ROWID
+    ** table or a INTEGER PRIMARY KEY DESC column, neither of which are
+    ** ROWID aliases.  To distinguish these cases, check to see if
+    ** there is a "pk" entry in "PRAGMA index_list".  There will be
+    ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID.
+    */
+    zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)"
+                           " WHERE origin='pk'", zTab);
+    shell_check_oom(zSql);
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    if( rc ){
+      freeColumnList(azCol);
+      return 0;
+    }
+    rc = sqlite3_step(pStmt);
+    sqlite3_finalize(pStmt);
+    preserveRowid = rc==SQLITE_ROW;
+  }
+  if( preserveRowid ){
+    /* Only preserve the rowid if we can find a name to use for the
+    ** rowid */
+    static char *azRowid[] = { "rowid", "_rowid_", "oid" };
+    int i, j;
+    for(j=0; j<3; j++){
+      for(i=1; i<=nCol; i++){
+        if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break;
+      }
+      if( i>nCol ){
+        /* At this point, we know that azRowid[j] is not the name of any
+        ** ordinary column in the table.  Verify that azRowid[j] is a valid
+        ** name for the rowid before adding it to azCol[0].  WITHOUT ROWID
+        ** tables will fail this last check */
+        rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
+        if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
+        break;
+      }
+    }
+  }
+  return azCol;
+}
+
+/*
+** Toggle the reverse_unordered_selects setting.
+*/
+static void toggleSelectOrder(sqlite3 *db){
+  sqlite3_stmt *pStmt = 0;
+  int iSetting = 0;
+  char zStmt[100];
+  sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0);
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    iSetting = sqlite3_column_int(pStmt, 0);
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_snprintf(sizeof(zStmt), zStmt,
+       "PRAGMA reverse_unordered_selects(%d)", !iSetting);
+  sqlite3_exec(db, zStmt, 0, 0, 0);
+}
+
+/*
+** This is a different callback routine used for dumping the database.
+** Each row received by this callback consists of a table name,
+** the table type ("index" or "table") and SQL to create the table.
+** This routine should print text sufficient to recreate the table.
+*/
+static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
+  int rc;
+  const char *zTable;
+  const char *zType;
+  const char *zSql;
+  ShellState *p = (ShellState *)pArg;
+  int dataOnly;
+  int noSys;
+
+  UNUSED_PARAMETER(azNotUsed);
+  if( nArg!=3 || azArg==0 ) return 0;
+  zTable = azArg[0];
+  zType = azArg[1];
+  zSql = azArg[2];
+  if( zTable==0 ) return 0;
+  if( zType==0 ) return 0;
+  dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0;
+  noSys    = (p->shellFlgs & SHFLG_DumpNoSys)!=0;
+
+  if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
+    /* no-op */
+  }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
+    if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
+  }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){
+    return 0;
+  }else if( dataOnly ){
+    /* no-op */
+  }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
+    char *zIns;
+    if( !p->writableSchema ){
+      sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
+      p->writableSchema = 1;
+    }
+    zIns = sqlite3_mprintf(
+       "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)"
+       "VALUES('table','%q','%q',0,'%q');",
+       zTable, zTable, zSql);
+    shell_check_oom(zIns);
+    sqlite3_fprintf(p->out, "%s\n", zIns);
+    sqlite3_free(zIns);
+    return 0;
+  }else{
+    printSchemaLine(p->out, zSql, ";\n");
+  }
+
+  if( cli_strcmp(zType, "table")==0 ){
+    ShellText sSelect;
+    ShellText sTable;
+    char **azCol;
+    int i;
+    char *savedDestTable;
+    int savedMode;
+
+    azCol = tableColumnList(p, zTable);
+    if( azCol==0 ){
+      p->nErr++;
+      return 0;
+    }
+
+    /* Always quote the table name, even if it appears to be pure ascii,
+    ** in case it is a keyword. Ex:  INSERT INTO "table" ... */
+    initText(&sTable);
+    appendText(&sTable, zTable, quoteChar(zTable));
+    /* If preserving the rowid, add a column list after the table name.
+    ** In other words:  "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
+    ** instead of the usual "INSERT INTO tab VALUES(...)".
+    */
+    if( azCol[0] ){
+      appendText(&sTable, "(", 0);
+      appendText(&sTable, azCol[0], 0);
+      for(i=1; azCol[i]; i++){
+        appendText(&sTable, ",", 0);
+        appendText(&sTable, azCol[i], quoteChar(azCol[i]));
+      }
+      appendText(&sTable, ")", 0);
+    }
+
+    /* Build an appropriate SELECT statement */
+    initText(&sSelect);
+    appendText(&sSelect, "SELECT ", 0);
+    if( azCol[0] ){
+      appendText(&sSelect, azCol[0], 0);
+      appendText(&sSelect, ",", 0);
+    }
+    for(i=1; azCol[i]; i++){
+      appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
+      if( azCol[i+1] ){
+        appendText(&sSelect, ",", 0);
+      }
+    }
+    freeColumnList(azCol);
+    appendText(&sSelect, " FROM ", 0);
+    appendText(&sSelect, zTable, quoteChar(zTable));
+
+    savedDestTable = p->zDestTable;
+    savedMode = p->mode;
+    p->zDestTable = sTable.z;
+    p->mode = p->cMode = MODE_Insert;
+    rc = shell_exec(p, sSelect.z, 0);
+    if( (rc&0xff)==SQLITE_CORRUPT ){
+      sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
+      toggleSelectOrder(p->db);
+      shell_exec(p, sSelect.z, 0);
+      toggleSelectOrder(p->db);
+    }
+    p->zDestTable = savedDestTable;
+    p->mode = savedMode;
+    freeText(&sTable);
+    freeText(&sSelect);
+    if( rc ) p->nErr++;
+  }
+  return 0;
+}
+
+/*
+** Run zQuery.  Use dump_callback() as the callback routine so that
+** the contents of the query are output as SQL statements.
+**
+** If we get a SQLITE_CORRUPT error, rerun the query after appending
+** "ORDER BY rowid DESC" to the end.
+*/
+static int run_schema_dump_query(
+  ShellState *p,
+  const char *zQuery
+){
+  int rc;
+  char *zErr = 0;
+  rc = sqlite3_exec(p->db, zQuery, dump_callback, p, &zErr);
+  if( rc==SQLITE_CORRUPT ){
+    char *zQ2;
+    int len = strlen30(zQuery);
+    sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
+    if( zErr ){
+      sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr);
+      sqlite3_free(zErr);
+      zErr = 0;
+    }
+    zQ2 = malloc( len+100 );
+    if( zQ2==0 ) return rc;
+    sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery);
+    rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr);
+    if( rc ){
+      sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr);
+    }else{
+      rc = SQLITE_CORRUPT;
+    }
+    sqlite3_free(zErr);
+    free(zQ2);
+  }
+  return rc;
+}
+
+/*
+** Text of help messages.
+**
+** The help text for each individual command begins with a line that starts
+** with ".".  Subsequent lines are supplemental information.
+**
+** There must be two or more spaces between the end of the command and the
+** start of the description of what that command does.
+*/
+static const char *(azHelp[]) = {
+#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \
+  && !defined(SQLITE_SHELL_FIDDLE)
+  ".archive ...             Manage SQL archives",
+  "   Each command must have exactly one of the following options:",
+  "     -c, --create               Create a new archive",
+  "     -u, --update               Add or update files with changed mtime",
+  "     -i, --insert               Like -u but always add even if unchanged",
+  "     -r, --remove               Remove files from archive",
+  "     -t, --list                 List contents of archive",
+  "     -x, --extract              Extract files from archive",
+  "   Optional arguments:",
+  "     -v, --verbose              Print each filename as it is processed",
+  "     -f FILE, --file FILE       Use archive FILE (default is current db)",
+  "     -a FILE, --append FILE     Open FILE using the apndvfs VFS",
+  "     -C DIR, --directory DIR    Read/extract files from directory DIR",
+  "     -g, --glob                 Use glob matching for names in archive",
+  "     -n, --dryrun               Show the SQL that would have occurred",
+  "   Examples:",
+  "     .ar -cf ARCHIVE foo bar  # Create ARCHIVE from files foo and bar",
+  "     .ar -tf ARCHIVE          # List members of ARCHIVE",
+  "     .ar -xvf ARCHIVE         # Verbosely extract files from ARCHIVE",
+  "   See also:",
+  "      http://sqlite.org/cli.html#sqlite_archive_support",
+#endif
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  ".auth ON|OFF             Show authorizer callbacks",
+#endif
+#ifndef SQLITE_SHELL_FIDDLE
+  ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
+  "   Options:",
+  "       --append            Use the appendvfs",
+  "       --async             Write to FILE without journal and fsync()",
+#endif
+  ".bail on|off             Stop after hitting an error.  Default OFF",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".cd DIRECTORY            Change the working directory to DIRECTORY",
+#endif
+  ".changes on|off          Show number of rows changed by SQL",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".check GLOB              Fail if output since .testcase does not match",
+  ".clone NEWDB             Clone data into NEWDB from the existing database",
+#endif
+  ".connection [close] [#]  Open or close an auxiliary database connection",
+  ".crlf ?on|off?           Whether or not to use \\r\\n line endings",
+  ".databases               List names and files of attached databases",
+  ".dbconfig ?op? ?val?     List or change sqlite3_db_config() options",
+#if SQLITE_SHELL_HAVE_RECOVER
+  ".dbinfo ?DB?             Show status information about the database",
+#endif
+  ".dump ?OBJECTS?          Render database content as SQL",
+  "   Options:",
+  "     --data-only            Output only INSERT statements",
+  "     --newlines             Allow unescaped newline characters in output",
+  "     --nosys                Omit system tables (ex: \"sqlite_stat1\")",
+  "     --preserve-rowids      Include ROWID values in the output",
+  "   OBJECTS is a LIKE pattern for tables, indexes, triggers or views to dump",
+  "   Additional LIKE patterns can be given in subsequent arguments",
+  ".echo on|off             Turn command echo on or off",
+  ".eqp on|off|full|...     Enable or disable automatic EXPLAIN QUERY PLAN",
+  "   Other Modes:",
+#ifdef SQLITE_DEBUG
+  "      test                  Show raw EXPLAIN QUERY PLAN output",
+  "      trace                 Like \"full\" but enable \"PRAGMA vdbe_trace\"",
+#endif
+  "      trigger               Like \"full\" but also show trigger bytecode",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".excel                   Display the output of next command in spreadsheet",
+  "   --bom                   Put a UTF8 byte-order mark on intermediate file",
+#endif
+#ifndef SQLITE_SHELL_FIDDLE
+  ".exit ?CODE?             Exit this program with return-code CODE",
+#endif
+  ".expert                  EXPERIMENTAL. Suggest indexes for queries",
+  ".explain ?on|off|auto?   Change the EXPLAIN formatting mode.  Default: auto",
+  ".filectrl CMD ...        Run various sqlite3_file_control() operations",
+  "   --schema SCHEMA         Use SCHEMA instead of \"main\"",
+  "   --help                  Show CMD details",
+  ".fullschema ?--indent?   Show schema and the content of sqlite_stat tables",
+  ".headers on|off          Turn display of headers on or off",
+  ".help ?-all? ?PATTERN?   Show help text for PATTERN",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".import FILE TABLE       Import data from FILE into TABLE",
+  "   Options:",
+  "     --ascii               Use \\037 and \\036 as column and row separators",
+  "     --csv                 Use , and \\n as column and row separators",
+  "     --skip N              Skip the first N rows of input",
+  "     --schema S            Target table to be S.TABLE",
+  "     -v                    \"Verbose\" - increase auxiliary output",
+  "   Notes:",
+  "     *  If TABLE does not exist, it is created.  The first row of input",
+  "        determines the column names.",
+  "     *  If neither --csv or --ascii are used, the input mode is derived",
+  "        from the \".mode\" output mode",
+  "     *  If FILE begins with \"|\" then it is a command that generates the",
+  "        input text.",
+#endif
+#ifndef SQLITE_OMIT_TEST_CONTROL
+  ",imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
+#endif
+  ".indexes ?TABLE?         Show names of indexes",
+  "                           If TABLE is specified, only show indexes for",
+  "                           tables matching TABLE using the LIKE operator.",
+  ".intck ?STEPS_PER_UNLOCK?  Run an incremental integrity check on the db",
+#ifdef SQLITE_ENABLE_IOTRACE
+  ",iotrace FILE            Enable I/O diagnostic logging to FILE",
+#endif
+  ".limit ?LIMIT? ?VAL?     Display or change the value of an SQLITE_LIMIT",
+  ".lint OPTIONS            Report potential schema issues.",
+  "     Options:",
+  "        fkey-indexes     Find missing foreign key indexes",
+#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE)
+  ".load FILE ?ENTRY?       Load an extension library",
+#endif
+#if !defined(SQLITE_SHELL_FIDDLE)
+  ".log FILE|on|off         Turn logging on or off.  FILE can be stderr/stdout",
+#else
+  ".log on|off              Turn logging on or off.",
+#endif
+  ".mode MODE ?OPTIONS?     Set output mode",
+  "   MODE is one of:",
+  "     ascii       Columns/rows delimited by 0x1F and 0x1E",
+  "     box         Tables using unicode box-drawing characters",
+  "     csv         Comma-separated values",
+  "     column      Output in columns.  (See .width)",
+  "     html        HTML  code",
+  "     insert      SQL insert statements for TABLE",
+  "     json        Results in a JSON array",
+  "     line        One value per line",
+  "     list        Values delimited by \"|\"",
+  "     markdown    Markdown table format",
+  "     qbox        Shorthand for \"box --wrap 60 --quote\"",
+  "     quote       Escape answers as for SQL",
+  "     table       ASCII-art table",
+  "     tabs        Tab-separated values",
+  "     tcl         TCL list elements",
+  "   OPTIONS: (for columnar modes or insert mode):",
+  "     --wrap N       Wrap output lines to no longer than N characters",
+  "     --wordwrap B   Wrap or not at word boundaries per B (on/off)",
+  "     --ww           Shorthand for \"--wordwrap 1\"",
+  "     --quote        Quote output text as SQL literals",
+  "     --noquote      Do not quote output text",
+  "     TABLE          The name of SQL table used for \"insert\" mode",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".nonce STRING            Suspend safe mode for one command if nonce matches",
+#endif
+  ".nullvalue STRING        Use STRING in place of NULL values",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".once ?OPTIONS? ?FILE?   Output for the next SQL command only to FILE",
+  "     If FILE begins with '|' then open as a pipe",
+  "       --bom    Put a UTF8 byte-order mark at the beginning",
+  "       -e       Send output to the system text editor",
+  "       --plain  Use text/plain output instead of HTML for -w option",
+  "       -w       Send output as HTML to a web browser (same as \".www\")",
+  "       -x       Send output as CSV to a spreadsheet (same as \".excel\")",
+  /* Note that .open is (partially) available in WASM builds but is
+  ** currently only intended to be used by the fiddle tool, not
+  ** end users, so is "undocumented." */
+  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
+  "     Options:",
+  "        --append        Use appendvfs to append database to the end of FILE",
+#endif
+#ifndef SQLITE_OMIT_DESERIALIZE
+  "        --deserialize   Load into memory using sqlite3_deserialize()",
+  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory db",
+  "        --maxsize N     Maximum size for --hexdb or --deserialized database",
+#endif
+  "        --new           Initialize FILE to an empty database",
+  "        --nofollow      Do not follow symbolic links",
+  "        --readonly      Open FILE readonly",
+  "        --zip           FILE is a ZIP archive",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
+  "   If FILE begins with '|' then open it as a pipe.",
+  "   Options:",
+  "     --bom                 Prefix output with a UTF8 byte-order mark",
+  "     -e                    Send output to the system text editor",
+  "     --plain               Use text/plain for -w option",
+  "     -w                    Send output to a web browser",
+  "     -x                    Send output as CSV to a spreadsheet",
+#endif
+  ".parameter CMD ...       Manage SQL parameter bindings",
+  "   clear                   Erase all bindings",
+  "   init                    Initialize the TEMP table that holds bindings",
+  "   list                    List the current parameter bindings",
+  "   set PARAMETER VALUE     Given SQL parameter PARAMETER a value of VALUE",
+  "                           PARAMETER should start with one of: $ : @ ?",
+  "   unset PARAMETER         Remove PARAMETER from the binding table",
+  ".print STRING...         Print literal STRING",
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  ".progress N              Invoke progress handler after every N opcodes",
+  "   --limit N                 Interrupt after N progress callbacks",
+  "   --once                    Do no more than one progress interrupt",
+  "   --quiet|-q                No output except at interrupts",
+  "   --reset                   Reset the count for each input and interrupt",
+#endif
+  ".prompt MAIN CONTINUE    Replace the standard prompts",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".quit                    Stop interpreting input stream, exit if primary.",
+  ".read FILE               Read input from FILE or command output",
+  "    If FILE begins with \"|\", it is a command that generates the input.",
+#endif
+#if SQLITE_SHELL_HAVE_RECOVER
+  ".recover                 Recover as much data as possible from corrupt db.",
+  "   --ignore-freelist        Ignore pages that appear to be on db freelist",
+  "   --lost-and-found TABLE   Alternative name for the lost-and-found table",
+  "   --no-rowids              Do not attempt to recover rowid values",
+  "                            that are not also INTEGER PRIMARY KEYs",
+#endif
+#ifndef SQLITE_SHELL_FIDDLE
+  ".restore ?DB? FILE       Restore content of DB (default \"main\") from FILE",
+  ".save ?OPTIONS? FILE     Write database to FILE (an alias for .backup ...)",
+#endif
+  ".scanstats on|off|est    Turn sqlite3_stmt_scanstatus() metrics on or off",
+  ".schema ?PATTERN?        Show the CREATE statements matching PATTERN",
+  "   Options:",
+  "      --indent             Try to pretty-print the schema",
+  "      --nosys              Omit objects whose names start with \"sqlite_\"",
+  ",selftest ?OPTIONS?      Run tests defined in the SELFTEST table",
+  "    Options:",
+  "       --init               Create a new SELFTEST table",
+  "       -v                   Verbose output",
+  ".separator COL ?ROW?     Change the column and row separators",
+#if defined(SQLITE_ENABLE_SESSION)
+  ".session ?NAME? CMD ...  Create or control sessions",
+  "   Subcommands:",
+  "     attach TABLE             Attach TABLE",
+  "     changeset FILE           Write a changeset into FILE",
+  "     close                    Close one session",
+  "     enable ?BOOLEAN?         Set or query the enable bit",
+  "     filter GLOB...           Reject tables matching GLOBs",
+  "     indirect ?BOOLEAN?       Mark or query the indirect status",
+  "     isempty                  Query whether the session is empty",
+  "     list                     List currently open session names",
+  "     open DB NAME             Open a new session on DB",
+  "     patchset FILE            Write a patchset into FILE",
+  "   If ?NAME? is omitted, the first defined session is used.",
+#endif
+  ".sha3sum ...             Compute a SHA3 hash of database content",
+  "    Options:",
+  "      --schema              Also hash the sqlite_schema table",
+  "      --sha3-224            Use the sha3-224 algorithm",
+  "      --sha3-256            Use the sha3-256 algorithm (default)",
+  "      --sha3-384            Use the sha3-384 algorithm",
+  "      --sha3-512            Use the sha3-512 algorithm",
+  "    Any other argument is a LIKE pattern for tables to hash",
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
+  ".shell CMD ARGS...       Run CMD ARGS... in a system shell",
+#endif
+  ".show                    Show the current values for various settings",
+  ".stats ?ARG?             Show stats or turn stats on or off",
+  "   off                      Turn off automatic stat display",
+  "   on                       Turn on automatic stat display",
+  "   stmt                     Show statement stats",
+  "   vmstep                   Show the virtual machine step count only",
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
+  ".system CMD ARGS...      Run CMD ARGS... in a system shell",
+#endif
+  ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
+#ifndef SQLITE_SHELL_FIDDLE
+  ",testcase NAME           Begin redirecting output to 'testcase-out.txt'",
+#endif
+  ",testctrl CMD ...        Run various sqlite3_test_control() operations",
+  "                           Run \".testctrl\" with no arguments for details",
+  ".timeout MS              Try opening locked tables for MS milliseconds",
+  ".timer on|off            Turn SQL timer on or off",
+#ifndef SQLITE_OMIT_TRACE
+  ".trace ?OPTIONS?         Output each SQL statement as it is run",
+  "    FILE                    Send output to FILE",
+  "    stdout                  Send output to stdout",
+  "    stderr                  Send output to stderr",
+  "    off                     Disable tracing",
+  "    --expanded              Expand query parameters",
+#ifdef SQLITE_ENABLE_NORMALIZE
+  "    --normalized            Normal the SQL statements",
+#endif
+  "    --plain                 Show SQL as it is input",
+  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
+  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
+  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
+  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
+#endif /* SQLITE_OMIT_TRACE */
+#ifdef SQLITE_DEBUG
+  ".unmodule NAME ...       Unregister virtual table modules",
+  "    --allexcept             Unregister everything except those named",
+#endif
+  ".version                 Show source, library and compiler versions",
+  ".vfsinfo ?AUX?           Information about the top-level VFS",
+  ".vfslist                 List all available VFSes",
+  ".vfsname ?AUX?           Print the name of the VFS stack",
+  ".width NUM1 NUM2 ...     Set minimum column widths for columnar output",
+  "     Negative values right-justify",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".www                     Display output of the next command in web browser",
+  "    --plain                 Show results as text/plain, not as HTML",
+#endif
+};
+
+/*
+** Output help text.
+**
+** zPattern describes the set of commands for which help text is provided.
+** If zPattern is NULL, then show all commands, but only give a one-line
+** description of each.
+**
+** Return the number of matches.
+*/
+static int showHelp(FILE *out, const char *zPattern){
+  int i = 0;
+  int j = 0;
+  int n = 0;
+  char *zPat;
+  if( zPattern==0
+   || zPattern[0]=='0'
+   || cli_strcmp(zPattern,"-a")==0
+   || cli_strcmp(zPattern,"-all")==0
+   || cli_strcmp(zPattern,"--all")==0
+  ){
+    enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 };
+    enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 };
+    /* Show all or most commands
+    ** *zPattern==0   => summary of documented commands only
+    ** *zPattern=='0' => whole help for undocumented commands
+    ** Otherwise      => whole help for documented commands
+    */
+    enum HelpWanted hw = HW_SummaryOnly;
+    enum HelpHave hh = HH_More;
+    if( zPattern!=0 ){
+      hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull;
+    }
+    for(i=0; ip);
+  sqlite3_free(pSession->zName);
+  for(i=0; inFilter; i++){
+    sqlite3_free(pSession->azFilter[i]);
+  }
+  sqlite3_free(pSession->azFilter);
+  memset(pSession, 0, sizeof(OpenSession));
+}
+#endif
+
+/*
+** Close all OpenSession objects and release all associated resources.
+*/
+#if defined(SQLITE_ENABLE_SESSION)
+static void session_close_all(ShellState *p, int i){
+  int j;
+  struct AuxDb *pAuxDb = i<0 ? p->pAuxDb : &p->aAuxDb[i];
+  for(j=0; jnSession; j++){
+    session_close(&pAuxDb->aSession[j]);
+  }
+  pAuxDb->nSession = 0;
+}
+#else
+# define session_close_all(X,Y)
+#endif
+
+/*
+** Implementation of the xFilter function for an open session.  Omit
+** any tables named by ".session filter" but let all other table through.
+*/
+#if defined(SQLITE_ENABLE_SESSION)
+static int session_filter(void *pCtx, const char *zTab){
+  OpenSession *pSession = (OpenSession*)pCtx;
+  int i;
+  for(i=0; inFilter; i++){
+    if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0;
+  }
+  return 1;
+}
+#endif
+
+/*
+** Try to deduce the type of file for zName based on its content.  Return
+** one of the SHELL_OPEN_* constants.
+**
+** If the file does not exist or is empty but its name looks like a ZIP
+** archive and the dfltZip flag is true, then assume it is a ZIP archive.
+** Otherwise, assume an ordinary database regardless of the filename if
+** the type cannot be determined from content.
+*/
+int deduceDatabaseType(const char *zName, int dfltZip){
+  FILE *f = sqlite3_fopen(zName, "rb");
+  size_t n;
+  int rc = SHELL_OPEN_UNSPEC;
+  char zBuf[100];
+  if( f==0 ){
+    if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){
+       return SHELL_OPEN_ZIPFILE;
+    }else{
+       return SHELL_OPEN_NORMAL;
+    }
+  }
+  n = fread(zBuf, 16, 1, f);
+  if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){
+    fclose(f);
+    return SHELL_OPEN_NORMAL;
+  }
+  fseek(f, -25, SEEK_END);
+  n = fread(zBuf, 25, 1, f);
+  if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){
+    rc = SHELL_OPEN_APPENDVFS;
+  }else{
+    fseek(f, -22, SEEK_END);
+    n = fread(zBuf, 22, 1, f);
+    if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05
+       && zBuf[3]==0x06 ){
+      rc = SHELL_OPEN_ZIPFILE;
+    }else if( n==0 && dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){
+      rc = SHELL_OPEN_ZIPFILE;
+    }
+  }
+  fclose(f);
+  return rc;
+}
+
+#ifndef SQLITE_OMIT_DESERIALIZE
+/*
+** Reconstruct an in-memory database using the output from the "dbtotxt"
+** program.  Read content from the file in p->aAuxDb[].zDbFilename.
+** If p->aAuxDb[].zDbFilename is 0, then read from standard input.
+*/
+static unsigned char *readHexDb(ShellState *p, int *pnData){
+  unsigned char *a = 0;
+  int nLine;
+  int n = 0;
+  int pgsz = 0;
+  int iOffset = 0;
+  int j, k;
+  int rc;
+  FILE *in;
+  const char *zDbFilename = p->pAuxDb->zDbFilename;
+  unsigned int x[16];
+  char zLine[1000];
+  if( zDbFilename ){
+    in = sqlite3_fopen(zDbFilename, "r");
+    if( in==0 ){
+      sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename);
+      return 0;
+    }
+    nLine = 0;
+  }else{
+    in = p->in;
+    nLine = p->lineno;
+    if( in==0 ) in = stdin;
+  }
+  *pnData = 0;
+  nLine++;
+  if( sqlite3_fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
+  rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
+  if( rc!=2 ) goto readHexDb_error;
+  if( n<0 ) goto readHexDb_error;
+  if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error;
+  n = (n+pgsz-1)&~(pgsz-1);  /* Round n up to the next multiple of pgsz */
+  a = sqlite3_malloc( n ? n : 1 );
+  shell_check_oom(a);
+  memset(a, 0, n);
+  if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
+    sqlite3_fputs("invalid pagesize\n", stderr);
+    goto readHexDb_error;
+  }
+  for(nLine++; sqlite3_fgets(zLine, sizeof(zLine), in)!=0; nLine++){
+    rc = sscanf(zLine, "| page %d offset %d", &j, &k);
+    if( rc==2 ){
+      iOffset = k;
+      continue;
+    }
+    if( cli_strncmp(zLine, "| end ", 6)==0 ){
+      break;
+    }
+    rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+                &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
+                &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
+    if( rc==17 ){
+      k = iOffset+j;
+      if( k+16<=n && k>=0 ){
+        int ii;
+        for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff;
+      }
+    }
+  }
+  *pnData = n;
+  if( in!=p->in ){
+    fclose(in);
+  }else{
+    p->lineno = nLine;
+  }
+  return a;
+
+readHexDb_error:
+  if( in!=p->in ){
+    fclose(in);
+  }else{
+    while( sqlite3_fgets(zLine, sizeof(zLine), p->in)!=0 ){
+      nLine++;
+      if(cli_strncmp(zLine, "| end ", 6)==0 ) break;
+    }
+    p->lineno = nLine;
+  }
+  sqlite3_free(a);
+  sqlite3_fprintf(stderr,"Error on line %d of --hexdb input\n", nLine);
+  return 0;
+}
+#endif /* SQLITE_OMIT_DESERIALIZE */
+
+/*
+** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X.
+*/
+static void shellUSleepFunc(
+  sqlite3_context *context,
+  int argcUnused,
+  sqlite3_value **argv
+){
+  int sleep = sqlite3_value_int(argv[0]);
+  (void)argcUnused;
+  sqlite3_sleep(sleep/1000);
+  sqlite3_result_int(context, sleep);
+}
+
+/* Flags for open_db().
+**
+** The default behavior of open_db() is to exit(1) if the database fails to
+** open.  The OPEN_DB_KEEPALIVE flag changes that so that it prints an error
+** but still returns without calling exit.
+**
+** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a
+** ZIP archive if the file does not exist or is empty and its name matches
+** the *.zip pattern.
+*/
+#define OPEN_DB_KEEPALIVE   0x001   /* Return after error if true */
+#define OPEN_DB_ZIPFILE     0x002   /* Open as ZIP if name matches *.zip */
+
+/*
+** Make sure the database is open.  If it is not, then open it.  If
+** the database fails to open, print an error message and exit.
+*/
+static void open_db(ShellState *p, int openFlags){
+  if( p->db==0 ){
+    const char *zDbFilename = p->pAuxDb->zDbFilename;
+    if( p->openMode==SHELL_OPEN_UNSPEC ){
+      if( zDbFilename==0 || zDbFilename[0]==0 ){
+        p->openMode = SHELL_OPEN_NORMAL;
+      }else{
+        p->openMode = (u8)deduceDatabaseType(zDbFilename,
+                             (openFlags & OPEN_DB_ZIPFILE)!=0);
+      }
+    }
+    switch( p->openMode ){
+      case SHELL_OPEN_APPENDVFS: {
+        sqlite3_open_v2(zDbFilename, &p->db,
+           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs");
+        break;
+      }
+      case SHELL_OPEN_HEXDB:
+      case SHELL_OPEN_DESERIALIZE: {
+        sqlite3_open(0, &p->db);
+        break;
+      }
+      case SHELL_OPEN_ZIPFILE: {
+        sqlite3_open(":memory:", &p->db);
+        break;
+      }
+      case SHELL_OPEN_READONLY: {
+        sqlite3_open_v2(zDbFilename, &p->db,
+            SQLITE_OPEN_READONLY|p->openFlags, 0);
+        break;
+      }
+      case SHELL_OPEN_UNSPEC:
+      case SHELL_OPEN_NORMAL: {
+        sqlite3_open_v2(zDbFilename, &p->db,
+           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0);
+        break;
+      }
+    }
+    if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
+      sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n",
+            zDbFilename, sqlite3_errmsg(p->db));
+      if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){
+        exit(1);
+      }
+      sqlite3_close(p->db);
+      sqlite3_open(":memory:", &p->db);
+      if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
+        sqlite3_fputs("Also: unable to open substitute in-memory database.\n",
+                      stderr);
+        exit(1);
+      }else{
+        sqlite3_fprintf(stderr,
+              "Notice: using substitute in-memory database instead of \"%s\"\n",
+              zDbFilename);
+      }
+    }
+    globalDb = p->db;
+    sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0);
+
+    /* Reflect the use or absence of --unsafe-testing invocation. */
+    {
+      int testmode_on = ShellHasFlag(p,SHFLG_TestingMode);
+      sqlite3_db_config(p->db, SQLITE_DBCONFIG_TRUSTED_SCHEMA, testmode_on,0);
+      sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, !testmode_on,0);
+    }
+
+#ifndef SQLITE_OMIT_LOAD_EXTENSION
+    sqlite3_enable_load_extension(p->db, 1);
+#endif
+    sqlite3_sha_init(p->db, 0, 0);
+    sqlite3_shathree_init(p->db, 0, 0);
+    sqlite3_uint_init(p->db, 0, 0);
+    sqlite3_stmtrand_init(p->db, 0, 0);
+    sqlite3_decimal_init(p->db, 0, 0);
+    sqlite3_percentile_init(p->db, 0, 0);
+    sqlite3_base64_init(p->db, 0, 0);
+    sqlite3_base85_init(p->db, 0, 0);
+    sqlite3_regexp_init(p->db, 0, 0);
+    sqlite3_ieee_init(p->db, 0, 0);
+    sqlite3_series_init(p->db, 0, 0);
+#ifndef SQLITE_SHELL_FIDDLE
+    sqlite3_fileio_init(p->db, 0, 0);
+    sqlite3_completion_init(p->db, 0, 0);
+#endif
+#ifdef SQLITE_HAVE_ZLIB
+    if( !p->bSafeModePersist ){
+      sqlite3_zipfile_init(p->db, 0, 0);
+      sqlite3_sqlar_init(p->db, 0, 0);
+    }
+#endif
+#ifdef SQLITE_SHELL_EXTFUNCS
+    /* Create a preprocessing mechanism for extensions to make
+     * their own provisions for being built into the shell.
+     * This is a short-span macro. See further below for usage.
+     */
+#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant
+#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant)
+    /* Let custom-included extensions get their ..._init() called.
+     * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause
+     * the extension's sqlite3_*_init( db, pzErrorMsg, pApi )
+     * initialization routine to be called.
+     */
+    {
+      int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db);
+    /* Let custom-included extensions expose their functionality.
+     * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause
+     * the SQL functions, virtual tables, collating sequences or
+     * VFS's implemented by the extension to be registered.
+     */
+      if( irc==SQLITE_OK
+          || irc==SQLITE_OK_LOAD_PERMANENTLY ){
+        SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0);
+      }
+#undef SHELL_SUB_MACRO
+#undef SHELL_SUBMACRO
+    }
+#endif
+
+    sqlite3_create_function(p->db, "strtod", 1, SQLITE_UTF8, 0,
+                            shellStrtod, 0, 0);
+    sqlite3_create_function(p->db, "dtostr", 1, SQLITE_UTF8, 0,
+                            shellDtostr, 0, 0);
+    sqlite3_create_function(p->db, "dtostr", 2, SQLITE_UTF8, 0,
+                            shellDtostr, 0, 0);
+    sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
+                            shellAddSchemaName, 0, 0);
+    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
+                            shellModuleSchema, 0, 0);
+    sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
+                            shellPutsFunc, 0, 0);
+    sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0,
+                            shellUSleepFunc, 0, 0);
+#ifndef SQLITE_NOHAVE_SYSTEM
+    sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
+                            editFunc, 0, 0);
+    sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0,
+                            editFunc, 0, 0);
+#endif
+
+    if( p->openMode==SHELL_OPEN_ZIPFILE ){
+      char *zSql = sqlite3_mprintf(
+         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename);
+      shell_check_oom(zSql);
+      sqlite3_exec(p->db, zSql, 0, 0, 0);
+      sqlite3_free(zSql);
+    }
+#ifndef SQLITE_OMIT_DESERIALIZE
+    else
+    if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){
+      int rc;
+      int nData = 0;
+      unsigned char *aData;
+      if( p->openMode==SHELL_OPEN_DESERIALIZE ){
+        aData = (unsigned char*)readFile(zDbFilename, &nData);
+      }else{
+        aData = readHexDb(p, &nData);
+      }
+      if( aData==0 ){
+        return;
+      }
+      rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
+                   SQLITE_DESERIALIZE_RESIZEABLE |
+                   SQLITE_DESERIALIZE_FREEONCLOSE);
+      if( rc ){
+        sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc);
+      }
+      if( p->szMax>0 ){
+        sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
+      }
+    }
+#endif
+  }
+  if( p->db!=0 ){
+    if( p->bSafeModePersist ){
+      sqlite3_set_authorizer(p->db, safeModeAuth, p);
+    }
+    sqlite3_db_config(
+        p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0
+    );
+  }
+}
+
+/*
+** Attempt to close the database connection.  Report errors.
+*/
+void close_db(sqlite3 *db){
+  int rc = sqlite3_close(db);
+  if( rc ){
+    sqlite3_fprintf(stderr,
+        "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db));
+  }
+}
+
+#if HAVE_READLINE || HAVE_EDITLINE
+/*
+** Readline completion callbacks
+*/
+static char *readline_completion_generator(const char *text, int state){
+  static sqlite3_stmt *pStmt = 0;
+  char *zRet;
+  if( state==0 ){
+    char *zSql;
+    sqlite3_finalize(pStmt);
+    zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase"
+                           "  FROM completion(%Q) ORDER BY 1", text);
+    shell_check_oom(zSql);
+    sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+  }
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *z = (const char*)sqlite3_column_text(pStmt,0);
+    zRet = z ? strdup(z) : 0;
+  }else{
+    sqlite3_finalize(pStmt);
+    pStmt = 0;
+    zRet = 0;
+  }
+  return zRet;
+}
+static char **readline_completion(const char *zText, int iStart, int iEnd){
+  (void)iStart;
+  (void)iEnd;
+  rl_attempted_completion_over = 1;
+  return rl_completion_matches(zText, readline_completion_generator);
+}
+
+#elif HAVE_LINENOISE
+/*
+** Linenoise completion callback. Note that the 3rd argument is from
+** the "msteveb" version of linenoise, not the "antirez" version.
+*/
+static void linenoise_completion(const char *zLine, linenoiseCompletions *lc,
+                                 void *pUserData){
+  i64 nLine = strlen(zLine);
+  i64 i, iStart;
+  sqlite3_stmt *pStmt = 0;
+  char *zSql;
+  char zBuf[1000];
+
+  UNUSED_PARAMETER(pUserData);
+  if( nLine>(i64)sizeof(zBuf)-30 ) return;
+  if( zLine[0]=='.' || zLine[0]=='#') return;
+  for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){}
+  if( i==nLine-1 ) return;
+  iStart = i+1;
+  memcpy(zBuf, zLine, iStart);
+  zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase"
+                         "  FROM completion(%Q,%Q) ORDER BY 1",
+                         &zLine[iStart], zLine);
+  shell_check_oom(zSql);
+  sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  sqlite3_exec(globalDb, "PRAGMA page_count", 0, 0, 0); /* Load the schema */
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zCompletion = (const char*)sqlite3_column_text(pStmt, 0);
+    int nCompletion = sqlite3_column_bytes(pStmt, 0);
+    if( iStart+nCompletion < (i64)sizeof(zBuf)-1 && zCompletion ){
+      memcpy(zBuf+iStart, zCompletion, nCompletion+1);
+      linenoiseAddCompletion(lc, zBuf);
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+#endif
+
+/*
+** Do C-language style dequoting.
+**
+**    \a    -> alarm
+**    \b    -> backspace
+**    \t    -> tab
+**    \n    -> newline
+**    \v    -> vertical tab
+**    \f    -> form feed
+**    \r    -> carriage return
+**    \s    -> space
+**    \"    -> "
+**    \'    -> '
+**    \\    -> backslash
+**    \NNN  -> ascii character NNN in octal
+**    \xHH  -> ascii character HH in hexadecimal
+*/
+static void resolve_backslashes(char *z){
+  int i, j;
+  char c;
+  while( *z && *z!='\\' ) z++;
+  for(i=j=0; (c = z[i])!=0; i++, j++){
+    if( c=='\\' && z[i+1]!=0 ){
+      c = z[++i];
+      if( c=='a' ){
+        c = '\a';
+      }else if( c=='b' ){
+        c = '\b';
+      }else if( c=='t' ){
+        c = '\t';
+      }else if( c=='n' ){
+        c = '\n';
+      }else if( c=='v' ){
+        c = '\v';
+      }else if( c=='f' ){
+        c = '\f';
+      }else if( c=='r' ){
+        c = '\r';
+      }else if( c=='"' ){
+        c = '"';
+      }else if( c=='\'' ){
+        c = '\'';
+      }else if( c=='\\' ){
+        c = '\\';
+      }else if( c=='x' ){
+        int nhd = 0, hdv;
+        u8 hv = 0;
+        while( nhd<2 && (c=z[i+1+nhd])!=0 && (hdv=hexDigitValue(c))>=0 ){
+          hv = (u8)((hv<<4)|hdv);
+          ++nhd;
+        }
+        i += nhd;
+        c = (u8)hv;
+      }else if( c>='0' && c<='7' ){
+        c -= '0';
+        if( z[i+1]>='0' && z[i+1]<='7' ){
+          i++;
+          c = (c<<3) + z[i] - '0';
+          if( z[i+1]>='0' && z[i+1]<='7' ){
+            i++;
+            c = (c<<3) + z[i] - '0';
+          }
+        }
+      }
+    }
+    z[j] = c;
+  }
+  if( j=0; i++){}
+  }else{
+    for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){}
+  }
+  if( i>0 && zArg[i]==0 ) return (int)(integerValue(zArg) & 0xffffffff);
+  if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){
+    return 1;
+  }
+  if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){
+    return 0;
+  }
+  sqlite3_fprintf(stderr,
+       "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg);
+  return 0;
+}
+
+/*
+** Set or clear a shell flag according to a boolean value.
+*/
+static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){
+  if( booleanValue(zArg) ){
+    ShellSetFlag(p, mFlag);
+  }else{
+    ShellClearFlag(p, mFlag);
+  }
+}
+
+/*
+** Close an output file, assuming it is not stderr or stdout
+*/
+static void output_file_close(FILE *f){
+  if( f && f!=stdout && f!=stderr ) fclose(f);
+}
+
+/*
+** Try to open an output file.   The names "stdout" and "stderr" are
+** recognized and do the right thing.  NULL is returned if the output
+** filename is "off".
+*/
+static FILE *output_file_open(const char *zFile){
+  FILE *f;
+  if( cli_strcmp(zFile,"stdout")==0 ){
+    f = stdout;
+  }else if( cli_strcmp(zFile, "stderr")==0 ){
+    f = stderr;
+  }else if( cli_strcmp(zFile, "off")==0 ){
+    f = 0;
+  }else{
+    f = sqlite3_fopen(zFile, "w");
+    if( f==0 ){
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
+    }
+  }
+  return f;
+}
+
+#ifndef SQLITE_OMIT_TRACE
+/*
+** A routine for handling output from sqlite3_trace().
+*/
+static int sql_trace_callback(
+  unsigned mType,         /* The trace type */
+  void *pArg,             /* The ShellState pointer */
+  void *pP,               /* Usually a pointer to sqlite_stmt */
+  void *pX                /* Auxiliary output */
+){
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_stmt *pStmt;
+  const char *zSql;
+  i64 nSql;
+  if( p->traceOut==0 ) return 0;
+  if( mType==SQLITE_TRACE_CLOSE ){
+    sputz(p->traceOut, "-- closing database connection\n");
+    return 0;
+  }
+  if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){
+    zSql = (const char*)pX;
+  }else{
+    pStmt = (sqlite3_stmt*)pP;
+    switch( p->eTraceType ){
+      case SHELL_TRACE_EXPANDED: {
+        zSql = sqlite3_expanded_sql(pStmt);
+        break;
+      }
+#ifdef SQLITE_ENABLE_NORMALIZE
+      case SHELL_TRACE_NORMALIZED: {
+        zSql = sqlite3_normalized_sql(pStmt);
+        break;
+      }
+#endif
+      default: {
+        zSql = sqlite3_sql(pStmt);
+        break;
+      }
+    }
+  }
+  if( zSql==0 ) return 0;
+  nSql = strlen(zSql);
+  if( nSql>1000000000 ) nSql = 1000000000;
+  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
+  switch( mType ){
+    case SQLITE_TRACE_ROW:
+    case SQLITE_TRACE_STMT: {
+      sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0;
+      sqlite3_fprintf(p->traceOut,
+                      "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
+      break;
+    }
+  }
+  return 0;
+}
+#endif
+
+/*
+** A no-op routine that runs with the ".breakpoint" doc-command.  This is
+** a useful spot to set a debugger breakpoint.
+**
+** This routine does not do anything practical.  The code are there simply
+** to prevent the compiler from optimizing this routine out.
+*/
+static void test_breakpoint(void){
+  static unsigned int nCall = 0;
+  if( (nCall++)==0xffffffff ) printf("Many .breakpoints have run\n");
+}
+
+/*
+** An object used to read a CSV and other files for import.
+*/
+typedef struct ImportCtx ImportCtx;
+struct ImportCtx {
+  const char *zFile;  /* Name of the input file */
+  FILE *in;           /* Read the CSV text from this input stream */
+  int (SQLITE_CDECL *xCloser)(FILE*);      /* Func to close in */
+  char *z;            /* Accumulated text for a field */
+  int n;              /* Number of bytes in z */
+  int nAlloc;         /* Space allocated for z[] */
+  int nLine;          /* Current line number */
+  int nRow;           /* Number of rows imported */
+  int nErr;           /* Number of errors encountered */
+  int bNotFirst;      /* True if one or more bytes already read */
+  int cTerm;          /* Character that terminated the most recent field */
+  int cColSep;        /* The column separator character.  (Usually ",") */
+  int cRowSep;        /* The row separator character.  (Usually "\n") */
+};
+
+/* Clean up resourced used by an ImportCtx */
+static void import_cleanup(ImportCtx *p){
+  if( p->in!=0 && p->xCloser!=0 ){
+    p->xCloser(p->in);
+    p->in = 0;
+  }
+  sqlite3_free(p->z);
+  p->z = 0;
+}
+
+/* Append a single byte to z[] */
+static void import_append_char(ImportCtx *p, int c){
+  if( p->n+1>=p->nAlloc ){
+    p->nAlloc += p->nAlloc + 100;
+    p->z = sqlite3_realloc64(p->z, p->nAlloc);
+    shell_check_oom(p->z);
+  }
+  p->z[p->n++] = (char)c;
+}
+
+/* Read a single field of CSV text.  Compatible with rfc4180 and extended
+** with the option of having a separator other than ",".
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc64().
+**   +  Use p->cSep as the column separator.  The default is ",".
+**   +  Use p->rSep as the row separator.  The default is "\n".
+**   +  Keep track of the line number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**   +  Report syntax errors on stderr
+*/
+static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
+  int c;
+  int cSep = (u8)p->cColSep;
+  int rSep = (u8)p->cRowSep;
+  p->n = 0;
+  c = fgetc(p->in);
+  if( c==EOF || seenInterrupt ){
+    p->cTerm = EOF;
+    return 0;
+  }
+  if( c=='"' ){
+    int pc, ppc;
+    int startLine = p->nLine;
+    int cQuote = c;
+    pc = ppc = 0;
+    while( 1 ){
+      c = fgetc(p->in);
+      if( c==rSep ) p->nLine++;
+      if( c==cQuote ){
+        if( pc==cQuote ){
+          pc = 0;
+          continue;
+        }
+      }
+      if( (c==cSep && pc==cQuote)
+       || (c==rSep && pc==cQuote)
+       || (c==rSep && pc=='\r' && ppc==cQuote)
+       || (c==EOF && pc==cQuote)
+      ){
+        do{ p->n--; }while( p->z[p->n]!=cQuote );
+        p->cTerm = c;
+        break;
+      }
+      if( pc==cQuote && c!='\r' ){
+        sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", 
+                        p->zFile, p->nLine, cQuote);
+      }
+      if( c==EOF ){
+        sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n",
+              p->zFile, startLine, cQuote);
+        p->cTerm = c;
+        break;
+      }
+      import_append_char(p, c);
+      ppc = pc;
+      pc = c;
+    }
+  }else{
+    /* If this is the first field being parsed and it begins with the
+    ** UTF-8 BOM  (0xEF BB BF) then skip the BOM */
+    if( (c&0xff)==0xef && p->bNotFirst==0 ){
+      import_append_char(p, c);
+      c = fgetc(p->in);
+      if( (c&0xff)==0xbb ){
+        import_append_char(p, c);
+        c = fgetc(p->in);
+        if( (c&0xff)==0xbf ){
+          p->bNotFirst = 1;
+          p->n = 0;
+          return csv_read_one_field(p);
+        }
+      }
+    }
+    while( c!=EOF && c!=cSep && c!=rSep ){
+      import_append_char(p, c);
+      c = fgetc(p->in);
+    }
+    if( c==rSep ){
+      p->nLine++;
+      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
+    }
+    p->cTerm = c;
+  }
+  if( p->z ) p->z[p->n] = 0;
+  p->bNotFirst = 1;
+  return p->z;
+}
+
+/* Read a single field of ASCII delimited text.
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc64().
+**   +  Use p->cSep as the column separator.  The default is "\x1F".
+**   +  Use p->rSep as the row separator.  The default is "\x1E".
+**   +  Keep track of the row number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**   +  Report syntax errors on stderr
+*/
+static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){
+  int c;
+  int cSep = (u8)p->cColSep;
+  int rSep = (u8)p->cRowSep;
+  p->n = 0;
+  c = fgetc(p->in);
+  if( c==EOF || seenInterrupt ){
+    p->cTerm = EOF;
+    return 0;
+  }
+  while( c!=EOF && c!=cSep && c!=rSep ){
+    import_append_char(p, c);
+    c = fgetc(p->in);
+  }
+  if( c==rSep ){
+    p->nLine++;
+  }
+  p->cTerm = c;
+  if( p->z ) p->z[p->n] = 0;
+  return p->z;
+}
+
+/*
+** Try to transfer data for table zTable.  If an error is seen while
+** moving forward, try to go backwards.  The backwards movement won't
+** work for WITHOUT ROWID tables.
+*/
+static void tryToCloneData(
+  ShellState *p,
+  sqlite3 *newDb,
+  const char *zTable
+){
+  sqlite3_stmt *pQuery = 0;
+  sqlite3_stmt *pInsert = 0;
+  char *zQuery = 0;
+  char *zInsert = 0;
+  int rc;
+  int i, j, n;
+  int nTable = strlen30(zTable);
+  int k = 0;
+  int cnt = 0;
+  const int spinRate = 10000;
+
+  zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable);
+  shell_check_oom(zQuery);
+  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
+  if( rc ){
+    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
+          sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
+    goto end_data_xfer;
+  }
+  n = sqlite3_column_count(pQuery);
+  zInsert = sqlite3_malloc64(200 + nTable + n*3);
+  shell_check_oom(zInsert);
+  sqlite3_snprintf(200+nTable,zInsert,
+                   "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
+  i = strlen30(zInsert);
+  for(j=1; jdb, zQuery, -1, &pQuery, 0);
+    if( rc ){
+      sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
+      break;
+    }
+  } /* End for(k=0...) */
+
+end_data_xfer:
+  sqlite3_finalize(pQuery);
+  sqlite3_finalize(pInsert);
+  sqlite3_free(zQuery);
+  sqlite3_free(zInsert);
+}
+
+
+/*
+** Try to transfer all rows of the schema that match zWhere.  For
+** each row, invoke xForEach() on the object defined by that row.
+** If an error is encountered while moving forward through the
+** sqlite_schema table, try again moving backwards.
+*/
+static void tryToCloneSchema(
+  ShellState *p,
+  sqlite3 *newDb,
+  const char *zWhere,
+  void (*xForEach)(ShellState*,sqlite3*,const char*)
+){
+  sqlite3_stmt *pQuery = 0;
+  char *zQuery = 0;
+  int rc;
+  const unsigned char *zName;
+  const unsigned char *zSql;
+  char *zErrMsg = 0;
+
+  zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema"
+                           " WHERE %s ORDER BY rowid ASC", zWhere);
+  shell_check_oom(zQuery);
+  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
+  if( rc ){
+    sqlite3_fprintf(stderr,
+          "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
+          sqlite3_errmsg(p->db), zQuery);
+    goto end_schema_xfer;
+  }
+  while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
+    zName = sqlite3_column_text(pQuery, 0);
+    zSql = sqlite3_column_text(pQuery, 1);
+    if( zName==0 || zSql==0 ) continue;
+    if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
+      sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
+      if( zErrMsg ){
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_free(zErrMsg);
+        zErrMsg = 0;
+      }
+    }
+    if( xForEach ){
+      xForEach(p, newDb, (const char*)zName);
+    }
+    sputz(stdout, "done\n");
+  }
+  if( rc!=SQLITE_DONE ){
+    sqlite3_finalize(pQuery);
+    sqlite3_free(zQuery);
+    zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema"
+                             " WHERE %s ORDER BY rowid DESC", zWhere);
+    shell_check_oom(zQuery);
+    rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
+    if( rc ){
+      sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n",
+            sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
+      goto end_schema_xfer;
+    }
+    while( sqlite3_step(pQuery)==SQLITE_ROW ){
+      zName = sqlite3_column_text(pQuery, 0);
+      zSql = sqlite3_column_text(pQuery, 1);
+      if( zName==0 || zSql==0 ) continue;
+      if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue;
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
+      sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
+      if( zErrMsg ){
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_free(zErrMsg);
+        zErrMsg = 0;
+      }
+      if( xForEach ){
+        xForEach(p, newDb, (const char*)zName);
+      }
+      sputz(stdout, "done\n");
+    }
+  }
+end_schema_xfer:
+  sqlite3_finalize(pQuery);
+  sqlite3_free(zQuery);
+}
+
+/*
+** Open a new database file named "zNewDb".  Try to recover as much information
+** as possible out of the main database (which might be corrupt) and write it
+** into zNewDb.
+*/
+static void tryToClone(ShellState *p, const char *zNewDb){
+  int rc;
+  sqlite3 *newDb = 0;
+  if( access(zNewDb,0)==0 ){
+    sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb);
+    return;
+  }
+  rc = sqlite3_open(zNewDb, &newDb);
+  if( rc ){
+    sqlite3_fprintf(stderr,
+        "Cannot create output database: %s\n", sqlite3_errmsg(newDb));
+  }else{
+    sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
+    sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0);
+    tryToCloneSchema(p, newDb, "type='table'", tryToCloneData);
+    tryToCloneSchema(p, newDb, "type!='table'", 0);
+    sqlite3_exec(newDb, "COMMIT;", 0, 0, 0);
+    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
+  }
+  close_db(newDb);
+}
+
+#ifndef SQLITE_SHELL_FIDDLE
+/*
+** Change the output stream (file or pipe or console) to something else.
+*/
+static void output_redir(ShellState *p, FILE *pfNew){
+  if( p->out != stdout ){
+    sqlite3_fputs("Output already redirected.\n", stderr);
+  }else{
+    p->out = pfNew;
+    setCrlfMode(p);
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs(
+        "\n"
+        "
\n",
+        p->out
+      );
+    }
+  }
+}
+
+/*
+** Change the output file back to stdout.
+**
+** If the p->doXdgOpen flag is set, that means the output was being
+** redirected to a temporary file named by p->zTempFile.  In that case,
+** launch start/open/xdg-open on that temporary file.
+*/
+static void output_reset(ShellState *p){
+  if( p->outfile[0]=='|' ){
+#ifndef SQLITE_OMIT_POPEN
+    pclose(p->out);
+#endif
+  }else{
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs("
\n", p->out);
+    }
+    output_file_close(p->out);
+#ifndef SQLITE_NOHAVE_SYSTEM
+    if( p->doXdgOpen ){
+      const char *zXdgOpenCmd =
+#if defined(_WIN32)
+      "start";
+#elif defined(__APPLE__)
+      "open";
+#else
+      "xdg-open";
+#endif
+      char *zCmd;
+      zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
+      if( system(zCmd) ){
+        sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd);
+      }else{
+        /* Give the start/open/xdg-open command some time to get
+        ** going before we continue, and potential delete the
+        ** p->zTempFile data file out from under it */
+        sqlite3_sleep(2000);
+      }
+      sqlite3_free(zCmd);
+      outputModePop(p);
+      p->doXdgOpen = 0;
+    }
+#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */
+  }
+  p->outfile[0] = 0;
+  p->out = stdout;
+  setCrlfMode(p);
+}
+#else
+# define output_redir(SS,pfO)
+# define output_reset(SS)
+#endif
+
+/*
+** Run an SQL command and return the single integer result.
+*/
+static int db_int(sqlite3 *db, const char *zSql){
+  sqlite3_stmt *pStmt;
+  int res = 0;
+  sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
+    res = sqlite3_column_int(pStmt,0);
+  }
+  sqlite3_finalize(pStmt);
+  return res;
+}
+
+#if SQLITE_SHELL_HAVE_RECOVER
+/*
+** Convert a 2-byte or 4-byte big-endian integer into a native integer
+*/
+static unsigned int get2byteInt(unsigned char *a){
+  return (a[0]<<8) + a[1];
+}
+static unsigned int get4byteInt(unsigned char *a){
+  return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3];
+}
+
+/*
+** Implementation of the ".dbinfo" command.
+**
+** Return 1 on error, 2 to exit, and 0 otherwise.
+*/
+static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
+  static const struct { const char *zName; int ofst; } aField[] = {
+     { "file change counter:",  24  },
+     { "database page count:",  28  },
+     { "freelist page count:",  36  },
+     { "schema cookie:",        40  },
+     { "schema format:",        44  },
+     { "default cache size:",   48  },
+     { "autovacuum top root:",  52  },
+     { "incremental vacuum:",   64  },
+     { "text encoding:",        56  },
+     { "user version:",         60  },
+     { "application id:",       68  },
+     { "software version:",     96  },
+  };
+  static const struct { const char *zName; const char *zSql; } aQuery[] = {
+     { "number of tables:",
+       "SELECT count(*) FROM %s WHERE type='table'" },
+     { "number of indexes:",
+       "SELECT count(*) FROM %s WHERE type='index'" },
+     { "number of triggers:",
+       "SELECT count(*) FROM %s WHERE type='trigger'" },
+     { "number of views:",
+       "SELECT count(*) FROM %s WHERE type='view'" },
+     { "schema size:",
+       "SELECT total(length(sql)) FROM %s" },
+  };
+  int i, rc;
+  unsigned iDataVersion;
+  char *zSchemaTab;
+  char *zDb = nArg>=2 ? azArg[1] : "main";
+  sqlite3_stmt *pStmt = 0;
+  unsigned char aHdr[100];
+  open_db(p, 0);
+  if( p->db==0 ) return 1;
+  rc = sqlite3_prepare_v2(p->db,
+             "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1",
+             -1, &pStmt, 0);
+  if( rc ){
+    sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db));
+    sqlite3_finalize(pStmt);
+    return 1;
+  }
+  sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC);
+  if( sqlite3_step(pStmt)==SQLITE_ROW
+   && sqlite3_column_bytes(pStmt,0)>100
+  ){
+    const u8 *pb = sqlite3_column_blob(pStmt,0);
+    shell_check_oom(pb);
+    memcpy(aHdr, pb, 100);
+    sqlite3_finalize(pStmt);
+  }else{
+    sqlite3_fputs("unable to read database header\n", stderr);
+    sqlite3_finalize(pStmt);
+    return 1;
+  }
+  i = get2byteInt(aHdr+16);
+  if( i==1 ) i = 65536;
+  sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]);
+  for(i=0; iout, "%-20s %u", aField[i].zName, val);
+    switch( ofst ){
+      case 56: {
+        if( val==1 ) sqlite3_fputs(" (utf8)", p->out);
+        if( val==2 ) sqlite3_fputs(" (utf16le)", p->out);
+        if( val==3 ) sqlite3_fputs(" (utf16be)", p->out);
+      }
+    }
+    sqlite3_fputs("\n", p->out);
+  }
+  if( zDb==0 ){
+    zSchemaTab = sqlite3_mprintf("main.sqlite_schema");
+  }else if( cli_strcmp(zDb,"temp")==0 ){
+    zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema");
+  }else{
+    zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb);
+  }
+  for(i=0; idb, zSql);
+    sqlite3_free(zSql);
+    sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val);
+  }
+  sqlite3_free(zSchemaTab);
+  sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion);
+  sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion);
+  return 0;
+}
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
+
+/*
+** Print the given string as an error message.
+*/
+static void shellEmitError(const char *zErr){
+  sqlite3_fprintf(stderr,"Error: %s\n", zErr);
+}
+/*
+** Print the current sqlite3_errmsg() value to stderr and return 1.
+*/
+static int shellDatabaseError(sqlite3 *db){
+  shellEmitError(sqlite3_errmsg(db));
+  return 1;
+}
+
+/*
+** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
+** if they match and FALSE (0) if they do not match.
+**
+** Globbing rules:
+**
+**      '*'       Matches any sequence of zero or more characters.
+**
+**      '?'       Matches exactly one character.
+**
+**     [...]      Matches one character from the enclosed list of
+**                characters.
+**
+**     [^...]     Matches one character not in the enclosed list.
+**
+**      '#'       Matches any sequence of one or more digits with an
+**                optional + or - sign in front
+**
+**      ' '       Any span of whitespace matches any other span of
+**                whitespace.
+**
+** Extra whitespace at the end of z[] is ignored.
+*/
+static int testcase_glob(const char *zGlob, const char *z){
+  int c, c2;
+  int invert;
+  int seen;
+
+  while( (c = (*(zGlob++)))!=0 ){
+    if( IsSpace(c) ){
+      if( !IsSpace(*z) ) return 0;
+      while( IsSpace(*zGlob) ) zGlob++;
+      while( IsSpace(*z) ) z++;
+    }else if( c=='*' ){
+      while( (c=(*(zGlob++))) == '*' || c=='?' ){
+        if( c=='?' && (*(z++))==0 ) return 0;
+      }
+      if( c==0 ){
+        return 1;
+      }else if( c=='[' ){
+        while( *z && testcase_glob(zGlob-1,z)==0 ){
+          z++;
+        }
+        return (*z)!=0;
+      }
+      while( (c2 = (*(z++)))!=0 ){
+        while( c2!=c ){
+          c2 = *(z++);
+          if( c2==0 ) return 0;
+        }
+        if( testcase_glob(zGlob,z) ) return 1;
+      }
+      return 0;
+    }else if( c=='?' ){
+      if( (*(z++))==0 ) return 0;
+    }else if( c=='[' ){
+      int prior_c = 0;
+      seen = 0;
+      invert = 0;
+      c = *(z++);
+      if( c==0 ) return 0;
+      c2 = *(zGlob++);
+      if( c2=='^' ){
+        invert = 1;
+        c2 = *(zGlob++);
+      }
+      if( c2==']' ){
+        if( c==']' ) seen = 1;
+        c2 = *(zGlob++);
+      }
+      while( c2 && c2!=']' ){
+        if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){
+          c2 = *(zGlob++);
+          if( c>=prior_c && c<=c2 ) seen = 1;
+          prior_c = 0;
+        }else{
+          if( c==c2 ){
+            seen = 1;
+          }
+          prior_c = c2;
+        }
+        c2 = *(zGlob++);
+      }
+      if( c2==0 || (seen ^ invert)==0 ) return 0;
+    }else if( c=='#' ){
+      if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++;
+      if( !IsDigit(z[0]) ) return 0;
+      z++;
+      while( IsDigit(z[0]) ){ z++; }
+    }else{
+      if( c!=(*(z++)) ) return 0;
+    }
+  }
+  while( IsSpace(*z) ){ z++; }
+  return *z==0;
+}
+
+
+/*
+** Compare the string as a command-line option with either one or two
+** initial "-" characters.
+*/
+static int optionMatch(const char *zStr, const char *zOpt){
+  if( zStr[0]!='-' ) return 0;
+  zStr++;
+  if( zStr[0]=='-' ) zStr++;
+  return cli_strcmp(zStr, zOpt)==0;
+}
+
+/*
+** Delete a file.
+*/
+int shellDeleteFile(const char *zFilename){
+  int rc;
+#ifdef _WIN32
+  wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename);
+  rc = _wunlink(z);
+  sqlite3_free(z);
+#else
+  rc = unlink(zFilename);
+#endif
+  return rc;
+}
+
+/*
+** Try to delete the temporary file (if there is one) and free the
+** memory used to hold the name of the temp file.
+*/
+static void clearTempFile(ShellState *p){
+  if( p->zTempFile==0 ) return;
+  if( p->doXdgOpen ) return;
+  if( shellDeleteFile(p->zTempFile) ) return;
+  sqlite3_free(p->zTempFile);
+  p->zTempFile = 0;
+}
+
+/*
+** Create a new temp file name with the given suffix.
+*/
+static void newTempFile(ShellState *p, const char *zSuffix){
+  clearTempFile(p);
+  sqlite3_free(p->zTempFile);
+  p->zTempFile = 0;
+  if( p->db ){
+    sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile);
+  }
+  if( p->zTempFile==0 ){
+    /* If p->db is an in-memory database then the TEMPFILENAME file-control
+    ** will not work and we will need to fallback to guessing */
+    char *zTemp;
+    sqlite3_uint64 r;
+    sqlite3_randomness(sizeof(r), &r);
+    zTemp = getenv("TEMP");
+    if( zTemp==0 ) zTemp = getenv("TMP");
+    if( zTemp==0 ){
+#ifdef _WIN32
+      zTemp = "\\tmp";
+#else
+      zTemp = "/tmp";
+#endif
+    }
+    p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix);
+  }else{
+    p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix);
+  }
+  shell_check_oom(p->zTempFile);
+}
+
+
+/*
+** The implementation of SQL scalar function fkey_collate_clause(), used
+** by the ".lint fkey-indexes" command. This scalar function is always
+** called with four arguments - the parent table name, the parent column name,
+** the child table name and the child column name.
+**
+**   fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col')
+**
+** If either of the named tables or columns do not exist, this function
+** returns an empty string. An empty string is also returned if both tables
+** and columns exist but have the same default collation sequence. Or,
+** if both exist but the default collation sequences are different, this
+** function returns the string " COLLATE ", where
+**  is the default collation sequence of the parent column.
+*/
+static void shellFkeyCollateClause(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  sqlite3 *db = sqlite3_context_db_handle(pCtx);
+  const char *zParent;
+  const char *zParentCol;
+  const char *zParentSeq;
+  const char *zChild;
+  const char *zChildCol;
+  const char *zChildSeq = 0;  /* Initialize to avoid false-positive warning */
+  int rc;
+
+  assert( nVal==4 );
+  zParent = (const char*)sqlite3_value_text(apVal[0]);
+  zParentCol = (const char*)sqlite3_value_text(apVal[1]);
+  zChild = (const char*)sqlite3_value_text(apVal[2]);
+  zChildCol = (const char*)sqlite3_value_text(apVal[3]);
+
+  sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
+  rc = sqlite3_table_column_metadata(
+      db, "main", zParent, zParentCol, 0, &zParentSeq, 0, 0, 0
+  );
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_table_column_metadata(
+        db, "main", zChild, zChildCol, 0, &zChildSeq, 0, 0, 0
+    );
+  }
+
+  if( rc==SQLITE_OK && sqlite3_stricmp(zParentSeq, zChildSeq) ){
+    char *z = sqlite3_mprintf(" COLLATE %s", zParentSeq);
+    sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT);
+    sqlite3_free(z);
+  }
+}
+
+
+/*
+** The implementation of dot-command ".lint fkey-indexes".
+*/
+static int lintFkeyIndexes(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  sqlite3 *db = pState->db;       /* Database handle to query "main" db of */
+  int bVerbose = 0;               /* If -verbose is present */
+  int bGroupByParent = 0;         /* If -groupbyparent is present */
+  int i;                          /* To iterate through azArg[] */
+  const char *zIndent = "";       /* How much to indent CREATE INDEX by */
+  int rc;                         /* Return code */
+  sqlite3_stmt *pSql = 0;         /* Compiled version of SQL statement below */
+  FILE *out = pState->out;        /* Send output here */
+
+  /*
+  ** This SELECT statement returns one row for each foreign key constraint
+  ** in the schema of the main database. The column values are:
+  **
+  ** 0. The text of an SQL statement similar to:
+  **
+  **      "EXPLAIN QUERY PLAN SELECT 1 FROM child_table WHERE child_key=?"
+  **
+  **    This SELECT is similar to the one that the foreign keys implementation
+  **    needs to run internally on child tables. If there is an index that can
+  **    be used to optimize this query, then it can also be used by the FK
+  **    implementation to optimize DELETE or UPDATE statements on the parent
+  **    table.
+  **
+  ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by
+  **    the EXPLAIN QUERY PLAN command matches this pattern, then the schema
+  **    contains an index that can be used to optimize the query.
+  **
+  ** 2. Human readable text that describes the child table and columns. e.g.
+  **
+  **       "child_table(child_key1, child_key2)"
+  **
+  ** 3. Human readable text that describes the parent table and columns. e.g.
+  **
+  **       "parent_table(parent_key1, parent_key2)"
+  **
+  ** 4. A full CREATE INDEX statement for an index that could be used to
+  **    optimize DELETE or UPDATE statements on the parent table. e.g.
+  **
+  **       "CREATE INDEX child_table_child_key ON child_table(child_key)"
+  **
+  ** 5. The name of the parent table.
+  **
+  ** These six values are used by the C logic below to generate the report.
+  */
+  const char *zSql =
+  "SELECT "
+    "     'EXPLAIN QUERY PLAN SELECT 1 FROM ' || quote(s.name) || ' WHERE '"
+    "  || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' "
+    "  || fkey_collate_clause("
+    "       f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')"
+    ", "
+    "     'SEARCH ' || s.name || ' USING COVERING INDEX*('"
+    "  || group_concat('*=?', ' AND ') || ')'"
+    ", "
+    "     s.name  || '(' || group_concat(f.[from],  ', ') || ')'"
+    ", "
+    "     f.[table] || '(' || group_concat(COALESCE(f.[to], p.[name])) || ')'"
+    ", "
+    "     'CREATE INDEX ' || quote(s.name ||'_'|| group_concat(f.[from], '_'))"
+    "  || ' ON ' || quote(s.name) || '('"
+    "  || group_concat(quote(f.[from]) ||"
+    "        fkey_collate_clause("
+    "          f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]), ', ')"
+    "  || ');'"
+    ", "
+    "     f.[table] "
+    "FROM sqlite_schema AS s, pragma_foreign_key_list(s.name) AS f "
+    "LEFT JOIN pragma_table_info AS p ON (pk-1=seq AND p.arg=f.[table]) "
+    "GROUP BY s.name, f.id "
+    "ORDER BY (CASE WHEN ? THEN f.[table] ELSE s.name END)"
+  ;
+  const char *zGlobIPK = "SEARCH * USING INTEGER PRIMARY KEY (rowid=?)";
+
+  for(i=2; i1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){
+      bVerbose = 1;
+    }
+    else if( n>1 && sqlite3_strnicmp("-groupbyparent", azArg[i], n)==0 ){
+      bGroupByParent = 1;
+      zIndent = "    ";
+    }
+    else{
+      sqlite3_fprintf(stderr,
+           "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]);
+      return SQLITE_ERROR;
+    }
+  }
+
+  /* Register the fkey_collate_clause() SQL function */
+  rc = sqlite3_create_function(db, "fkey_collate_clause", 4, SQLITE_UTF8,
+      0, shellFkeyCollateClause, 0, 0
+  );
+
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0);
+  }
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int(pSql, 1, bGroupByParent);
+  }
+
+  if( rc==SQLITE_OK ){
+    int rc2;
+    char *zPrev = 0;
+    while( SQLITE_ROW==sqlite3_step(pSql) ){
+      int res = -1;
+      sqlite3_stmt *pExplain = 0;
+      const char *zEQP = (const char*)sqlite3_column_text(pSql, 0);
+      const char *zGlob = (const char*)sqlite3_column_text(pSql, 1);
+      const char *zFrom = (const char*)sqlite3_column_text(pSql, 2);
+      const char *zTarget = (const char*)sqlite3_column_text(pSql, 3);
+      const char *zCI = (const char*)sqlite3_column_text(pSql, 4);
+      const char *zParent = (const char*)sqlite3_column_text(pSql, 5);
+
+      if( zEQP==0 ) continue;
+      if( zGlob==0 ) continue;
+      rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+      if( rc!=SQLITE_OK ) break;
+      if( SQLITE_ROW==sqlite3_step(pExplain) ){
+        const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3);
+        res = zPlan!=0 && (  0==sqlite3_strglob(zGlob, zPlan)
+                          || 0==sqlite3_strglob(zGlobIPK, zPlan));
+      }
+      rc = sqlite3_finalize(pExplain);
+      if( rc!=SQLITE_OK ) break;
+
+      if( res<0 ){
+        sqlite3_fputs("Error: internal error", stderr);
+        break;
+      }else{
+        if( bGroupByParent
+        && (bVerbose || res==0)
+        && (zPrev==0 || sqlite3_stricmp(zParent, zPrev))
+        ){
+          sqlite3_fprintf(out, "-- Parent table %s\n", zParent);
+          sqlite3_free(zPrev);
+          zPrev = sqlite3_mprintf("%s", zParent);
+        }
+
+        if( res==0 ){
+          sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget);
+        }else if( bVerbose ){
+          sqlite3_fprintf(out,
+                "%s/* no extra indexes required for %s -> %s */\n",
+                zIndent, zFrom, zTarget
+          );
+        }
+      }
+    }
+    sqlite3_free(zPrev);
+
+    if( rc!=SQLITE_OK ){
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
+    }
+
+    rc2 = sqlite3_finalize(pSql);
+    if( rc==SQLITE_OK && rc2!=SQLITE_OK ){
+      rc = rc2;
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
+    }
+  }else{
+    sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
+  }
+
+  return rc;
+}
+
+/*
+** Implementation of ".lint" dot command.
+*/
+static int lintDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int n;
+  n = (nArg>=2 ? strlen30(azArg[1]) : 0);
+  if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage;
+  return lintFkeyIndexes(pState, azArg, nArg);
+
+ usage:
+  sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]);
+  sqlite3_fprintf(stderr, "Where sub-commands are:\n");
+  sqlite3_fprintf(stderr, "    fkey-indexes\n");
+  return SQLITE_ERROR;
+}
+
+static void shellPrepare(
+  sqlite3 *db,
+  int *pRc,
+  const char *zSql,
+  sqlite3_stmt **ppStmt
+){
+  *ppStmt = 0;
+  if( *pRc==SQLITE_OK ){
+    int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+    if( rc!=SQLITE_OK ){
+      sqlite3_fprintf(stderr,
+         "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db));
+      *pRc = rc;
+    }
+  }
+}
+
+/*
+** Create a prepared statement using printf-style arguments for the SQL.
+*/
+static void shellPreparePrintf(
+  sqlite3 *db,
+  int *pRc,
+  sqlite3_stmt **ppStmt,
+  const char *zFmt,
+  ...
+){
+  *ppStmt = 0;
+  if( *pRc==SQLITE_OK ){
+    va_list ap;
+    char *z;
+    va_start(ap, zFmt);
+    z = sqlite3_vmprintf(zFmt, ap);
+    va_end(ap);
+    if( z==0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      shellPrepare(db, pRc, z, ppStmt);
+      sqlite3_free(z);
+    }
+  }
+}
+
+/* 
+** Finalize the prepared statement created using shellPreparePrintf().
+*/
+static void shellFinalize(
+  int *pRc,
+  sqlite3_stmt *pStmt
+){
+  if( pStmt ){
+    sqlite3 *db = sqlite3_db_handle(pStmt);
+    int rc = sqlite3_finalize(pStmt);
+    if( *pRc==SQLITE_OK ){
+      if( rc!=SQLITE_OK ){
+        sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
+      }
+      *pRc = rc;
+    }
+  }
+}
+
+#if !defined SQLITE_OMIT_VIRTUALTABLE
+/* Reset the prepared statement created using shellPreparePrintf().
+**
+** This routine is could be marked "static".  But it is not always used,
+** depending on compile-time options.  By omitting the "static", we avoid
+** nuisance compiler warnings about "defined but not used".
+*/
+void shellReset(
+  int *pRc,
+  sqlite3_stmt *pStmt
+){
+  int rc = sqlite3_reset(pStmt);
+  if( *pRc==SQLITE_OK ){
+    if( rc!=SQLITE_OK ){
+      sqlite3 *db = sqlite3_db_handle(pStmt);
+      sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
+    }
+    *pRc = rc;
+  }
+}
+#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */
+
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+/******************************************************************************
+** The ".archive" or ".ar" command.
+*/
+/*
+** Structure representing a single ".ar" command.
+*/
+typedef struct ArCommand ArCommand;
+struct ArCommand {
+  u8 eCmd;                        /* An AR_CMD_* value */
+  u8 bVerbose;                    /* True if --verbose */
+  u8 bZip;                        /* True if the archive is a ZIP */
+  u8 bDryRun;                     /* True if --dry-run */
+  u8 bAppend;                     /* True if --append */
+  u8 bGlob;                       /* True if --glob */
+  u8 fromCmdLine;                 /* Run from -A instead of .archive */
+  int nArg;                       /* Number of command arguments */
+  char *zSrcTable;                /* "sqlar", "zipfile($file)" or "zip" */
+  const char *zFile;              /* --file argument, or NULL */
+  const char *zDir;               /* --directory argument, or NULL */
+  char **azArg;                   /* Array of command arguments */
+  ShellState *p;                  /* Shell state */
+  FILE *out;                      /* Output to this stream */
+  sqlite3 *db;                    /* Database containing the archive */
+};
+
+/*
+** Print a usage message for the .ar command to stderr and return SQLITE_ERROR.
+*/
+static int arUsage(FILE *f){
+  showHelp(f,"archive");
+  return SQLITE_ERROR;
+}
+
+/*
+** Print an error message for the .ar command to stderr and return
+** SQLITE_ERROR.
+*/
+static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){
+  va_list ap;
+  char *z;
+  va_start(ap, zFmt);
+  z = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+  shellEmitError(z);
+  if( pAr->fromCmdLine ){
+    sqlite3_fputs("Use \"-A\" for more help\n", stderr);
+  }else{
+    sqlite3_fputs("Use \".archive --help\" for more help\n", stderr);
+  }
+  sqlite3_free(z);
+  return SQLITE_ERROR;
+}
+
+/*
+** Values for ArCommand.eCmd.
+*/
+#define AR_CMD_CREATE       1
+#define AR_CMD_UPDATE       2
+#define AR_CMD_INSERT       3
+#define AR_CMD_EXTRACT      4
+#define AR_CMD_LIST         5
+#define AR_CMD_HELP         6
+#define AR_CMD_REMOVE       7
+
+/*
+** Other (non-command) switches.
+*/
+#define AR_SWITCH_VERBOSE     8
+#define AR_SWITCH_FILE        9
+#define AR_SWITCH_DIRECTORY  10
+#define AR_SWITCH_APPEND     11
+#define AR_SWITCH_DRYRUN     12
+#define AR_SWITCH_GLOB       13
+
+static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
+  switch( eSwitch ){
+    case AR_CMD_CREATE:
+    case AR_CMD_EXTRACT:
+    case AR_CMD_LIST:
+    case AR_CMD_REMOVE:
+    case AR_CMD_UPDATE:
+    case AR_CMD_INSERT:
+    case AR_CMD_HELP:
+      if( pAr->eCmd ){
+        return arErrorMsg(pAr, "multiple command options");
+      }
+      pAr->eCmd = eSwitch;
+      break;
+
+    case AR_SWITCH_DRYRUN:
+      pAr->bDryRun = 1;
+      break;
+    case AR_SWITCH_GLOB:
+      pAr->bGlob = 1;
+      break;
+    case AR_SWITCH_VERBOSE:
+      pAr->bVerbose = 1;
+      break;
+    case AR_SWITCH_APPEND:
+      pAr->bAppend = 1;
+      deliberate_fall_through;
+    case AR_SWITCH_FILE:
+      pAr->zFile = zArg;
+      break;
+    case AR_SWITCH_DIRECTORY:
+      pAr->zDir = zArg;
+      break;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Parse the command line for an ".ar" command. The results are written into
+** structure (*pAr). SQLITE_OK is returned if the command line is parsed
+** successfully, otherwise an error message is written to stderr and
+** SQLITE_ERROR returned.
+*/
+static int arParseCommand(
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg,                       /* Number of entries in azArg[] */
+  ArCommand *pAr                  /* Populate this object */
+){
+  struct ArSwitch {
+    const char *zLong;
+    char cShort;
+    u8 eSwitch;
+    u8 bArg;
+  } aSwitch[] = {
+    { "create",    'c', AR_CMD_CREATE,       0 },
+    { "extract",   'x', AR_CMD_EXTRACT,      0 },
+    { "insert",    'i', AR_CMD_INSERT,       0 },
+    { "list",      't', AR_CMD_LIST,         0 },
+    { "remove",    'r', AR_CMD_REMOVE,       0 },
+    { "update",    'u', AR_CMD_UPDATE,       0 },
+    { "help",      'h', AR_CMD_HELP,         0 },
+    { "verbose",   'v', AR_SWITCH_VERBOSE,   0 },
+    { "file",      'f', AR_SWITCH_FILE,      1 },
+    { "append",    'a', AR_SWITCH_APPEND,    1 },
+    { "directory", 'C', AR_SWITCH_DIRECTORY, 1 },
+    { "dryrun",    'n', AR_SWITCH_DRYRUN,    0 },
+    { "glob",      'g', AR_SWITCH_GLOB,      0 },
+  };
+  int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);
+  struct ArSwitch *pEnd = &aSwitch[nSwitch];
+
+  if( nArg<=1 ){
+    sqlite3_fprintf(stderr, "Wrong number of arguments.  Usage:\n");
+    return arUsage(stderr);
+  }else{
+    char *z = azArg[1];
+    if( z[0]!='-' ){
+      /* Traditional style [tar] invocation */
+      int i;
+      int iArg = 2;
+      for(i=0; z[i]; i++){
+        const char *zArg = 0;
+        struct ArSwitch *pOpt;
+        for(pOpt=&aSwitch[0]; pOptcShort ) break;
+        }
+        if( pOpt==pEnd ){
+          return arErrorMsg(pAr, "unrecognized option: %c", z[i]);
+        }
+        if( pOpt->bArg ){
+          if( iArg>=nArg ){
+            return arErrorMsg(pAr, "option requires an argument: %c",z[i]);
+          }
+          zArg = azArg[iArg++];
+        }
+        if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR;
+      }
+      pAr->nArg = nArg-iArg;
+      if( pAr->nArg>0 ){
+        pAr->azArg = &azArg[iArg];
+      }
+    }else{
+      /* Non-traditional invocation */
+      int iArg;
+      for(iArg=1; iArgazArg = &azArg[iArg];
+          pAr->nArg = nArg-iArg;
+          break;
+        }
+        n = strlen30(z);
+
+        if( z[1]!='-' ){
+          int i;
+          /* One or more short options */
+          for(i=1; icShort ) break;
+            }
+            if( pOpt==pEnd ){
+              return arErrorMsg(pAr, "unrecognized option: %c", z[i]);
+            }
+            if( pOpt->bArg ){
+              if( i<(n-1) ){
+                zArg = &z[i+1];
+                i = n;
+              }else{
+                if( iArg>=(nArg-1) ){
+                  return arErrorMsg(pAr, "option requires an argument: %c",
+                                    z[i]);
+                }
+                zArg = azArg[++iArg];
+              }
+            }
+            if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR;
+          }
+        }else if( z[2]=='\0' ){
+          /* A -- option, indicating that all remaining command line words
+          ** are command arguments.  */
+          pAr->azArg = &azArg[iArg+1];
+          pAr->nArg = nArg-iArg-1;
+          break;
+        }else{
+          /* A long option */
+          const char *zArg = 0;             /* Argument for option, if any */
+          struct ArSwitch *pMatch = 0;      /* Matching option */
+          struct ArSwitch *pOpt;            /* Iterator */
+          for(pOpt=&aSwitch[0]; pOptzLong;
+            if( (n-2)<=strlen30(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
+              if( pMatch ){
+                return arErrorMsg(pAr, "ambiguous option: %s",z);
+              }else{
+                pMatch = pOpt;
+              }
+            }
+          }
+
+          if( pMatch==0 ){
+            return arErrorMsg(pAr, "unrecognized option: %s", z);
+          }
+          if( pMatch->bArg ){
+            if( iArg>=(nArg-1) ){
+              return arErrorMsg(pAr, "option requires an argument: %s", z);
+            }
+            zArg = azArg[++iArg];
+          }
+          if( arProcessSwitch(pAr, pMatch->eSwitch, zArg) ) return SQLITE_ERROR;
+        }
+      }
+    }
+  }
+  if( pAr->eCmd==0 ){
+    sqlite3_fprintf(stderr, "Required argument missing.  Usage:\n");
+    return arUsage(stderr);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function assumes that all arguments within the ArCommand.azArg[]
+** array refer to archive members, as for the --extract, --list or --remove
+** commands. It checks that each of them are "present". If any specified
+** file is not present in the archive, an error is printed to stderr and an
+** error code returned. Otherwise, if all specified arguments are present
+** in the archive, SQLITE_OK is returned. Here, "present" means either an
+** exact equality when pAr->bGlob is false or a "name GLOB pattern" match
+** when pAr->bGlob is true.
+**
+** This function strips any trailing '/' characters from each argument.
+** This is consistent with the way the [tar] command seems to work on
+** Linux.
+*/
+static int arCheckEntries(ArCommand *pAr){
+  int rc = SQLITE_OK;
+  if( pAr->nArg ){
+    int i, j;
+    sqlite3_stmt *pTest = 0;
+    const char *zSel = (pAr->bGlob)
+      ? "SELECT name FROM %s WHERE glob($name,name)"
+      : "SELECT name FROM %s WHERE name=$name";
+
+    shellPreparePrintf(pAr->db, &rc, &pTest, zSel, pAr->zSrcTable);
+    j = sqlite3_bind_parameter_index(pTest, "$name");
+    for(i=0; inArg && rc==SQLITE_OK; i++){
+      char *z = pAr->azArg[i];
+      int n = strlen30(z);
+      int bOk = 0;
+      while( n>0 && z[n-1]=='/' ) n--;
+      z[n] = '\0';
+      sqlite3_bind_text(pTest, j, z, -1, SQLITE_STATIC);
+      if( SQLITE_ROW==sqlite3_step(pTest) ){
+        bOk = 1;
+      }
+      shellReset(&rc, pTest);
+      if( rc==SQLITE_OK && bOk==0 ){
+        sqlite3_fprintf(stderr,"not found in archive: %s\n", z);
+        rc = SQLITE_ERROR;
+      }
+    }
+    shellFinalize(&rc, pTest);
+  }
+  return rc;
+}
+
+/*
+** Format a WHERE clause that can be used against the "sqlar" table to
+** identify all archive members that match the command arguments held
+** in (*pAr). Leave this WHERE clause in (*pzWhere) before returning.
+** The caller is responsible for eventually calling sqlite3_free() on
+** any non-NULL (*pzWhere) value. Here, "match" means strict equality
+** when pAr->bGlob is false and GLOB match when pAr->bGlob is true.
+*/
+static void arWhereClause(
+  int *pRc,
+  ArCommand *pAr,
+  char **pzWhere                  /* OUT: New WHERE clause */
+){
+  char *zWhere = 0;
+  const char *zSameOp = (pAr->bGlob)? "GLOB" : "=";
+  if( *pRc==SQLITE_OK ){
+    if( pAr->nArg==0 ){
+      zWhere = sqlite3_mprintf("1");
+    }else{
+      int i;
+      const char *zSep = "";
+      for(i=0; inArg; i++){
+        const char *z = pAr->azArg[i];
+        zWhere = sqlite3_mprintf(
+          "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'",
+          zWhere, zSep, zSameOp, z, strlen30(z)+1, zSameOp, z
+        );
+        if( zWhere==0 ){
+          *pRc = SQLITE_NOMEM;
+          break;
+        }
+        zSep = " OR ";
+      }
+    }
+  }
+  *pzWhere = zWhere;
+}
+
+/*
+** Implementation of .ar "lisT" command.
+*/
+static int arListCommand(ArCommand *pAr){
+  const char *zSql = "SELECT %s FROM %s WHERE %s";
+  const char *azCols[] = {
+    "name",
+    "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name"
+  };
+
+  char *zWhere = 0;
+  sqlite3_stmt *pSql = 0;
+  int rc;
+
+  rc = arCheckEntries(pAr);
+  arWhereClause(&rc, pAr, &zWhere);
+
+  shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
+                     pAr->zSrcTable, zWhere);
+  if( pAr->bDryRun ){
+    sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
+  }else{
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+      if( pAr->bVerbose ){
+        sqlite3_fprintf(pAr->out, "%s % 10d  %s  %s\n",
+              sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1),
+              sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3));
+      }else{
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
+      }
+    }
+  }
+  shellFinalize(&rc, pSql);
+  sqlite3_free(zWhere);
+  return rc;
+}
+
+/*
+** Implementation of .ar "Remove" command.
+*/
+static int arRemoveCommand(ArCommand *pAr){
+  int rc = 0;
+  char *zSql = 0;
+  char *zWhere = 0;
+
+  if( pAr->nArg ){
+    /* Verify that args actually exist within the archive before proceeding.
+    ** And formulate a WHERE clause to match them.  */
+    rc = arCheckEntries(pAr);
+    arWhereClause(&rc, pAr, &zWhere);
+  }
+  if( rc==SQLITE_OK ){
+    zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;",
+                           pAr->zSrcTable, zWhere);
+    if( pAr->bDryRun ){
+      sqlite3_fprintf(pAr->out, "%s\n", zSql);
+    }else{
+      char *zErr = 0;
+      rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr);
+        if( rc!=SQLITE_OK ){
+          sqlite3_exec(pAr->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
+        }else{
+          rc = sqlite3_exec(pAr->db, "RELEASE ar;", 0, 0, 0);
+        }
+      }
+      if( zErr ){
+        sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */
+        sqlite3_free(zErr);
+      }
+    }
+  }
+  sqlite3_free(zWhere);
+  sqlite3_free(zSql);
+  return rc;
+}
+
+/*
+** Implementation of .ar "eXtract" command.
+*/
+static int arExtractCommand(ArCommand *pAr){
+  const char *zSql1 =
+    "SELECT "
+    " ($dir || name),"
+    " writefile(($dir || name), %s, mode, mtime) "
+    "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)"
+    " AND name NOT GLOB '*..[/\\]*'";
+
+  const char *azExtraArg[] = {
+    "sqlar_uncompress(data, sz)",
+    "data"
+  };
+
+  sqlite3_stmt *pSql = 0;
+  int rc = SQLITE_OK;
+  char *zDir = 0;
+  char *zWhere = 0;
+  int i, j;
+
+  /* If arguments are specified, check that they actually exist within
+  ** the archive before proceeding. And formulate a WHERE clause to
+  ** match them.  */
+  rc = arCheckEntries(pAr);
+  arWhereClause(&rc, pAr, &zWhere);
+
+  if( rc==SQLITE_OK ){
+    if( pAr->zDir ){
+      zDir = sqlite3_mprintf("%s/", pAr->zDir);
+    }else{
+      zDir = sqlite3_mprintf("");
+    }
+    if( zDir==0 ) rc = SQLITE_NOMEM;
+  }
+
+  shellPreparePrintf(pAr->db, &rc, &pSql, zSql1,
+      azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere
+  );
+
+  if( rc==SQLITE_OK ){
+    j = sqlite3_bind_parameter_index(pSql, "$dir");
+    sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC);
+
+    /* Run the SELECT statement twice. The first time, writefile() is called
+    ** for all archive members that should be extracted. The second time,
+    ** only for the directories. This is because the timestamps for
+    ** extracted directories must be reset after they are populated (as
+    ** populating them changes the timestamp).  */
+    for(i=0; i<2; i++){
+      j = sqlite3_bind_parameter_index(pSql, "$dirOnly");
+      sqlite3_bind_int(pSql, j, i);
+      if( pAr->bDryRun ){
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
+      }else{
+        while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+          if( i==0 && pAr->bVerbose ){
+            sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
+          }
+        }
+      }
+      shellReset(&rc, pSql);
+    }
+    shellFinalize(&rc, pSql);
+  }
+
+  sqlite3_free(zDir);
+  sqlite3_free(zWhere);
+  return rc;
+}
+
+/*
+** Run the SQL statement in zSql.  Or if doing a --dryrun, merely print it out.
+*/
+static int arExecSql(ArCommand *pAr, const char *zSql){
+  int rc;
+  if( pAr->bDryRun ){
+    sqlite3_fprintf(pAr->out, "%s\n", zSql);
+    rc = SQLITE_OK;
+  }else{
+    char *zErr = 0;
+    rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr);
+    if( zErr ){
+      sqlite3_fprintf(stdout, "ERROR: %s\n", zErr);
+      sqlite3_free(zErr);
+    }
+  }
+  return rc;
+}
+
+
+/*
+** Implementation of .ar "create", "insert", and "update" commands.
+**
+**     create    ->     Create a new SQL archive
+**     insert    ->     Insert or reinsert all files listed
+**     update    ->     Insert files that have changed or that were not
+**                      previously in the archive
+**
+** Create the "sqlar" table in the database if it does not already exist.
+** Then add each file in the azFile[] array to the archive. Directories
+** are added recursively. If argument bVerbose is non-zero, a message is
+** printed on stdout for each file archived.
+**
+** The create command is the same as update, except that it drops
+** any existing "sqlar" table before beginning.  The "insert" command
+** always overwrites every file named on the command-line, where as
+** "update" only overwrites if the size or mtime or mode has changed.
+*/
+static int arCreateOrUpdateCommand(
+  ArCommand *pAr,                 /* Command arguments and options */
+  int bUpdate,                    /* true for a --create. */
+  int bOnlyIfChanged              /* Only update if file has changed */
+){
+  const char *zCreate =
+      "CREATE TABLE IF NOT EXISTS sqlar(\n"
+      "  name TEXT PRIMARY KEY,  -- name of the file\n"
+      "  mode INT,               -- access permissions\n"
+      "  mtime INT,              -- last modification time\n"
+      "  sz INT,                 -- original file size\n"
+      "  data BLOB               -- compressed content\n"
+      ")";
+  const char *zDrop = "DROP TABLE IF EXISTS sqlar";
+  const char *zInsertFmt[2] = {
+     "REPLACE INTO %s(name,mode,mtime,sz,data)\n"
+     "  SELECT\n"
+     "    %s,\n"
+     "    mode,\n"
+     "    mtime,\n"
+     "    CASE substr(lsmode(mode),1,1)\n"
+     "      WHEN '-' THEN length(data)\n"
+     "      WHEN 'd' THEN 0\n"
+     "      ELSE -1 END,\n"
+     "    sqlar_compress(data)\n"
+     "  FROM fsdir(%Q,%Q) AS disk\n"
+     "  WHERE lsmode(mode) NOT LIKE '?%%'%s;"
+     ,
+     "REPLACE INTO %s(name,mode,mtime,data)\n"
+     "  SELECT\n"
+     "    %s,\n"
+     "    mode,\n"
+     "    mtime,\n"
+     "    data\n"
+     "  FROM fsdir(%Q,%Q) AS disk\n"
+     "  WHERE lsmode(mode) NOT LIKE '?%%'%s;"
+  };
+  int i;                          /* For iterating through azFile[] */
+  int rc;                         /* Return code */
+  const char *zTab = 0;           /* SQL table into which to insert */
+  char *zSql;
+  char zTemp[50];
+  char *zExists = 0;
+
+  arExecSql(pAr, "PRAGMA page_size=512");
+  rc = arExecSql(pAr, "SAVEPOINT ar;");
+  if( rc!=SQLITE_OK ) return rc;
+  zTemp[0] = 0;
+  if( pAr->bZip ){
+    /* Initialize the zipfile virtual table, if necessary */
+    if( pAr->zFile ){
+      sqlite3_uint64 r;
+      sqlite3_randomness(sizeof(r),&r);
+      sqlite3_snprintf(sizeof(zTemp),zTemp,"zip%016llx",r);
+      zTab = zTemp;
+      zSql = sqlite3_mprintf(
+         "CREATE VIRTUAL TABLE temp.%s USING zipfile(%Q)",
+         zTab, pAr->zFile
+      );
+      rc = arExecSql(pAr, zSql);
+      sqlite3_free(zSql);
+    }else{
+      zTab = "zip";
+    }
+  }else{
+    /* Initialize the table for an SQLAR */
+    zTab = "sqlar";
+    if( bUpdate==0 ){
+      rc = arExecSql(pAr, zDrop);
+      if( rc!=SQLITE_OK ) goto end_ar_transaction;
+    }
+    rc = arExecSql(pAr, zCreate);
+  }
+  if( bOnlyIfChanged ){
+    zExists = sqlite3_mprintf(
+      " AND NOT EXISTS("
+          "SELECT 1 FROM %s AS mem"
+          " WHERE mem.name=disk.name"
+          " AND mem.mtime=disk.mtime"
+          " AND mem.mode=disk.mode)", zTab);
+  }else{
+    zExists = sqlite3_mprintf("");
+  }
+  if( zExists==0 ) rc = SQLITE_NOMEM;
+  for(i=0; inArg && rc==SQLITE_OK; i++){
+    char *zSql2 = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab,
+        pAr->bVerbose ? "shell_putsnl(name)" : "name",
+        pAr->azArg[i], pAr->zDir, zExists);
+    rc = arExecSql(pAr, zSql2);
+    sqlite3_free(zSql2);
+  }
+end_ar_transaction:
+  if( rc!=SQLITE_OK ){
+    sqlite3_exec(pAr->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
+  }else{
+    rc = arExecSql(pAr, "RELEASE ar;");
+    if( pAr->bZip && pAr->zFile ){
+      zSql = sqlite3_mprintf("DROP TABLE %s", zTemp);
+      arExecSql(pAr, zSql);
+      sqlite3_free(zSql);
+    }
+  }
+  sqlite3_free(zExists);
+  return rc;
+}
+
+/*
+** Implementation of ".ar" dot command.
+*/
+static int arDotCommand(
+  ShellState *pState,          /* Current shell tool state */
+  int fromCmdLine,             /* True if -A command-line option, not .ar cmd */
+  char **azArg,                /* Array of arguments passed to dot command */
+  int nArg                     /* Number of entries in azArg[] */
+){
+  ArCommand cmd;
+  int rc;
+  memset(&cmd, 0, sizeof(cmd));
+  cmd.fromCmdLine = fromCmdLine;
+  rc = arParseCommand(azArg, nArg, &cmd);
+  if( rc==SQLITE_OK ){
+    int eDbType = SHELL_OPEN_UNSPEC;
+    cmd.p = pState;
+    cmd.out = pState->out;
+    cmd.db = pState->db;
+    if( cmd.zFile ){
+      eDbType = deduceDatabaseType(cmd.zFile, 1);
+    }else{
+      eDbType = pState->openMode;
+    }
+    if( eDbType==SHELL_OPEN_ZIPFILE ){
+      if( cmd.eCmd==AR_CMD_EXTRACT || cmd.eCmd==AR_CMD_LIST ){
+        if( cmd.zFile==0 ){
+          cmd.zSrcTable = sqlite3_mprintf("zip");
+        }else{
+          cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile);
+        }
+      }
+      cmd.bZip = 1;
+    }else if( cmd.zFile ){
+      int flags;
+      if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS;
+      if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT
+           || cmd.eCmd==AR_CMD_REMOVE || cmd.eCmd==AR_CMD_UPDATE ){
+        flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE;
+      }else{
+        flags = SQLITE_OPEN_READONLY;
+      }
+      cmd.db = 0;
+      if( cmd.bDryRun ){
+        sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile,
+              eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : "");
+      }
+      rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags,
+             eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0);
+      if( rc!=SQLITE_OK ){
+        sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n",
+                        cmd.zFile, sqlite3_errmsg(cmd.db));
+        goto end_ar_command;
+      }
+      sqlite3_fileio_init(cmd.db, 0, 0);
+      sqlite3_sqlar_init(cmd.db, 0, 0);
+      sqlite3_create_function(cmd.db, "shell_putsnl", 1, SQLITE_UTF8, cmd.p,
+                              shellPutsFunc, 0, 0);
+
+    }
+    if( cmd.zSrcTable==0 && cmd.bZip==0 && cmd.eCmd!=AR_CMD_HELP ){
+      if( cmd.eCmd!=AR_CMD_CREATE
+       && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0)
+      ){
+        sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n");
+        rc = SQLITE_ERROR;
+        goto end_ar_command;
+      }
+      cmd.zSrcTable = sqlite3_mprintf("sqlar");
+    }
+
+    switch( cmd.eCmd ){
+      case AR_CMD_CREATE:
+        rc = arCreateOrUpdateCommand(&cmd, 0, 0);
+        break;
+
+      case AR_CMD_EXTRACT:
+        rc = arExtractCommand(&cmd);
+        break;
+
+      case AR_CMD_LIST:
+        rc = arListCommand(&cmd);
+        break;
+
+      case AR_CMD_HELP:
+        arUsage(pState->out);
+        break;
+
+      case AR_CMD_INSERT:
+        rc = arCreateOrUpdateCommand(&cmd, 1, 0);
+        break;
+
+      case AR_CMD_REMOVE:
+        rc = arRemoveCommand(&cmd);
+        break;
+
+      default:
+        assert( cmd.eCmd==AR_CMD_UPDATE );
+        rc = arCreateOrUpdateCommand(&cmd, 1, 1);
+        break;
+    }
+  }
+end_ar_command:
+  if( cmd.db!=pState->db ){
+    close_db(cmd.db);
+  }
+  sqlite3_free(cmd.zSrcTable);
+
+  return rc;
+}
+/* End of the ".archive" or ".ar" command logic
+*******************************************************************************/
+#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */
+
+#if SQLITE_SHELL_HAVE_RECOVER
+
+/*
+** This function is used as a callback by the recover extension. Simply
+** print the supplied SQL statement to stdout.
+*/
+static int recoverSqlCb(void *pCtx, const char *zSql){
+  ShellState *pState = (ShellState*)pCtx;
+  sqlite3_fprintf(pState->out, "%s;\n", zSql);
+  return SQLITE_OK;
+}
+
+/*
+** This function is called to recover data from the database. A script
+** to construct a new database containing all recovered data is output
+** on stream pState->out.
+*/
+static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
+  int rc = SQLITE_OK;
+  const char *zRecoveryDb = "";   /* Name of "recovery" database.  Debug only */
+  const char *zLAF = "lost_and_found";
+  int bFreelist = 1;              /* 0 if --ignore-freelist is specified */
+  int bRowids = 1;                /* 0 if --no-rowids */
+  sqlite3_recover *p = 0;
+  int i = 0;
+
+  for(i=1; iout, azArg[0]);
+      return 1;
+    }
+  }
+
+  p = sqlite3_recover_init_sql(
+      pState->db, "main", recoverSqlCb, (void*)pState
+  );
+
+  sqlite3_recover_config(p, 789, (void*)zRecoveryDb);  /* Debug use only */
+  sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF);
+  sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids);
+  sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist);
+
+  sqlite3_recover_run(p);
+  if( sqlite3_recover_errcode(p)!=SQLITE_OK ){
+    const char *zErr = sqlite3_recover_errmsg(p);
+    int errCode = sqlite3_recover_errcode(p);
+    sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode);
+  }
+  rc = sqlite3_recover_finish(p);
+  return rc;
+}
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
+
+/*
+** Implementation of ".intck STEPS_PER_UNLOCK" command.
+*/
+static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){
+  sqlite3_intck *p = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_intck_open(pState->db, "main", &p);
+  if( rc==SQLITE_OK ){
+    i64 nStep = 0;
+    i64 nError = 0;
+    const char *zErr = 0;
+    while( SQLITE_OK==sqlite3_intck_step(p) ){
+      const char *zMsg = sqlite3_intck_message(p);
+      if( zMsg ){
+        sqlite3_fprintf(pState->out, "%s\n", zMsg);
+        nError++;
+      }
+      nStep++;
+      if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){
+        sqlite3_intck_unlock(p);
+      }
+    }
+    rc = sqlite3_intck_error(p, &zErr);
+    if( zErr ){
+      sqlite3_fprintf(stderr,"%s\n", zErr);
+    }
+    sqlite3_intck_close(p);
+
+    sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError);
+  }
+
+  return rc;
+}
+
+/*
+ * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it.
+ * zAutoColumn(0, &db, ?) => (db!=0) Form columns spec for CREATE TABLE,
+ *   close db and set it to 0, and return the columns spec, to later
+ *   be sqlite3_free()'ed by the caller.
+ * The return is 0 when either:
+ *   (a) The db was not initialized and zCol==0 (There are no columns.)
+ *   (b) zCol!=0  (Column was added, db initialized as needed.)
+ * The 3rd argument, pRenamed, references an out parameter. If the
+ * pointer is non-zero, its referent will be set to a summary of renames
+ * done if renaming was necessary, or set to 0 if none was done. The out
+ * string (if any) must be sqlite3_free()'ed by the caller.
+ */
+#ifdef SHELL_DEBUG
+#define rc_err_oom_die(rc) \
+  if( rc==SQLITE_NOMEM ) shell_check_oom(0); \
+  else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \
+    sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0)
+#else
+static void rc_err_oom_die(int rc){
+  if( rc==SQLITE_NOMEM ) shell_check_oom(0);
+  assert(rc==SQLITE_OK||rc==SQLITE_DONE);
+}
+#endif
+
+#ifdef SHELL_COLFIX_DB /* If this is set, the DB can be in a file. */
+static char zCOL_DB[] = SHELL_STRINGIFY(SHELL_COLFIX_DB);
+#else  /* Otherwise, memory is faster/better for the transient DB. */
+static const char *zCOL_DB = ":memory:";
+#endif
+
+/* Define character (as C string) to separate generated column ordinal
+ * from protected part of incoming column names. This defaults to "_"
+ * so that incoming column identifiers that did not need not be quoted
+ * remain usable without being quoted. It must be one character.
+ */
+#ifndef SHELL_AUTOCOLUMN_SEP
+# define AUTOCOLUMN_SEP "_"
+#else
+# define AUTOCOLUMN_SEP SHELL_STRINGIFY(SHELL_AUTOCOLUMN_SEP)
+#endif
+
+static char *zAutoColumn(const char *zColNew, sqlite3 **pDb, char **pzRenamed){
+  /* Queries and D{D,M}L used here */
+  static const char * const zTabMake = "\
+CREATE TABLE ColNames(\
+ cpos INTEGER PRIMARY KEY,\
+ name TEXT, nlen INT, chop INT, reps INT, suff TEXT);\
+CREATE VIEW RepeatedNames AS \
+SELECT DISTINCT t.name FROM ColNames t \
+WHERE t.name COLLATE NOCASE IN (\
+ SELECT o.name FROM ColNames o WHERE o.cpos<>t.cpos\
+);\
+";
+  static const char * const zTabFill = "\
+INSERT INTO ColNames(name,nlen,chop,reps,suff)\
+ VALUES(iif(length(?1)>0,?1,'?'),max(length(?1),1),0,0,'')\
+";
+  static const char * const zHasDupes = "\
+SELECT count(DISTINCT (substring(name,1,nlen-chop)||suff) COLLATE NOCASE)\
+ 1, printf('%c%0*d', '"AUTOCOLUMN_SEP"', $1, cpos), '')"
+#else /* ...RENAME_MINIMAL_ONE_PASS */
+"WITH Lzn(nlz) AS (" /* Find minimum extraneous leading 0's for uniqueness */
+"  SELECT 0 AS nlz"
+"  UNION"
+"  SELECT nlz+1 AS nlz FROM Lzn"
+"  WHERE EXISTS("
+"   SELECT 1"
+"   FROM ColNames t, ColNames o"
+"   WHERE"
+"    iif(t.name IN (SELECT * FROM RepeatedNames),"
+"     printf('%s"AUTOCOLUMN_SEP"%s',"
+"      t.name, substring(printf('%.*c%0.*d',nlz+1,'0',$1,t.cpos),2)),"
+"     t.name"
+"    )"
+"    ="
+"    iif(o.name IN (SELECT * FROM RepeatedNames),"
+"     printf('%s"AUTOCOLUMN_SEP"%s',"
+"      o.name, substring(printf('%.*c%0.*d',nlz+1,'0',$1,o.cpos),2)),"
+"     o.name"
+"    )"
+"    COLLATE NOCASE"
+"    AND o.cpos<>t.cpos"
+"   GROUP BY t.cpos"
+"  )"
+") UPDATE Colnames AS t SET"
+" chop = 0," /* No chopping, never touch incoming names. */
+" suff = iif(name IN (SELECT * FROM RepeatedNames),"
+"  printf('"AUTOCOLUMN_SEP"%s', substring("
+"   printf('%.*c%0.*d',(SELECT max(nlz) FROM Lzn)+1,'0',1,t.cpos),2)),"
+"  ''"
+" )"
+#endif
+    ;
+  static const char * const zCollectVar = "\
+SELECT\
+ '('||x'0a'\
+ || group_concat(\
+  cname||' TEXT',\
+  ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\
+ ||')' AS ColsSpec \
+FROM (\
+ SELECT cpos, printf('\"%w\"',printf('%!.*s%s', nlen-chop,name,suff)) AS cname \
+ FROM ColNames ORDER BY cpos\
+)";
+  static const char * const zRenamesDone =
+    "SELECT group_concat("
+    " printf('\"%w\" to \"%w\"',name,printf('%!.*s%s', nlen-chop, name, suff)),"
+    " ','||x'0a')"
+    "FROM ColNames WHERE suff<>'' OR chop!=0"
+    ;
+  int rc;
+  sqlite3_stmt *pStmt = 0;
+  assert(pDb!=0);
+  if( zColNew ){
+    /* Add initial or additional column. Init db if necessary. */
+    if( *pDb==0 ){
+      if( SQLITE_OK!=sqlite3_open(zCOL_DB, pDb) ) return 0;
+#ifdef SHELL_COLFIX_DB
+      if(*zCOL_DB!=':')
+        sqlite3_exec(*pDb,"drop table if exists ColNames;"
+                     "drop view if exists RepeatedNames;",0,0,0);
+#endif
+#undef SHELL_COLFIX_DB
+      rc = sqlite3_exec(*pDb, zTabMake, 0, 0, 0);
+      rc_err_oom_die(rc);
+    }
+    assert(*pDb!=0);
+    rc = sqlite3_prepare_v2(*pDb, zTabFill, -1, &pStmt, 0);
+    rc_err_oom_die(rc);
+    rc = sqlite3_bind_text(pStmt, 1, zColNew, -1, 0);
+    rc_err_oom_die(rc);
+    rc = sqlite3_step(pStmt);
+    rc_err_oom_die(rc);
+    sqlite3_finalize(pStmt);
+    return 0;
+  }else if( *pDb==0 ){
+    return 0;
+  }else{
+    /* Formulate the columns spec, close the DB, zero *pDb. */
+    char *zColsSpec = 0;
+    int hasDupes = db_int(*pDb, zHasDupes);
+    int nDigits = (hasDupes)? db_int(*pDb, zColDigits) : 0;
+    if( hasDupes ){
+#ifdef SHELL_COLUMN_RENAME_CLEAN
+      rc = sqlite3_exec(*pDb, zDedoctor, 0, 0, 0);
+      rc_err_oom_die(rc);
+#endif
+      rc = sqlite3_exec(*pDb, zSetReps, 0, 0, 0);
+      rc_err_oom_die(rc);
+      rc = sqlite3_prepare_v2(*pDb, zRenameRank, -1, &pStmt, 0);
+      rc_err_oom_die(rc);
+      sqlite3_bind_int(pStmt, 1, nDigits);
+      rc = sqlite3_step(pStmt);
+      sqlite3_finalize(pStmt);
+      if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM);
+    }
+    assert(db_int(*pDb, zHasDupes)==0); /* Consider: remove this */
+    rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0);
+    rc_err_oom_die(rc);
+    rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      zColsSpec = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
+    }else{
+      zColsSpec = 0;
+    }
+    if( pzRenamed!=0 ){
+      if( !hasDupes ) *pzRenamed = 0;
+      else{
+        sqlite3_finalize(pStmt);
+        if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0)
+            && SQLITE_ROW==sqlite3_step(pStmt) ){
+          *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
+        }else
+          *pzRenamed = 0;
+      }
+    }
+    sqlite3_finalize(pStmt);
+    sqlite3_close(*pDb);
+    *pDb = 0;
+    return zColsSpec;
+  }
+}
+
+/*
+** Check if the sqlite_schema table contains one or more virtual tables. If
+** parameter zLike is not NULL, then it is an SQL expression that the
+** sqlite_schema row must also match. If one or more such rows are found,
+** print the following warning to the output:
+**
+** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled
+*/
+static int outputDumpWarning(ShellState *p, const char *zLike){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt = 0;
+  shellPreparePrintf(p->db, &rc, &pStmt,
+    "SELECT 1 FROM sqlite_schema o WHERE "
+    "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true"
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+    sqlite3_fputs("/* WARNING: "
+          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n",
+          p->out
+    );
+  }
+  shellFinalize(&rc, pStmt);
+  return rc;
+}
+
+/*
+** Fault-Simulator state and logic.
+*/
+static struct {
+  int iId;           /* ID that triggers a simulated fault.  -1 means "any" */
+  int iErr;          /* The error code to return on a fault */
+  int iCnt;          /* Trigger the fault only if iCnt is already zero */
+  int iInterval;     /* Reset iCnt to this value after each fault */
+  int eVerbose;      /* When to print output */
+  int nHit;          /* Number of hits seen so far */
+  int nRepeat;       /* Turn off after this many hits.  0 for never */
+  int nSkip;         /* Skip this many before first fault */
+} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+** This is the fault-sim callback
+*/
+static int faultsim_callback(int iArg){
+  if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){
+    return SQLITE_OK;
+  }
+  if( faultsim_state.iCnt ){
+    if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--;
+    if( faultsim_state.eVerbose>=2 ){
+      sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt);
+    }
+    return SQLITE_OK;
+  }
+  if( faultsim_state.eVerbose>=1 ){
+    sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr);
+  }
+  faultsim_state.iCnt = faultsim_state.iInterval;
+  faultsim_state.nHit++;
+  if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){
+    faultsim_state.iCnt = -1;
+  }
+  return faultsim_state.iErr;
+}
+
+/*
+** If an input line begins with "." then invoke this routine to
+** process that line.
+**
+** Return 1 on error, 2 to exit, and 0 otherwise.
+*/
+static int do_meta_command(char *zLine, ShellState *p){
+  int h = 1;
+  int nArg = 0;
+  int n, c;
+  int rc = 0;
+  char *azArg[52];
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( p->expert.pExpert ){
+    expertFinish(p, 1, 0);
+  }
+#endif
+
+  /* Parse the input line into tokens.
+  */
+  while( zLine[h] && nArgdb, shellAuth, p);
+    }else if( p->bSafeModePersist ){
+      sqlite3_set_authorizer(p->db, safeModeAuth, p);
+    }else{
+      sqlite3_set_authorizer(p->db, 0, 0);
+    }
+  }else
+#endif
+
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \
+  && !defined(SQLITE_SHELL_FIDDLE)
+  if( c=='a' && cli_strncmp(azArg[0], "archive", n)==0 ){
+    open_db(p, 0);
+    failIfSafeMode(p, "cannot run .archive in safe mode");
+    rc = arDotCommand(p, 0, azArg, nArg);
+  }else
+#endif
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( (c=='b' && n>=3 && cli_strncmp(azArg[0], "backup", n)==0)
+   || (c=='s' && n>=3 && cli_strncmp(azArg[0], "save", n)==0)
+  ){
+    const char *zDestFile = 0;
+    const char *zDb = 0;
+    sqlite3 *pDest;
+    sqlite3_backup *pBackup;
+    int j;
+    int bAsync = 0;
+    const char *zVfs = 0;
+    failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
+    for(j=1; jdb, zDb);
+    if( pBackup==0 ){
+      shellDatabaseError(pDest);
+      close_db(pDest);
+      return 1;
+    }
+    while(  (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK ){}
+    sqlite3_backup_finish(pBackup);
+    if( rc==SQLITE_DONE ){
+      rc = 0;
+    }else{
+      shellDatabaseError(pDest);
+      rc = 1;
+    }
+    close_db(pDest);
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='b' && n>=3 && cli_strncmp(azArg[0], "bail", n)==0 ){
+    if( nArg==2 ){
+      bail_on_error = booleanValue(azArg[1]);
+    }else{
+      eputz("Usage: .bail on|off\n");
+      rc = 1;
+    }
+  }else
+
+  /* Undocumented.  Legacy only.  See "crlf" below */
+  if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){
+    eputz("The \".binary\" command is deprecated.\n");
+    rc = 1;
+  }else
+
+  /* The undocumented ".breakpoint" command causes a call to the no-op
+  ** routine named test_breakpoint().
+  */
+  if( c=='b' && n>=3 && cli_strncmp(azArg[0], "breakpoint", n)==0 ){
+    test_breakpoint();
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='c' && cli_strcmp(azArg[0],"cd")==0 ){
+    failIfSafeMode(p, "cannot run .cd in safe mode");
+    if( nArg==2 ){
+#if defined(_WIN32) || defined(WIN32)
+      wchar_t *z = sqlite3_win32_utf8_to_unicode(azArg[1]);
+      rc = !SetCurrentDirectoryW(z);
+      sqlite3_free(z);
+#else
+      rc = chdir(azArg[1]);
+#endif
+      if( rc ){
+        sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]);
+        rc = 1;
+      }
+    }else{
+      eputz("Usage: .cd DIRECTORY\n");
+      rc = 1;
+    }
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='c' && n>=3 && cli_strncmp(azArg[0], "changes", n)==0 ){
+    if( nArg==2 ){
+      setOrClearFlag(p, SHFLG_CountChanges, azArg[1]);
+    }else{
+      eputz("Usage: .changes on|off\n");
+      rc = 1;
+    }
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  /* Cancel output redirection, if it is currently set (by .testcase)
+  ** Then read the content of the testcase-out.txt file and compare against
+  ** azArg[1].  If there are differences, report an error and exit.
+  */
+  if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){
+    char *zRes = 0;
+    output_reset(p);
+    if( nArg!=2 ){
+      eputz("Usage: .check GLOB-PATTERN\n");
+      rc = 2;
+    }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){
+      rc = 2;
+    }else if( testcase_glob(azArg[1],zRes)==0 ){
+      sqlite3_fprintf(stderr,
+            "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
+            p->zTestcase, azArg[1], zRes);
+      rc = 1;
+    }else{
+      sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase);
+      p->nCheck++;
+    }
+    sqlite3_free(zRes);
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){
+    failIfSafeMode(p, "cannot run .clone in safe mode");
+    if( nArg==2 ){
+      tryToClone(p, azArg[1]);
+    }else{
+      eputz("Usage: .clone FILENAME\n");
+      rc = 1;
+    }
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='c' && cli_strncmp(azArg[0], "connection", n)==0 ){
+    if( nArg==1 ){
+      /* List available connections */
+      int i;
+      for(i=0; iaAuxDb); i++){
+        const char *zFile = p->aAuxDb[i].zDbFilename;
+        if( p->aAuxDb[i].db==0 && p->pAuxDb!=&p->aAuxDb[i] ){
+          zFile = "(not open)";
+        }else if( zFile==0 ){
+          zFile = "(memory)";
+        }else if( zFile[0]==0 ){
+          zFile = "(temporary-file)";
+        }
+        if( p->pAuxDb == &p->aAuxDb[i] ){
+          sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile);
+        }else if( p->aAuxDb[i].db!=0 ){
+          sqlite3_fprintf(stdout, "       %d: %s\n", i, zFile);
+        }
+      }
+    }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){
+      int i = azArg[1][0] - '0';
+      if( p->pAuxDb != &p->aAuxDb[i] && i>=0 && iaAuxDb) ){
+        p->pAuxDb->db = p->db;
+        p->pAuxDb = &p->aAuxDb[i];
+        globalDb = p->db = p->pAuxDb->db;
+        p->pAuxDb->db = 0;
+      }
+    }else if( nArg==3 && cli_strcmp(azArg[1], "close")==0
+           && IsDigit(azArg[2][0]) && azArg[2][1]==0 ){
+      int i = azArg[2][0] - '0';
+      if( i<0 || i>=ArraySize(p->aAuxDb) ){
+        /* No-op */
+      }else if( p->pAuxDb == &p->aAuxDb[i] ){
+        eputz("cannot close the active database connection\n");
+        rc = 1;
+      }else if( p->aAuxDb[i].db ){
+        session_close_all(p, i);
+        close_db(p->aAuxDb[i].db);
+        p->aAuxDb[i].db = 0;
+      }
+    }else{
+      eputz("Usage: .connection [close] [CONNECTION-NUMBER]\n");
+      rc = 1;
+    }
+  }else
+
+  if( c=='c' && n==4
+   && (cli_strncmp(azArg[0], "crlf", n)==0
+       || cli_strncmp(azArg[0], "crnl",n)==0)
+  ){
+    if( nArg==2 ){
+#ifdef _WIN32
+      p->crlfMode = booleanValue(azArg[1]);
+#else
+      p->crlfMode = 0;
+#endif
+    }
+    sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF");
+  }else
+
+  if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){
+    char **azName = 0;
+    int nName = 0;
+    sqlite3_stmt *pStmt;
+    int i;
+    open_db(p, 0);
+    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
+    if( rc ){
+      shellDatabaseError(p->db);
+      rc = 1;
+    }else{
+      while( sqlite3_step(pStmt)==SQLITE_ROW ){
+        const char *zSchema = (const char *)sqlite3_column_text(pStmt,1);
+        const char *zFile = (const char*)sqlite3_column_text(pStmt,2);
+        if( zSchema==0 || zFile==0 ) continue;
+        azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*));
+        shell_check_oom(azName);
+        azName[nName*2] = strdup(zSchema);
+        azName[nName*2+1] = strdup(zFile);
+        nName++;
+      }
+    }
+    sqlite3_finalize(pStmt);
+    for(i=0; idb, azName[i*2]);
+      int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]);
+      const char *z = azName[i*2+1];
+      sqlite3_fprintf(p->out, "%s: %s %s%s\n",
+            azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w",
+            eTxn==SQLITE_TXN_NONE ? "" :
+            eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn");
+      free(azName[i*2]);
+      free(azName[i*2+1]);
+    }
+    sqlite3_free(azName);
+  }else
+
+  if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbconfig", n)==0 ){
+    static const struct DbConfigChoices {
+      const char *zName;
+      int op;
+    } aDbConfig[] = {
+        { "defensive",          SQLITE_DBCONFIG_DEFENSIVE             },
+        { "dqs_ddl",            SQLITE_DBCONFIG_DQS_DDL               },
+        { "dqs_dml",            SQLITE_DBCONFIG_DQS_DML               },
+        { "enable_fkey",        SQLITE_DBCONFIG_ENABLE_FKEY           },
+        { "enable_qpsg",        SQLITE_DBCONFIG_ENABLE_QPSG           },
+        { "enable_trigger",     SQLITE_DBCONFIG_ENABLE_TRIGGER        },
+        { "enable_view",        SQLITE_DBCONFIG_ENABLE_VIEW           },
+        { "fts3_tokenizer",     SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER },
+        { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE    },
+        { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    },
+        { "load_extension",     SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION },
+        { "no_ckpt_on_close",   SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      },
+        { "reset_database",     SQLITE_DBCONFIG_RESET_DATABASE        },
+        { "reverse_scanorder",  SQLITE_DBCONFIG_REVERSE_SCANORDER     },
+        { "stmt_scanstatus",    SQLITE_DBCONFIG_STMT_SCANSTATUS       },
+        { "trigger_eqp",        SQLITE_DBCONFIG_TRIGGER_EQP           },
+        { "trusted_schema",     SQLITE_DBCONFIG_TRUSTED_SCHEMA        },
+        { "writable_schema",    SQLITE_DBCONFIG_WRITABLE_SCHEMA       },
+    };
+    int ii, v;
+    open_db(p, 0);
+    for(ii=0; ii1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue;
+      if( nArg>=3 ){
+        sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0);
+      }
+      sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v);
+      sqlite3_fprintf(p->out, "%19s %s\n",
+                      aDbConfig[ii].zName, v ? "on" : "off");
+      if( nArg>1 ) break;
+    }
+    if( nArg>1 && ii==ArraySize(aDbConfig) ){
+      sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]);
+      eputz("Enter \".dbconfig\" with no arguments for a list\n");
+    }
+  }else
+
+#if SQLITE_SHELL_HAVE_RECOVER
+  if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){
+    rc = shell_dbinfo_command(p, nArg, azArg);
+  }else
+
+  if( c=='r' && cli_strncmp(azArg[0], "recover", n)==0 ){
+    open_db(p, 0);
+    rc = recoverDatabaseCmd(p, nArg, azArg);
+  }else
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
+
+  if( c=='d' && cli_strncmp(azArg[0], "dump", n)==0 ){
+    char *zLike = 0;
+    char *zSql;
+    int i;
+    int savedShowHeader = p->showHeader;
+    int savedShellFlags = p->shellFlgs;
+    ShellClearFlag(p,
+       SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo
+       |SHFLG_DumpDataOnly|SHFLG_DumpNoSys);
+    for(i=1; ishellFlgs & SHFLG_DumpDataOnly)==0 ){
+      /* When playing back a "dump", the content might appear in an order
+      ** which causes immediate foreign key constraints to be violated.
+      ** So disable foreign-key constraint enforcement to prevent problems. */
+      sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out);
+      sqlite3_fputs("BEGIN TRANSACTION;\n", p->out);
+    }
+    p->writableSchema = 0;
+    p->showHeader = 0;
+    /* Set writable_schema=ON since doing so forces SQLite to initialize
+    ** as much of the schema as it can even if the sqlite_schema table is
+    ** corrupt. */
+    sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0);
+    p->nErr = 0;
+    if( zLike==0 ) zLike = sqlite3_mprintf("true");
+    zSql = sqlite3_mprintf(
+      "SELECT name, type, sql FROM sqlite_schema AS o "
+      "WHERE (%s) AND type=='table'"
+      "  AND sql NOT NULL"
+      " ORDER BY tbl_name='sqlite_sequence', rowid",
+      zLike
+    );
+    run_schema_dump_query(p,zSql);
+    sqlite3_free(zSql);
+    if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
+      zSql = sqlite3_mprintf(
+        "SELECT sql FROM sqlite_schema AS o "
+        "WHERE (%s) AND sql NOT NULL"
+        "  AND type IN ('index','trigger','view') "
+        "ORDER BY type COLLATE NOCASE DESC",
+        zLike
+      );
+      run_table_dump_query(p, zSql);
+      sqlite3_free(zSql);
+    }
+    sqlite3_free(zLike);
+    if( p->writableSchema ){
+      sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out);
+      p->writableSchema = 0;
+    }
+    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
+    sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
+    if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
+      sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out);
+    }
+    p->showHeader = savedShowHeader;
+    p->shellFlgs = savedShellFlags;
+  }else
+
+  if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){
+    if( nArg==2 ){
+      setOrClearFlag(p, SHFLG_Echo, azArg[1]);
+    }else{
+      eputz("Usage: .echo on|off\n");
+      rc = 1;
+    }
+  }else
+
+  if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){
+    if( nArg==2 ){
+      p->autoEQPtest = 0;
+      if( p->autoEQPtrace ){
+        if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0);
+        p->autoEQPtrace = 0;
+      }
+      if( cli_strcmp(azArg[1],"full")==0 ){
+        p->autoEQP = AUTOEQP_full;
+      }else if( cli_strcmp(azArg[1],"trigger")==0 ){
+        p->autoEQP = AUTOEQP_trigger;
+#ifdef SQLITE_DEBUG
+      }else if( cli_strcmp(azArg[1],"test")==0 ){
+        p->autoEQP = AUTOEQP_on;
+        p->autoEQPtest = 1;
+      }else if( cli_strcmp(azArg[1],"trace")==0 ){
+        p->autoEQP = AUTOEQP_full;
+        p->autoEQPtrace = 1;
+        open_db(p, 0);
+        sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0);
+        sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0);
+#endif
+      }else{
+        p->autoEQP = (u8)booleanValue(azArg[1]);
+      }
+    }else{
+      eputz("Usage: .eqp off|on|trace|trigger|full\n");
+      rc = 1;
+    }
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){
+    if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc);
+    rc = 2;
+  }else
+#endif
+
+  /* The ".explain" command is automatic now.  It is largely pointless.  It
+  ** retained purely for backwards compatibility */
+  if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){
+    int val = 1;
+    if( nArg>=2 ){
+      if( cli_strcmp(azArg[1],"auto")==0 ){
+        val = 99;
+      }else{
+        val =  booleanValue(azArg[1]);
+      }
+    }
+    if( val==1 && p->mode!=MODE_Explain ){
+      p->normalMode = p->mode;
+      p->mode = MODE_Explain;
+      p->autoExplain = 0;
+    }else if( val==0 ){
+      if( p->mode==MODE_Explain ) p->mode = p->normalMode;
+      p->autoExplain = 0;
+    }else if( val==99 ){
+      if( p->mode==MODE_Explain ) p->mode = p->normalMode;
+      p->autoExplain = 1;
+    }
+  }else
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){
+    if( p->bSafeMode ){
+      sqlite3_fprintf(stderr,
+            "Cannot run experimental commands such as \"%s\" in safe mode\n",
+            azArg[0]);
+      rc = 1;
+    }else{
+      open_db(p, 0);
+      expertDotCommand(p, azArg, nArg);
+    }
+  }else
+#endif
+
+  if( c=='f' && cli_strncmp(azArg[0], "filectrl", n)==0 ){
+    static const struct {
+       const char *zCtrlName;   /* Name of a test-control option */
+       int ctrlCode;            /* Integer code for that option */
+       const char *zUsage;      /* Usage notes */
+    } aCtrl[] = {
+      { "chunk_size",     SQLITE_FCNTL_CHUNK_SIZE,      "SIZE"           },
+      { "data_version",   SQLITE_FCNTL_DATA_VERSION,    ""               },
+      { "has_moved",      SQLITE_FCNTL_HAS_MOVED,       ""               },
+      { "lock_timeout",   SQLITE_FCNTL_LOCK_TIMEOUT,    "MILLISEC"       },
+      { "persist_wal",    SQLITE_FCNTL_PERSIST_WAL,     "[BOOLEAN]"      },
+   /* { "pragma",         SQLITE_FCNTL_PRAGMA,          "NAME ARG"       },*/
+      { "psow",       SQLITE_FCNTL_POWERSAFE_OVERWRITE, "[BOOLEAN]"      },
+      { "reserve_bytes",  SQLITE_FCNTL_RESERVE_BYTES,   "[N]"            },
+      { "size_limit",     SQLITE_FCNTL_SIZE_LIMIT,      "[LIMIT]"        },
+      { "tempfilename",   SQLITE_FCNTL_TEMPFILENAME,    ""               },
+   /* { "win32_av_retry", SQLITE_FCNTL_WIN32_AV_RETRY,  "COUNT DELAY"    },*/
+    };
+    int filectrl = -1;
+    int iCtrl = -1;
+    sqlite3_int64 iRes = 0;  /* Integer result to display if rc2==1 */
+    int isOk = 0;            /* 0: usage  1: %lld  2: no-result */
+    int n2, i;
+    const char *zCmd = 0;
+    const char *zSchema = 0;
+
+    open_db(p, 0);
+    zCmd = nArg>=2 ? azArg[1] : "help";
+
+    if( zCmd[0]=='-'
+     && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0)
+     && nArg>=4
+    ){
+      zSchema = azArg[2];
+      for(i=3; iout);
+      for(i=0; iout,
+               "  .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage);
+      }
+      rc = 1;
+      goto meta_command_exit;
+    }
+
+    /* convert filectrl text option to value. allow any unique prefix
+    ** of the option name, or a numerical value. */
+    n2 = strlen30(zCmd);
+    for(i=0; idb, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes);
+          isOk = 1;
+          break;
+        }
+        case SQLITE_FCNTL_LOCK_TIMEOUT:
+        case SQLITE_FCNTL_CHUNK_SIZE: {
+          int x;
+          if( nArg!=3 ) break;
+          x = (int)integerValue(azArg[2]);
+          sqlite3_file_control(p->db, zSchema, filectrl, &x);
+          isOk = 2;
+          break;
+        }
+        case SQLITE_FCNTL_PERSIST_WAL:
+        case SQLITE_FCNTL_POWERSAFE_OVERWRITE: {
+          int x;
+          if( nArg!=2 && nArg!=3 ) break;
+          x = nArg==3 ? booleanValue(azArg[2]) : -1;
+          sqlite3_file_control(p->db, zSchema, filectrl, &x);
+          iRes = x;
+          isOk = 1;
+          break;
+        }
+        case SQLITE_FCNTL_DATA_VERSION:
+        case SQLITE_FCNTL_HAS_MOVED: {
+          int x;
+          if( nArg!=2 ) break;
+          sqlite3_file_control(p->db, zSchema, filectrl, &x);
+          iRes = x;
+          isOk = 1;
+          break;
+        }
+        case SQLITE_FCNTL_TEMPFILENAME: {
+          char *z = 0;
+          if( nArg!=2 ) break;
+          sqlite3_file_control(p->db, zSchema, filectrl, &z);
+          if( z ){
+            sqlite3_fprintf(p->out, "%s\n", z);
+            sqlite3_free(z);
+          }
+          isOk = 2;
+          break;
+        }
+        case SQLITE_FCNTL_RESERVE_BYTES: {
+          int x;
+          if( nArg>=3 ){
+            x = atoi(azArg[2]);
+            sqlite3_file_control(p->db, zSchema, filectrl, &x);
+          }
+          x = -1;
+          sqlite3_file_control(p->db, zSchema, filectrl, &x);
+          sqlite3_fprintf(p->out, "%d\n", x);
+          isOk = 2;
+          break;
+        }
+      }
+    }
+    if( isOk==0 && iCtrl>=0 ){
+      sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n",
+                      zCmd, aCtrl[iCtrl].zUsage);
+      rc = 1;
+    }else if( isOk==1 ){
+      char zBuf[100];
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes);
+      sqlite3_fprintf(p->out, "%s\n", zBuf);
+    }
+  }else
+
+  if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){
+    ShellState data;
+    int doStats = 0;
+    memcpy(&data, p, sizeof(data));
+    data.showHeader = 0;
+    data.cMode = data.mode = MODE_Semi;
+    if( nArg==2 && optionMatch(azArg[1], "indent") ){
+      data.cMode = data.mode = MODE_Pretty;
+      nArg = 1;
+    }
+    if( nArg!=1 ){
+      eputz("Usage: .fullschema ?--indent?\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    rc = sqlite3_exec(p->db,
+       "SELECT sql FROM"
+       "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
+       "     FROM sqlite_schema UNION ALL"
+       "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) "
+       "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' "
+       "ORDER BY x",
+       callback, &data, 0
+    );
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pStmt;
+      rc = sqlite3_prepare_v2(p->db,
+               "SELECT rowid FROM sqlite_schema"
+               " WHERE name GLOB 'sqlite_stat[134]'",
+               -1, &pStmt, 0);
+      if( rc==SQLITE_OK ){
+        doStats = sqlite3_step(pStmt)==SQLITE_ROW;
+        sqlite3_finalize(pStmt);
+      }
+    }
+    if( doStats==0 ){
+      sqlite3_fputs("/* No STAT tables available */\n", p->out);
+    }else{
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
+      data.cMode = data.mode = MODE_Insert;
+      data.zDestTable = "sqlite_stat1";
+      shell_exec(&data, "SELECT * FROM sqlite_stat1", 0);
+      data.zDestTable = "sqlite_stat4";
+      shell_exec(&data, "SELECT * FROM sqlite_stat4", 0);
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
+    }
+  }else
+
+  if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){
+    if( nArg==2 ){
+      p->showHeader = booleanValue(azArg[1]);
+      p->shellFlgs |= SHFLG_HeaderSet;
+    }else{
+      eputz("Usage: .headers on|off\n");
+      rc = 1;
+    }
+  }else
+
+  if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){
+    if( nArg>=2 ){
+      n = showHelp(p->out, azArg[1]);
+      if( n==0 ){
+        sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]);
+      }
+    }else{
+      showHelp(p->out, 0);
+    }
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){
+    char *zTable = 0;           /* Insert data into this table */
+    char *zSchema = 0;          /* Schema of zTable */
+    char *zFile = 0;            /* Name of file to extra content from */
+    sqlite3_stmt *pStmt = NULL; /* A statement */
+    int nCol;                   /* Number of columns in the table */
+    i64 nByte;                  /* Number of bytes in an SQL string */
+    int i, j;                   /* Loop counters */
+    int needCommit;             /* True to COMMIT or ROLLBACK at end */
+    int nSep;                   /* Number of bytes in p->colSeparator[] */
+    char *zSql = 0;             /* An SQL statement */
+    ImportCtx sCtx;             /* Reader context */
+    char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */
+    int eVerbose = 0;           /* Larger for more console output */
+    int nSkip = 0;              /* Initial lines to skip */
+    int useOutputMode = 1;      /* Use output mode to determine separators */
+    char *zCreate = 0;          /* CREATE TABLE statement text */
+
+    failIfSafeMode(p, "cannot run .import in safe mode");
+    memset(&sCtx, 0, sizeof(sCtx));
+    if( p->mode==MODE_Ascii ){
+      xRead = ascii_read_one_field;
+    }else{
+      xRead = csv_read_one_field;
+    }
+    rc = 1;
+    for(i=1; iout, "ERROR: extra argument: \"%s\". Usage:\n",z);
+          showHelp(p->out, "import");
+          goto meta_command_exit;
+        }
+      }else if( cli_strcmp(z,"-v")==0 ){
+        eVerbose++;
+      }else if( cli_strcmp(z,"-schema")==0 && iout, "ERROR: unknown option: \"%s\".  Usage:\n", z);
+        showHelp(p->out, "import");
+        goto meta_command_exit;
+      }
+    }
+    if( zTable==0 ){
+      sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n",
+            zFile==0 ? "FILE" : "TABLE");
+      showHelp(p->out, "import");
+      goto meta_command_exit;
+    }
+    seenInterrupt = 0;
+    open_db(p, 0);
+    if( useOutputMode ){
+      /* If neither the --csv or --ascii options are specified, then set
+      ** the column and row separator characters from the output mode. */
+      nSep = strlen30(p->colSeparator);
+      if( nSep==0 ){
+        eputz("Error: non-null column separator required for import\n");
+        goto meta_command_exit;
+      }
+      if( nSep>1 ){
+        eputz("Error: multi-character column separators not allowed"
+              " for import\n");
+        goto meta_command_exit;
+      }
+      nSep = strlen30(p->rowSeparator);
+      if( nSep==0 ){
+        eputz("Error: non-null row separator required for import\n");
+        goto meta_command_exit;
+      }
+      if( nSep==2 && p->mode==MODE_Csv
+       && cli_strcmp(p->rowSeparator,SEP_CrLf)==0
+      ){
+        /* When importing CSV (only), if the row separator is set to the
+        ** default output row separator, change it to the default input
+        ** row separator.  This avoids having to maintain different input
+        ** and output row separators. */
+        sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+        nSep = strlen30(p->rowSeparator);
+      }
+      if( nSep>1 ){
+        eputz("Error: multi-character row separators not allowed"
+              " for import\n");
+        goto meta_command_exit;
+      }
+      sCtx.cColSep = (u8)p->colSeparator[0];
+      sCtx.cRowSep = (u8)p->rowSeparator[0];
+    }
+    sCtx.zFile = zFile;
+    sCtx.nLine = 1;
+    if( sCtx.zFile[0]=='|' ){
+#ifdef SQLITE_OMIT_POPEN
+      eputz("Error: pipes are not supported in this OS\n");
+      goto meta_command_exit;
+#else
+      sCtx.in = sqlite3_popen(sCtx.zFile+1, "r");
+      sCtx.zFile = "";
+      sCtx.xCloser = pclose;
+#endif
+    }else{
+      sCtx.in = sqlite3_fopen(sCtx.zFile, "rb");
+      sCtx.xCloser = fclose;
+    }
+    if( sCtx.in==0 ){
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
+      goto meta_command_exit;
+    }
+    if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){
+      char zSep[2];
+      zSep[1] = 0;
+      zSep[0] = sCtx.cColSep;
+      sqlite3_fputs("Column separator ", p->out);
+      output_c_string(p->out, zSep);
+      sqlite3_fputs(", row separator ", p->out);
+      zSep[0] = sCtx.cRowSep;
+      output_c_string(p->out, zSep);
+      sqlite3_fputs("\n", p->out);
+    }
+    sCtx.z = sqlite3_malloc64(120);
+    if( sCtx.z==0 ){
+      import_cleanup(&sCtx);
+      shell_out_of_memory();
+    }
+    /* Below, resources must be freed before exit. */
+    while( (nSkip--)>0 ){
+      while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){}
+    }
+    import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
+    if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){
+      /* Table does not exist.  Create it. */
+      sqlite3 *dbCols = 0;
+      char *zRenames = 0;
+      char *zColDefs;
+      zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", 
+                    zSchema ? zSchema : "main", zTable);
+      while( xRead(&sCtx) ){
+        zAutoColumn(sCtx.z, &dbCols, 0);
+        if( sCtx.cTerm!=sCtx.cColSep ) break;
+      }
+      zColDefs = zAutoColumn(0, &dbCols, &zRenames);
+      if( zRenames!=0 ){
+        sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr,
+              "Columns renamed during .import %s due to duplicates:\n"
+              "%s\n", sCtx.zFile, zRenames);
+        sqlite3_free(zRenames);
+      }
+      assert(dbCols==0);
+      if( zColDefs==0 ){
+        sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile);
+        import_cleanup(&sCtx);
+        rc = 1;
+        sqlite3_free(zCreate);
+        goto meta_command_exit;
+      }
+      zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs);
+      if( zCreate==0 ){
+        import_cleanup(&sCtx);
+        shell_out_of_memory();
+      }
+      if( eVerbose>=1 ){
+        sqlite3_fprintf(p->out, "%s\n", zCreate);
+      }
+      rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
+      if( rc ){
+        sqlite3_fprintf(stderr,
+             "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db));
+      }
+      sqlite3_free(zCreate);
+      zCreate = 0;
+      if( rc ){
+        import_cleanup(&sCtx);
+        rc = 1;
+        goto meta_command_exit;
+      }
+    }
+    zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);",
+                           zTable, zSchema);
+    if( zSql==0 ){
+      import_cleanup(&sCtx);
+      shell_out_of_memory();
+    }
+    rc =  sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    zSql = 0;
+    if( rc ){
+      if (pStmt) sqlite3_finalize(pStmt);
+      shellDatabaseError(p->db);
+      import_cleanup(&sCtx);
+      rc = 1;
+      goto meta_command_exit;
+    }
+    if( sqlite3_step(pStmt)==SQLITE_ROW ){
+      nCol = sqlite3_column_int(pStmt, 0);
+    }else{
+      nCol = 0;
+    }
+    sqlite3_finalize(pStmt);
+    pStmt = 0;
+    if( nCol==0 ) return 0; /* no columns, no error */
+
+    nByte = 64                 /* space for "INSERT INTO", "VALUES(", ")\0" */
+          + (zSchema ? strlen(zSchema)*2 + 2: 0)  /* Quoted schema name */
+          + strlen(zTable)*2 + 2                  /* Quoted table name */
+          + nCol*2;            /* Space for ",?" for each column */
+    zSql = sqlite3_malloc64( nByte );
+    if( zSql==0 ){
+      import_cleanup(&sCtx);
+      shell_out_of_memory();
+    }
+    if( zSchema ){
+      sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", 
+                       zSchema, zTable);
+    }else{
+      sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
+    }
+    j = strlen30(zSql);
+    for(i=1; i=2 ){
+      sqlite3_fprintf(p->out, "Insert using: %s\n", zSql);
+    }
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    zSql = 0;
+    if( rc ){
+      shellDatabaseError(p->db);
+      if (pStmt) sqlite3_finalize(pStmt);
+      import_cleanup(&sCtx);
+      rc = 1;
+      goto meta_command_exit;
+    }
+    needCommit = sqlite3_get_autocommit(p->db);
+    if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
+    do{
+      int startLine = sCtx.nLine;
+      for(i=0; imode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break;
+        /*
+        ** For CSV mode, per RFC 4180, accept EOF in lieu of final
+        ** record terminator but only for last field of multi-field row.
+        ** (If there are too few fields, it's not valid CSV anyway.)
+        */
+        if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){
+          z = "";
+        }
+        sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
+        if( i=nCol ){
+        sqlite3_step(pStmt);
+        rc = sqlite3_reset(pStmt);
+        if( rc!=SQLITE_OK ){
+          sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n",
+                sCtx.zFile, startLine, sqlite3_errmsg(p->db));
+          sCtx.nErr++;
+        }else{
+          sCtx.nRow++;
+        }
+      }
+    }while( sCtx.cTerm!=EOF );
+
+    import_cleanup(&sCtx);
+    sqlite3_finalize(pStmt);
+    if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
+    if( eVerbose>0 ){
+      sqlite3_fprintf(p->out,
+            "Added %d rows with %d errors using %d lines of input\n",
+            sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
+    }
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+#ifndef SQLITE_UNTESTABLE
+  if( c=='i' && cli_strncmp(azArg[0], "imposter", n)==0 ){
+    char *zSql;
+    char *zCollist = 0;
+    sqlite3_stmt *pStmt;
+    int tnum = 0;
+    int isWO = 0;  /* True if making an imposter of a WITHOUT ROWID table */
+    int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */
+    int i;
+    if( !ShellHasFlag(p,SHFLG_TestingMode) ){
+      sqlite3_fprintf(stderr,".%s unavailable without --unsafe-testing\n",
+            "imposter");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
+      eputz("Usage: .imposter INDEX IMPOSTER\n"
+            "       .imposter off\n");
+      /* Also allowed, but not documented:
+      **
+      **    .imposter TABLE IMPOSTER
+      **
+      ** where TABLE is a WITHOUT ROWID table.  In that case, the
+      ** imposter is another WITHOUT ROWID table with the columns in
+      ** storage order. */
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    if( nArg==2 ){
+      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1);
+      goto meta_command_exit;
+    }
+    zSql = sqlite3_mprintf(
+      "SELECT rootpage, 0 FROM sqlite_schema"
+      " WHERE name='%q' AND type='index'"
+      "UNION ALL "
+      "SELECT rootpage, 1 FROM sqlite_schema"
+      " WHERE name='%q' AND type='table'"
+      "   AND sql LIKE '%%without%%rowid%%'",
+      azArg[1], azArg[1]
+    );
+    sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    if( sqlite3_step(pStmt)==SQLITE_ROW ){
+      tnum = sqlite3_column_int(pStmt, 0);
+      isWO = sqlite3_column_int(pStmt, 1);
+    }
+    sqlite3_finalize(pStmt);
+    zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]);
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    i = 0;
+    while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+      char zLabel[20];
+      const char *zCol = (const char*)sqlite3_column_text(pStmt,2);
+      i++;
+      if( zCol==0 ){
+        if( sqlite3_column_int(pStmt,1)==-1 ){
+          zCol = "_ROWID_";
+        }else{
+          sqlite3_snprintf(sizeof(zLabel),zLabel,"expr%d",i);
+          zCol = zLabel;
+        }
+      }
+      if( isWO && lenPK==0 && sqlite3_column_int(pStmt,5)==0 && zCollist ){
+        lenPK = (int)strlen(zCollist);
+      }
+      if( zCollist==0 ){
+        zCollist = sqlite3_mprintf("\"%w\"", zCol);
+      }else{
+        zCollist = sqlite3_mprintf("%z,\"%w\"", zCollist, zCol);
+      }
+    }
+    sqlite3_finalize(pStmt);
+    if( i==0 || tnum==0 ){
+      sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]);
+      rc = 1;
+      sqlite3_free(zCollist);
+      goto meta_command_exit;
+    }
+    if( lenPK==0 ) lenPK = 100000;
+    zSql = sqlite3_mprintf(
+          "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID",
+          azArg[2], zCollist, lenPK, zCollist);
+    sqlite3_free(zCollist);
+    rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0);
+      if( rc ){
+        sqlite3_fprintf(stderr,
+              "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
+      }else{
+        sqlite3_fprintf(stdout, "%s;\n", zSql);
+        sqlite3_fprintf(stdout,
+              "WARNING: writing to an imposter table will corrupt"
+              " the \"%s\" %s!\n", azArg[1], isWO ? "table" : "index");
+      }
+    }else{
+      sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
+      rc = 1;
+    }
+    sqlite3_free(zSql);
+  }else
+#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */
+
+  if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){
+    i64 iArg = 0;
+    if( nArg==2 ){
+      iArg = integerValue(azArg[1]);
+      if( iArg==0 ) iArg = -1;
+    }
+    if( (nArg!=1 && nArg!=2) || iArg<0 ){
+      sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    rc = intckDatabaseCmd(p, iArg);
+  }else
+
+#ifdef SQLITE_ENABLE_IOTRACE
+  if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){
+    SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...);
+    if( iotrace && iotrace!=stdout ) fclose(iotrace);
+    iotrace = 0;
+    if( nArg<2 ){
+      sqlite3IoTrace = 0;
+    }else if( cli_strcmp(azArg[1], "-")==0 ){
+      sqlite3IoTrace = iotracePrintf;
+      iotrace = stdout;
+    }else{
+      iotrace = sqlite3_fopen(azArg[1], "w");
+      if( iotrace==0 ){
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
+        sqlite3IoTrace = 0;
+        rc = 1;
+      }else{
+        sqlite3IoTrace = iotracePrintf;
+      }
+    }
+  }else
+#endif
+
+  if( c=='l' && n>=5 && cli_strncmp(azArg[0], "limits", n)==0 ){
+    static const struct {
+       const char *zLimitName;   /* Name of a limit */
+       int limitCode;            /* Integer code for that limit */
+    } aLimit[] = {
+      { "length",                SQLITE_LIMIT_LENGTH                    },
+      { "sql_length",            SQLITE_LIMIT_SQL_LENGTH                },
+      { "column",                SQLITE_LIMIT_COLUMN                    },
+      { "expr_depth",            SQLITE_LIMIT_EXPR_DEPTH                },
+      { "compound_select",       SQLITE_LIMIT_COMPOUND_SELECT           },
+      { "vdbe_op",               SQLITE_LIMIT_VDBE_OP                   },
+      { "function_arg",          SQLITE_LIMIT_FUNCTION_ARG              },
+      { "attached",              SQLITE_LIMIT_ATTACHED                  },
+      { "like_pattern_length",   SQLITE_LIMIT_LIKE_PATTERN_LENGTH       },
+      { "variable_number",       SQLITE_LIMIT_VARIABLE_NUMBER           },
+      { "trigger_depth",         SQLITE_LIMIT_TRIGGER_DEPTH             },
+      { "worker_threads",        SQLITE_LIMIT_WORKER_THREADS            },
+    };
+    int i, n2;
+    open_db(p, 0);
+    if( nArg==1 ){
+      for(i=0; idb, aLimit[i].limitCode, -1));
+      }
+    }else if( nArg>3 ){
+      eputz("Usage: .limit NAME ?NEW-VALUE?\n");
+      rc = 1;
+      goto meta_command_exit;
+    }else{
+      int iLimit = -1;
+      n2 = strlen30(azArg[1]);
+      for(i=0; idb, aLimit[iLimit].limitCode,
+                      (int)integerValue(azArg[2]));
+      }
+      sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName,
+            sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
+    }
+  }else
+
+  if( c=='l' && n>2 && cli_strncmp(azArg[0], "lint", n)==0 ){
+    open_db(p, 0);
+    lintDotCommand(p, azArg, nArg);
+  }else
+
+#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE)
+  if( c=='l' && cli_strncmp(azArg[0], "load", n)==0 ){
+    const char *zFile, *zProc;
+    char *zErrMsg = 0;
+    failIfSafeMode(p, "cannot run .load in safe mode");
+    if( nArg<2 || azArg[1][0]==0 ){
+      /* Must have a non-empty FILE. (Will not load self.) */
+      eputz("Usage: .load FILE ?ENTRYPOINT?\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    zFile = azArg[1];
+    zProc = nArg>=3 ? azArg[2] : 0;
+    open_db(p, 0);
+    rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg);
+    if( rc!=SQLITE_OK ){
+      shellEmitError(zErrMsg);
+      sqlite3_free(zErrMsg);
+      rc = 1;
+    }
+  }else
+#endif
+
+  if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){
+    if( nArg!=2 ){
+      eputz("Usage: .log FILENAME\n");
+      rc = 1;
+    }else{
+      const char *zFile = azArg[1];
+      if( p->bSafeMode
+       && cli_strcmp(zFile,"on")!=0
+       && cli_strcmp(zFile,"off")!=0
+      ){
+        sputz(stdout, "cannot set .log to anything other"
+              " than \"on\" or \"off\"\n");
+        zFile = "off";
+      }
+      output_file_close(p->pLog);
+      if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout";
+      p->pLog = output_file_open(zFile);
+    }
+  }else
+
+  if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){
+    const char *zMode = 0;
+    const char *zTabname = 0;
+    int i, n2;
+    ColModeOpts cmOpts = ColModeOpts_default;
+    for(i=1; imode==MODE_Column
+       || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
+      ){
+        sqlite3_fprintf(p->out,
+              "current output mode: %s --wrap %d --wordwrap %s --%squote\n",
+              modeDescr[p->mode], p->cmOpts.iWrap,
+              p->cmOpts.bWordWrap ? "on" : "off",
+              p->cmOpts.bQuote ? "" : "no");
+      }else{
+        sqlite3_fprintf(p->out,
+              "current output mode: %s\n", modeDescr[p->mode]);
+      }
+      zMode = modeDescr[p->mode];
+    }
+    n2 = strlen30(zMode);
+    if( cli_strncmp(zMode,"lines",n2)==0 ){
+      p->mode = MODE_Line;
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+    }else if( cli_strncmp(zMode,"columns",n2)==0 ){
+      p->mode = MODE_Column;
+      if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){
+        p->showHeader = 1;
+      }
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+      p->cmOpts = cmOpts;
+    }else if( cli_strncmp(zMode,"list",n2)==0 ){
+      p->mode = MODE_List;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column);
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+    }else if( cli_strncmp(zMode,"html",n2)==0 ){
+      p->mode = MODE_Html;
+    }else if( cli_strncmp(zMode,"tcl",n2)==0 ){
+      p->mode = MODE_Tcl;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space);
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+    }else if( cli_strncmp(zMode,"csv",n2)==0 ){
+      p->mode = MODE_Csv;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
+    }else if( cli_strncmp(zMode,"tabs",n2)==0 ){
+      p->mode = MODE_List;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab);
+    }else if( cli_strncmp(zMode,"insert",n2)==0 ){
+      p->mode = MODE_Insert;
+      set_table_name(p, zTabname ? zTabname : "table");
+    }else if( cli_strncmp(zMode,"quote",n2)==0 ){
+      p->mode = MODE_Quote;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
+    }else if( cli_strncmp(zMode,"ascii",n2)==0 ){
+      p->mode = MODE_Ascii;
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit);
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record);
+    }else if( cli_strncmp(zMode,"markdown",n2)==0 ){
+      p->mode = MODE_Markdown;
+      p->cmOpts = cmOpts;
+    }else if( cli_strncmp(zMode,"table",n2)==0 ){
+      p->mode = MODE_Table;
+      p->cmOpts = cmOpts;
+    }else if( cli_strncmp(zMode,"box",n2)==0 ){
+      p->mode = MODE_Box;
+      p->cmOpts = cmOpts;
+    }else if( cli_strncmp(zMode,"count",n2)==0 ){
+      p->mode = MODE_Count;
+    }else if( cli_strncmp(zMode,"off",n2)==0 ){
+      p->mode = MODE_Off;
+    }else if( cli_strncmp(zMode,"json",n2)==0 ){
+      p->mode = MODE_Json;
+    }else{
+      eputz("Error: mode should be one of: "
+            "ascii box column csv html insert json line list markdown "
+            "qbox quote table tabs tcl\n");
+      rc = 1;
+    }
+    p->cMode = p->mode;
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){
+    if( nArg!=2 ){
+      eputz("Usage: .nonce NONCE\n");
+      rc = 1;
+    }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){
+      sqlite3_fprintf(stderr,"line %d: incorrect nonce: \"%s\"\n",
+            p->lineno, azArg[1]);
+      exit(1);
+    }else{
+      p->bSafeMode = 0;
+      return 0;  /* Return immediately to bypass the safe mode reset
+                 ** at the end of this procedure */
+    }
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){
+    if( nArg==2 ){
+      sqlite3_snprintf(sizeof(p->nullValue), p->nullValue,
+                       "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]);
+    }else{
+      eputz("Usage: .nullvalue STRING\n");
+      rc = 1;
+    }
+  }else
+
+  if( c=='o' && cli_strncmp(azArg[0], "open", n)==0 && n>=2 ){
+    const char *zFN = 0;     /* Pointer to constant filename */
+    char *zNewFilename = 0;  /* Name of the database file to open */
+    int iName = 1;           /* Index in azArg[] of the filename */
+    int newFlag = 0;         /* True to delete file before opening */
+    int openMode = SHELL_OPEN_UNSPEC;
+
+    /* Check for command-line arguments */
+    for(iName=1; iNameopenFlags |= SQLITE_OPEN_NOFOLLOW;
+#ifndef SQLITE_OMIT_DESERIALIZE
+      }else if( optionMatch(z, "deserialize") ){
+        openMode = SHELL_OPEN_DESERIALIZE;
+      }else if( optionMatch(z, "hexdb") ){
+        openMode = SHELL_OPEN_HEXDB;
+      }else if( optionMatch(z, "maxsize") && iName+1szMax = integerValue(azArg[++iName]);
+#endif /* SQLITE_OMIT_DESERIALIZE */
+      }else
+#endif /* !SQLITE_SHELL_FIDDLE */
+      if( z[0]=='-' ){
+        sqlite3_fprintf(stderr,"unknown option: %s\n", z);
+        rc = 1;
+        goto meta_command_exit;
+      }else if( zFN ){
+        sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z);
+        rc = 1;
+        goto meta_command_exit;
+      }else{
+        zFN = z;
+      }
+    }
+
+    /* Close the existing database */
+    session_close_all(p, -1);
+    close_db(p->db);
+    p->db = 0;
+    p->pAuxDb->zDbFilename = 0;
+    sqlite3_free(p->pAuxDb->zFreeOnClose);
+    p->pAuxDb->zFreeOnClose = 0;
+    p->openMode = openMode;
+    p->openFlags = 0;
+    p->szMax = 0;
+
+    /* If a filename is specified, try to open it first */
+    if( zFN || p->openMode==SHELL_OPEN_HEXDB ){
+      if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN);
+#ifndef SQLITE_SHELL_FIDDLE
+      if( p->bSafeMode
+       && p->openMode!=SHELL_OPEN_HEXDB
+       && zFN
+       && cli_strcmp(zFN,":memory:")!=0
+      ){
+        failIfSafeMode(p, "cannot open disk-based database files in safe mode");
+      }
+#else
+      /* WASM mode has its own sandboxed pseudo-filesystem. */
+#endif
+      if( zFN ){
+        zNewFilename = sqlite3_mprintf("%s", zFN);
+        shell_check_oom(zNewFilename);
+      }else{
+        zNewFilename = 0;
+      }
+      p->pAuxDb->zDbFilename = zNewFilename;
+      open_db(p, OPEN_DB_KEEPALIVE);
+      if( p->db==0 ){
+        sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename);
+        sqlite3_free(zNewFilename);
+      }else{
+        p->pAuxDb->zFreeOnClose = zNewFilename;
+      }
+    }
+    if( p->db==0 ){
+      /* As a fall-back open a TEMP database */
+      p->pAuxDb->zDbFilename = 0;
+      open_db(p, 0);
+    }
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( (c=='o'
+        && (cli_strncmp(azArg[0], "output", n)==0
+            || cli_strncmp(azArg[0], "once", n)==0))
+   || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0)
+   || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0)
+  ){
+    char *zFile = 0;
+    int i;
+    int eMode = 0;
+    int bOnce = 0;            /* 0: .output, 1: .once, 2: .excel/.www */
+    int bPlain = 0;           /* --plain option */
+    static const char *zBomUtf8 = "\357\273\277";
+    const char *zBom = 0;
+
+    failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
+    if( c=='e' ){
+      eMode = 'x';
+      bOnce = 2;
+    }else if( c=='w' ){
+      eMode = 'w';
+      bOnce = 2;
+    }else if( cli_strncmp(azArg[0],"once",n)==0 ){
+      bOnce = 1;
+    }
+    for(i=1; iout, 
+                          "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]);
+          showHelp(p->out, azArg[0]);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else if( zFile==0 && eMode==0 ){
+        zFile = sqlite3_mprintf("%s", z);
+        if( zFile && zFile[0]=='|' ){
+          while( i+1out,
+            "ERROR: extra parameter: \"%s\".  Usage:\n", azArg[i]);
+        showHelp(p->out, azArg[0]);
+        rc = 1;
+        sqlite3_free(zFile);
+        goto meta_command_exit;
+      }
+    }
+    if( zFile==0 ){
+      zFile = sqlite3_mprintf("stdout");
+    }
+    if( bOnce ){
+      p->outCount = 2;
+    }else{
+      p->outCount = 0;
+    }
+    output_reset(p);
+#ifndef SQLITE_NOHAVE_SYSTEM
+    if( eMode=='e' || eMode=='x' || eMode=='w' ){
+      p->doXdgOpen = 1;
+      outputModePush(p);
+      if( eMode=='x' ){
+        /* spreadsheet mode.  Output as CSV. */
+        newTempFile(p, "csv");
+        ShellClearFlag(p, SHFLG_Echo);
+        p->mode = MODE_Csv;
+        sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
+        sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
+#ifdef _WIN32
+        zBom = zBomUtf8;  /* Always include the BOM on Windows, as Excel does
+                          ** not work without it. */
+#endif
+      }else if( eMode=='w' ){
+        /* web-browser mode. */
+        newTempFile(p, "html");
+        if( !bPlain ) p->mode = MODE_Www;
+      }else{
+        /* text editor mode */
+        newTempFile(p, "txt");
+      }
+      sqlite3_free(zFile);
+      zFile = sqlite3_mprintf("%s", p->zTempFile);
+    }
+#endif /* SQLITE_NOHAVE_SYSTEM */
+    shell_check_oom(zFile);
+    if( zFile[0]=='|' ){
+#ifdef SQLITE_OMIT_POPEN
+      eputz("Error: pipes are not supported in this OS\n");
+      rc = 1;
+      output_redir(p, stdout);
+#else
+      FILE *pfPipe = sqlite3_popen(zFile + 1, "w");
+      if( pfPipe==0 ){
+        sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1);
+        rc = 1;
+      }else{
+        output_redir(p, pfPipe);
+        if( zBom ) sqlite3_fputs(zBom, pfPipe);
+        sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
+      }
+#endif
+    }else{
+      FILE *pfFile = output_file_open(zFile);
+      if( pfFile==0 ){
+        if( cli_strcmp(zFile,"off")!=0 ){
+          sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile);
+        }
+        rc = 1;
+      } else {
+        output_redir(p, pfFile);
+        if( zBom ) sqlite3_fputs(zBom, pfFile);
+        if( bPlain && eMode=='w' ){
+          sqlite3_fputs(
+            "\n\n\n",
+            pfFile
+          );
+        }
+        sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
+      }
+    }
+    sqlite3_free(zFile);
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='p' && n>=3 && cli_strncmp(azArg[0], "parameter", n)==0 ){
+    open_db(p,0);
+    if( nArg<=1 ) goto parameter_syntax_error;
+
+    /* .parameter clear
+    ** Clear all bind parameters by dropping the TEMP table that holds them.
+    */
+    if( nArg==2 && cli_strcmp(azArg[1],"clear")==0 ){
+      sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;",
+                   0, 0, 0);
+    }else
+
+    /* .parameter list
+    ** List all bind parameters.
+    */
+    if( nArg==2 && cli_strcmp(azArg[1],"list")==0 ){
+      sqlite3_stmt *pStmt = 0;
+      int rx;
+      int len = 0;
+      rx = sqlite3_prepare_v2(p->db,
+             "SELECT max(length(key)) "
+             "FROM temp.sqlite_parameters;", -1, &pStmt, 0);
+      if( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+        len = sqlite3_column_int(pStmt, 0);
+        if( len>40 ) len = 40;
+      }
+      sqlite3_finalize(pStmt);
+      pStmt = 0;
+      if( len ){
+        rx = sqlite3_prepare_v2(p->db,
+             "SELECT key, quote(value) "
+             "FROM temp.sqlite_parameters;", -1, &pStmt, 0);
+        while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+          sqlite3_fprintf(p->out,
+                "%-*s %s\n", len, sqlite3_column_text(pStmt,0),
+                sqlite3_column_text(pStmt,1));
+        }
+        sqlite3_finalize(pStmt);
+      }
+    }else
+
+    /* .parameter init
+    ** Make sure the TEMP table used to hold bind parameters exists.
+    ** Create it if necessary.
+    */
+    if( nArg==2 && cli_strcmp(azArg[1],"init")==0 ){
+      bind_table_init(p);
+    }else
+
+    /* .parameter set NAME VALUE
+    ** Set or reset a bind parameter.  NAME should be the full parameter
+    ** name exactly as it appears in the query.  (ex: $abc, @def).  The
+    ** VALUE can be in either SQL literal notation, or if not it will be
+    ** understood to be a text string.
+    */
+    if( nArg==4 && cli_strcmp(azArg[1],"set")==0 ){
+      int rx;
+      char *zSql;
+      sqlite3_stmt *pStmt;
+      const char *zKey = azArg[2];
+      const char *zValue = azArg[3];
+      bind_table_init(p);
+      zSql = sqlite3_mprintf(
+                  "REPLACE INTO temp.sqlite_parameters(key,value)"
+                  "VALUES(%Q,%s);", zKey, zValue);
+      shell_check_oom(zSql);
+      pStmt = 0;
+      rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+      sqlite3_free(zSql);
+      if( rx!=SQLITE_OK ){
+        sqlite3_finalize(pStmt);
+        pStmt = 0;
+        zSql = sqlite3_mprintf(
+                   "REPLACE INTO temp.sqlite_parameters(key,value)"
+                   "VALUES(%Q,%Q);", zKey, zValue);
+        shell_check_oom(zSql);
+        rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+        sqlite3_free(zSql);
+        if( rx!=SQLITE_OK ){
+          sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db));
+          sqlite3_finalize(pStmt);
+          pStmt = 0;
+          rc = 1;
+        }
+      }
+      sqlite3_step(pStmt);
+      sqlite3_finalize(pStmt);
+    }else
+
+    /* .parameter unset NAME
+    ** Remove the NAME binding from the parameter binding table, if it
+    ** exists.
+    */
+    if( nArg==3 && cli_strcmp(azArg[1],"unset")==0 ){
+      char *zSql = sqlite3_mprintf(
+          "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]);
+      shell_check_oom(zSql);
+      sqlite3_exec(p->db, zSql, 0, 0, 0);
+      sqlite3_free(zSql);
+    }else
+    /* If no command name matches, show a syntax error */
+    parameter_syntax_error:
+    showHelp(p->out, "parameter");
+  }else
+
+  if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){
+    int i;
+    for(i=1; i<nArg; i++){
+      if( i>1 ) sqlite3_fputs(" ", p->out);
+      sqlite3_fputs(azArg[i], p->out);
+    }
+    sqlite3_fputs("\n", p->out);
+  }else
+
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  if( c=='p' && n>=3 && cli_strncmp(azArg[0], "progress", n)==0 ){
+    int i;
+    int nn = 0;
+    p->flgProgress = 0;
+    p->mxProgress = 0;
+    p->nProgress = 0;
+    for(i=1; i<nArg; i++){
+      const char *z = azArg[i];
+      if( z[0]=='-' ){
+        z++;
+        if( z[0]=='-' ) z++;
+        if( cli_strcmp(z,"quiet")==0 || cli_strcmp(z,"q")==0 ){
+          p->flgProgress |= SHELL_PROGRESS_QUIET;
+          continue;
+        }
+        if( cli_strcmp(z,"reset")==0 ){
+          p->flgProgress |= SHELL_PROGRESS_RESET;
+          continue;
+        }
+        if( cli_strcmp(z,"once")==0 ){
+          p->flgProgress |= SHELL_PROGRESS_ONCE;
+          continue;
+        }
+        if( cli_strcmp(z,"limit")==0 ){
+          if( i+1>=nArg ){
+            eputz("Error: missing argument on --limit\n");
+            rc = 1;
+            goto meta_command_exit;
+          }else{
+            p->mxProgress = (int)integerValue(azArg[++i]);
+          }
+          continue;
+        }
+        sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]);
+        rc = 1;
+        goto meta_command_exit;
+      }else{
+        nn = (int)integerValue(z);
+      }
+    }
+    open_db(p, 0);
+    sqlite3_progress_handler(p->db, nn, progress_handler, p);
+  }else
+#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */
+
+  if( c=='p' && cli_strncmp(azArg[0], "prompt", n)==0 ){
+    if( nArg >= 2) {
+      shell_strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1);
+    }
+    if( nArg >= 3) {
+      shell_strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1);
+    }
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='q' && cli_strncmp(azArg[0], "quit", n)==0 ){
+    rc = 2;
+  }else
+#endif
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){
+    FILE *inSaved = p->in;
+    int savedLineno = p->lineno;
+    failIfSafeMode(p, "cannot run .read in safe mode");
+    if( nArg!=2 ){
+      eputz("Usage: .read FILE\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    if( azArg[1][0]=='|' ){
+#ifdef SQLITE_OMIT_POPEN
+      eputz("Error: pipes are not supported in this OS\n");
+      rc = 1;
+#else
+      p->in = sqlite3_popen(azArg[1]+1, "r");
+      if( p->in==0 ){
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
+        rc = 1;
+      }else{
+        rc = process_input(p);
+        pclose(p->in);
+      }
+#endif
+    }else if( (p->in = openChrSource(azArg[1]))==0 ){
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
+      rc = 1;
+    }else{
+      rc = process_input(p);
+      fclose(p->in);
+    }
+    p->in = inSaved;
+    p->lineno = savedLineno;
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+#ifndef SQLITE_SHELL_FIDDLE
+  if( c=='r' && n>=3 && cli_strncmp(azArg[0], "restore", n)==0 ){
+    const char *zSrcFile;
+    const char *zDb;
+    sqlite3 *pSrc;
+    sqlite3_backup *pBackup;
+    int nTimeout = 0;
+
+    failIfSafeMode(p, "cannot run .restore in safe mode");
+    if( nArg==2 ){
+      zSrcFile = azArg[1];
+      zDb = "main";
+    }else if( nArg==3 ){
+      zSrcFile = azArg[2];
+      zDb = azArg[1];
+    }else{
+      eputz("Usage: .restore ?DB? FILE\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    rc = sqlite3_open(zSrcFile, &pSrc);
+    if( rc!=SQLITE_OK ){
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile);
+      close_db(pSrc);
+      return 1;
+    }
+    open_db(p, 0);
+    pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main");
+    if( pBackup==0 ){
+      shellDatabaseError(p->db);
+      close_db(pSrc);
+      return 1;
+    }
+    while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK
+          || rc==SQLITE_BUSY  ){
+      if( rc==SQLITE_BUSY ){
+        if( nTimeout++ >= 3 ) break;
+        sqlite3_sleep(100);
+      }
+    }
+    sqlite3_backup_finish(pBackup);
+    if( rc==SQLITE_DONE ){
+      rc = 0;
+    }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){
+      eputz("Error: source database is busy\n");
+      rc = 1;
+    }else{
+      shellDatabaseError(p->db);
+      rc = 1;
+    }
+    close_db(pSrc);
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
+    if( nArg==2 ){
+      if( cli_strcmp(azArg[1], "vm")==0 ){
+        p->scanstatsOn = 3;
+      }else
+      if( cli_strcmp(azArg[1], "est")==0 ){
+        p->scanstatsOn = 2;
+      }else{
+        p->scanstatsOn = (u8)booleanValue(azArg[1]);
+      }
+      open_db(p, 0);
+      sqlite3_db_config(
+          p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0
+      );
+#if !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+      eputz("Warning: .scanstats not available in this build.\n");
+#elif !defined(SQLITE_ENABLE_BYTECODE_VTAB)
+      if( p->scanstatsOn==3 ){
+        eputz("Warning: \".scanstats vm\" not available in this build.\n");
+      }
+#endif
+    }else{
+      eputz("Usage: .scanstats on|off|est\n");
+      rc = 1;
+    }
+  }else
+
+  if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){
+    ShellText sSelect;
+    ShellState data;
+    char *zErrMsg = 0;
+    const char *zDiv = "(";
+    const char *zName = 0;
+    int iSchema = 0;
+    int bDebug = 0;
+    int bNoSystemTabs = 0;
+    int ii;
+
+    open_db(p, 0);
+    memcpy(&data, p, sizeof(data));
+    data.showHeader = 0;
+    data.cMode = data.mode = MODE_Semi;
+    initText(&sSelect);
+    for(ii=1; ii<nArg; ii++){
+      if( optionMatch(azArg[ii],"indent") ){
+        data.cMode = data.mode = MODE_Pretty;
+      }else if( optionMatch(azArg[ii],"debug") ){
+        bDebug = 1;
+      }else if( optionMatch(azArg[ii],"nosys") ){
+        bNoSystemTabs = 1;
+      }else if( azArg[ii][0]=='-' ){
+        sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]);
+        rc = 1;
+        goto meta_command_exit;
+      }else if( zName==0 ){
+        zName = azArg[ii];
+      }else{
+        eputz("Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
+    }
+    if( zName!=0 ){
+      int isSchema = sqlite3_strlike(zName, "sqlite_master", '\\')==0
+                  || sqlite3_strlike(zName, "sqlite_schema", '\\')==0
+                  || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0
+                  || sqlite3_strlike(zName,"sqlite_temp_schema", '\\')==0;
+      if( isSchema ){
+        char *new_argv[2], *new_colv[2];
+        new_argv[0] = sqlite3_mprintf(
+                      "CREATE TABLE %s (\n"
+                      "  type text,\n"
+                      "  name text,\n"
+                      "  tbl_name text,\n"
+                      "  rootpage integer,\n"
+                      "  sql text\n"
+                      ")", zName);
+        shell_check_oom(new_argv[0]);
+        new_argv[1] = 0;
+        new_colv[0] = "sql";
+        new_colv[1] = 0;
+        callback(&data, 1, new_argv, new_colv);
+        sqlite3_free(new_argv[0]);
+      }
+    }
+    if( zDiv ){
+      sqlite3_stmt *pStmt = 0;
+      rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list",
+                              -1, &pStmt, 0);
+      if( rc ){
+        shellDatabaseError(p->db);
+        sqlite3_finalize(pStmt);
+        rc = 1;
+        goto meta_command_exit;
+      }
+      appendText(&sSelect, "SELECT sql FROM", 0);
+      iSchema = 0;
+      while( sqlite3_step(pStmt)==SQLITE_ROW ){
+        const char *zDb = (const char*)sqlite3_column_text(pStmt, 0);
+        char zScNum[30];
+        sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema);
+        appendText(&sSelect, zDiv, 0);
+        zDiv = " UNION ALL ";
+        appendText(&sSelect, "SELECT shell_add_schema(sql,", 0);
+        if( sqlite3_stricmp(zDb, "main")!=0 ){
+          appendText(&sSelect, zDb, '\'');
+        }else{
+          appendText(&sSelect, "NULL", 0);
+        }
+        appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0);
+        appendText(&sSelect, zScNum, 0);
+        appendText(&sSelect, " AS snum, ", 0);
+        appendText(&sSelect, zDb, '\'');
+        appendText(&sSelect, " AS sname FROM ", 0);
+        appendText(&sSelect, zDb, quoteChar(zDb));
+        appendText(&sSelect, ".sqlite_schema", 0);
+      }
+      sqlite3_finalize(pStmt);
+#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS
+      if( zName ){
+        appendText(&sSelect,
+           " UNION ALL SELECT shell_module_schema(name),"
+           " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list",
+        0);
+      }
+#endif
+      appendText(&sSelect, ") WHERE ", 0);
+      if( zName ){
+        char *zQarg = sqlite3_mprintf("%Q", zName);
+        int bGlob;
+        shell_check_oom(zQarg);
+        bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 ||
+                strchr(zName, '[') != 0;
+        if( strchr(zName, '.') ){
+          appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0);
+        }else{
+          appendText(&sSelect, "lower(tbl_name)", 0);
+        }
+        appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0);
+        appendText(&sSelect, zQarg, 0);
+        if( !bGlob ){
+          appendText(&sSelect, " ESCAPE '\\' ", 0);
+        }
+        appendText(&sSelect, " AND ", 0);
+        sqlite3_free(zQarg);
+      }
+      if( bNoSystemTabs ){
+        appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0);
+      }
+      appendText(&sSelect, "sql IS NOT NULL"
+                           " ORDER BY snum, rowid", 0);
+      if( bDebug ){
+        sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.z);
+      }else{
+        rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
+      }
+      freeText(&sSelect);
+    }
+    if( zErrMsg ){
+      shellEmitError(zErrMsg);
+      sqlite3_free(zErrMsg);
+      rc = 1;
+    }else if( rc != SQLITE_OK ){
+      eputz("Error: querying schema information\n");
+      rc = 1;
+    }else{
+      rc = 0;
+    }
+  }else
+
+  if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0)
+   || (c=='t' && n==9  && cli_strncmp(azArg[0], "treetrace", n)==0)
+  ){
+    unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff;
+    sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x);
+  }else
+
+#if defined(SQLITE_ENABLE_SESSION)
+  if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){
+    struct AuxDb *pAuxDb = p->pAuxDb;
+    OpenSession *pSession = &pAuxDb->aSession[0];
+    char **azCmd = &azArg[1];
+    int iSes = 0;
+    int nCmd = nArg - 1;
+    int i;
+    if( nArg<=1 ) goto session_syntax_error;
+    open_db(p, 0);
+    if( nArg>=3 ){
+      for(iSes=0; iSes<pAuxDb->nSession; iSes++){
+        if( cli_strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break;
+      }
+      if( iSes<pAuxDb->nSession ){
+        pSession = &pAuxDb->aSession[iSes];
+        azCmd++;
+        nCmd--;
+      }else{
+        pSession = &pAuxDb->aSession[0];
+        iSes = 0;
+      }
+    }
+
+    /* .session attach TABLE
+    ** Invoke the sqlite3session_attach() interface to attach a particular
+    ** table so that it is never filtered.
+    */
+    if( cli_strcmp(azCmd[0],"attach")==0 ){
+      if( nCmd!=2 ) goto session_syntax_error;
+      if( pSession->p==0 ){
+        session_not_open:
+        eputz("ERROR: No sessions are open\n");
+      }else{
+        rc = sqlite3session_attach(pSession->p, azCmd[1]);
+        if( rc ){
+          sqlite3_fprintf(stderr,
+               "ERROR: sqlite3session_attach() returns %d\n",rc);
+          rc = 0;
+        }
+      }
+    }else
+
+    /* .session changeset FILE
+    ** .session patchset FILE
+    ** Write a changeset or patchset into a file.  The file is overwritten.
+    */
+    if( cli_strcmp(azCmd[0],"changeset")==0
+     || cli_strcmp(azCmd[0],"patchset")==0
+    ){
+      FILE *out = 0;
+      failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]);
+      if( nCmd!=2 ) goto session_syntax_error;
+      if( pSession->p==0 ) goto session_not_open;
+      out = sqlite3_fopen(azCmd[1], "wb");
+      if( out==0 ){
+        sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n",
+              azCmd[1]);
+      }else{
+        int szChng;
+        void *pChng;
+        if( azCmd[0][0]=='c' ){
+          rc = sqlite3session_changeset(pSession->p, &szChng, &pChng);
+        }else{
+          rc = sqlite3session_patchset(pSession->p, &szChng, &pChng);
+        }
+        if( rc ){
+          sqlite3_fprintf(stdout, "Error: error code %d\n", rc);
+          rc = 0;
+        }
+        if( pChng
+          && fwrite(pChng, szChng, 1, out)!=1 ){
+          sqlite3_fprintf(stderr,
+              "ERROR: Failed to write entire %d-byte output\n", szChng);
+        }
+        sqlite3_free(pChng);
+        fclose(out);
+      }
+    }else
+
+    /* .session close
+    ** Close the identified session
+    */
+    if( cli_strcmp(azCmd[0], "close")==0 ){
+      if( nCmd!=1 ) goto session_syntax_error;
+      if( pAuxDb->nSession ){
+        session_close(pSession);
+        pAuxDb->aSession[iSes] = pAuxDb->aSession[--pAuxDb->nSession];
+      }
+    }else
+
+    /* .session enable ?BOOLEAN?
+    ** Query or set the enable flag
+    */
+    if( cli_strcmp(azCmd[0], "enable")==0 ){
+      int ii;
+      if( nCmd>2 ) goto session_syntax_error;
+      ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
+      if( pAuxDb->nSession ){
+        ii = sqlite3session_enable(pSession->p, ii);
+        sqlite3_fprintf(p->out,
+            "session %s enable flag = %d\n", pSession->zName, ii);
+      }
+    }else
+
+    /* .session filter GLOB ....
+    ** Set a list of GLOB patterns of table names to be excluded.
+    */
+    if( cli_strcmp(azCmd[0], "filter")==0 ){
+      int ii, nByte;
+      if( nCmd<2 ) goto session_syntax_error;
+      if( pAuxDb->nSession ){
+        for(ii=0; ii<pSession->nFilter; ii++){
+          sqlite3_free(pSession->azFilter[ii]);
+        }
+        sqlite3_free(pSession->azFilter);
+        nByte = sizeof(pSession->azFilter[0])*(nCmd-1);
+        pSession->azFilter = sqlite3_malloc( nByte );
+        shell_check_oom( pSession->azFilter );
+        for(ii=1; ii<nCmd; ii++){
+          char *x = pSession->azFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]);
+          shell_check_oom(x);
+        }
+        pSession->nFilter = ii-1;
+      }
+    }else
+
+    /* .session indirect ?BOOLEAN?
+    ** Query or set the indirect flag
+    */
+    if( cli_strcmp(azCmd[0], "indirect")==0 ){
+      int ii;
+      if( nCmd>2 ) goto session_syntax_error;
+      ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
+      if( pAuxDb->nSession ){
+        ii = sqlite3session_indirect(pSession->p, ii);
+        sqlite3_fprintf(p->out,
+            "session %s indirect flag = %d\n", pSession->zName, ii);
+      }
+    }else
+
+    /* .session isempty
+    ** Determine if the session is empty
+    */
+    if( cli_strcmp(azCmd[0], "isempty")==0 ){
+      int ii;
+      if( nCmd!=1 ) goto session_syntax_error;
+      if( pAuxDb->nSession ){
+        ii = sqlite3session_isempty(pSession->p);
+        sqlite3_fprintf(p->out,
+             "session %s isempty flag = %d\n", pSession->zName, ii);
+      }
+    }else
+
+    /* .session list
+    ** List all currently open sessions
+    */
+    if( cli_strcmp(azCmd[0],"list")==0 ){
+      for(i=0; i<pAuxDb->nSession; i++){
+        sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName);
+      }
+    }else
+
+    /* .session open DB NAME
+    ** Open a new session called NAME on the attached database DB.
+    ** DB is normally "main".
+    */
+    if( cli_strcmp(azCmd[0],"open")==0 ){
+      char *zName;
+      if( nCmd!=3 ) goto session_syntax_error;
+      zName = azCmd[2];
+      if( zName[0]==0 ) goto session_syntax_error;
+      for(i=0; i<pAuxDb->nSession; i++){
+        if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){
+          sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName);
+          goto meta_command_exit;
+        }
+      }
+      if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){
+        sqlite3_fprintf(stderr,
+           "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession));
+        goto meta_command_exit;
+      }
+      pSession = &pAuxDb->aSession[pAuxDb->nSession];
+      rc = sqlite3session_create(p->db, azCmd[1], &pSession->p);
+      if( rc ){
+        sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc);
+        rc = 0;
+        goto meta_command_exit;
+      }
+      pSession->nFilter = 0;
+      sqlite3session_table_filter(pSession->p, session_filter, pSession);
+      pAuxDb->nSession++;
+      pSession->zName = sqlite3_mprintf("%s", zName);
+      shell_check_oom(pSession->zName);
+    }else
+    /* If no command name matches, show a syntax error */
+    session_syntax_error:
+    showHelp(p->out, "session");
+  }else
+#endif
+
+#ifdef SQLITE_DEBUG
+  /* Undocumented commands for internal testing.  Subject to change
+  ** without notice. */
+  if( c=='s' && n>=10 && cli_strncmp(azArg[0], "selftest-", 9)==0 ){
+    if( cli_strncmp(azArg[0]+9, "boolean", n-9)==0 ){
+      int i, v;
+      for(i=1; i<nArg; i++){
+        v = booleanValue(azArg[i]);
+        sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
+      }
+    }
+    if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){
+      int i; sqlite3_int64 v;
+      for(i=1; i<nArg; i++){
+        char zBuf[200];
+        v = integerValue(azArg[i]);
+        sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v);
+        sqlite3_fputs(zBuf, p->out);
+      }
+    }
+  }else
+#endif
+
+  if( c=='s' && n>=4 && cli_strncmp(azArg[0],"selftest",n)==0 ){
+    int bIsInit = 0;         /* True to initialize the SELFTEST table */
+    int bVerbose = 0;        /* Verbose output */
+    int bSelftestExists;     /* True if SELFTEST already exists */
+    int i, k;                /* Loop counters */
+    int nTest = 0;           /* Number of tests runs */
+    int nErr = 0;            /* Number of errors seen */
+    ShellText str;           /* Answer for a query */
+    sqlite3_stmt *pStmt = 0; /* Query against the SELFTEST table */
+
+    open_db(p,0);
+    for(i=1; i<nArg; i++){
+      const char *z = azArg[i];
+      if( z[0]=='-' && z[1]=='-' ) z++;
+      if( cli_strcmp(z,"-init")==0 ){
+        bIsInit = 1;
+      }else
+      if( cli_strcmp(z,"-v")==0 ){
+        bVerbose++;
+      }else
+      {
+        sqlite3_fprintf(stderr,
+              "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
+        sqlite3_fputs("Should be one of: --init -v\n", stderr);
+        rc = 1;
+        goto meta_command_exit;
+      }
+    }
+    if( sqlite3_table_column_metadata(p->db,"main","selftest",0,0,0,0,0,0)
+           != SQLITE_OK ){
+      bSelftestExists = 0;
+    }else{
+      bSelftestExists = 1;
+    }
+    if( bIsInit ){
+      createSelftestTable(p);
+      bSelftestExists = 1;
+    }
+    initText(&str);
+    appendText(&str, "x", 0);
+    for(k=bSelftestExists; k>=0; k--){
+      if( k==1 ){
+        rc = sqlite3_prepare_v2(p->db,
+            "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno",
+            -1, &pStmt, 0);
+      }else{
+        rc = sqlite3_prepare_v2(p->db,
+          "VALUES(0,'memo','Missing SELFTEST table - default checks only',''),"
+          "      (1,'run','PRAGMA integrity_check','ok')",
+          -1, &pStmt, 0);
+      }
+      if( rc ){
+        eputz("Error querying the selftest table\n");
+        rc = 1;
+        sqlite3_finalize(pStmt);
+        goto meta_command_exit;
+      }
+      for(i=1; sqlite3_step(pStmt)==SQLITE_ROW; i++){
+        int tno = sqlite3_column_int(pStmt, 0);
+        const char *zOp = (const char*)sqlite3_column_text(pStmt, 1);
+        const char *zSql = (const char*)sqlite3_column_text(pStmt, 2);
+        const char *zAns = (const char*)sqlite3_column_text(pStmt, 3);
+
+        if( zOp==0 ) continue;
+        if( zSql==0 ) continue;
+        if( zAns==0 ) continue;
+        k = 0;
+        if( bVerbose>0 ){
+          sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql);
+        }
+        if( cli_strcmp(zOp,"memo")==0 ){
+          sqlite3_fprintf(p->out, "%s\n", zSql);
+        }else
+        if( cli_strcmp(zOp,"run")==0 ){
+          char *zErrMsg = 0;
+          str.n = 0;
+          str.z[0] = 0;
+          rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
+          nTest++;
+          if( bVerbose ){
+            sqlite3_fprintf(p->out, "Result: %s\n", str.z);
+          }
+          if( rc || zErrMsg ){
+            nErr++;
+            rc = 1;
+            sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg);
+            sqlite3_free(zErrMsg);
+          }else if( cli_strcmp(zAns,str.z)!=0 ){
+            nErr++;
+            rc = 1;
+            sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns);
+            sqlite3_fprintf(p->out, "%d:      Got: [%s]\n", tno, str.z);
+          }
+        }
+        else{
+          sqlite3_fprintf(stderr,
+                "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
+          rc = 1;
+          break;
+        }
+      } /* End loop over rows of content from SELFTEST */
+      sqlite3_finalize(pStmt);
+    } /* End loop over k */
+    freeText(&str);
+    sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest);
+  }else
+
+  if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){
+    if( nArg<2 || nArg>3 ){
+      eputz("Usage: .separator COL ?ROW?\n");
+      rc = 1;
+    }
+    if( nArg>=2 ){
+      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator,
+                       "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]);
+    }
+    if( nArg>=3 ){
+      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator,
+                       "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]);
+    }
+  }else
+
+  if( c=='s' && n>=4 && cli_strncmp(azArg[0],"sha3sum",n)==0 ){
+    const char *zLike = 0;   /* Which table to checksum. 0 means everything */
+    int i;                   /* Loop counter */
+    int bSchema = 0;         /* Also hash the schema */
+    int bSeparate = 0;       /* Hash each table separately */
+    int iSize = 224;         /* Hash algorithm to use */
+    int bDebug = 0;          /* Only show the query that would have run */
+    sqlite3_stmt *pStmt;     /* For querying tables names */
+    char *zSql;              /* SQL to be run */
+    char *zSep;              /* Separator */
+    ShellText sSql;          /* Complete SQL for the query to run the hash */
+    ShellText sQuery;        /* Set of queries used to read all content */
+    open_db(p, 0);
+    for(i=1; i<nArg; i++){
+      const char *z = azArg[i];
+      if( z[0]=='-' ){
+        z++;
+        if( z[0]=='-' ) z++;
+        if( cli_strcmp(z,"schema")==0 ){
+          bSchema = 1;
+        }else
+        if( cli_strcmp(z,"sha3-224")==0 || cli_strcmp(z,"sha3-256")==0
+         || cli_strcmp(z,"sha3-384")==0 || cli_strcmp(z,"sha3-512")==0
+        ){
+          iSize = atoi(&z[5]);
+        }else
+        if( cli_strcmp(z,"debug")==0 ){
+          bDebug = 1;
+        }else
+        {
+          sqlite3_fprintf(stderr,
+                  "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
+          showHelp(p->out, azArg[0]);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else if( zLike ){
+        eputz("Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n");
+        rc = 1;
+        goto meta_command_exit;
+      }else{
+        zLike = z;
+        bSeparate = 1;
+        if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1;
+      }
+    }
+    if( bSchema ){
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
+             " WHERE type='table' AND coalesce(rootpage,0)>1"
+             " UNION ALL SELECT 'sqlite_schema'"
+             " ORDER BY 1 collate nocase";
+    }else{
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
+             " WHERE type='table' AND coalesce(rootpage,0)>1"
+             " AND name NOT LIKE 'sqlite_%'"
+             " ORDER BY 1 collate nocase";
+    }
+    sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    initText(&sQuery);
+    initText(&sSql);
+    appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0);
+    zSep = "VALUES(";
+    while( SQLITE_ROW==sqlite3_step(pStmt) ){
+      const char *zTab = (const char*)sqlite3_column_text(pStmt,0);
+      if( zTab==0 ) continue;
+      if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue;
+      if( cli_strncmp(zTab, "sqlite_",7)!=0 ){
+        appendText(&sQuery,"SELECT * FROM ", 0);
+        appendText(&sQuery,zTab,'"');
+        appendText(&sQuery," NOT INDEXED;", 0);
+      }else if( cli_strcmp(zTab, "sqlite_schema")==0 ){
+        appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema"
+                           " ORDER BY name;", 0);
+      }else if( cli_strcmp(zTab, "sqlite_sequence")==0 ){
+        appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence"
+                           " ORDER BY name;", 0);
+      }else if( cli_strcmp(zTab, "sqlite_stat1")==0 ){
+        appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1"
+                           " ORDER BY tbl,idx;", 0);
+      }else if( cli_strcmp(zTab, "sqlite_stat4")==0 ){
+        appendText(&sQuery, "SELECT * FROM ", 0);
+        appendText(&sQuery, zTab, 0);
+        appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0);
+      }
+      appendText(&sSql, zSep, 0);
+      appendText(&sSql, sQuery.z, '\'');
+      sQuery.n = 0;
+      appendText(&sSql, ",", 0);
+      appendText(&sSql, zTab, '\'');
+      zSep = "),(";
+    }
+    sqlite3_finalize(pStmt);
+    if( bSeparate ){
+      zSql = sqlite3_mprintf(
+          "%s))"
+          " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label"
+          "   FROM [sha3sum$query]",
+          sSql.z, iSize);
+    }else{
+      zSql = sqlite3_mprintf(
+          "%s))"
+          " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash"
+          "   FROM [sha3sum$query]",
+          sSql.z, iSize);
+    }
+    shell_check_oom(zSql);
+    freeText(&sQuery);
+    freeText(&sSql);
+    if( bDebug ){
+      sqlite3_fprintf(p->out, "%s\n", zSql);
+    }else{
+      shell_exec(p, zSql, 0);
+    }
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+    {
+      int lrc;
+      char *zRevText = /* Query for reversible to-blob-to-text check */
+        "SELECT lower(name) as tname FROM sqlite_schema\n"
+        "WHERE type='table' AND coalesce(rootpage,0)>1\n"
+        "AND name NOT LIKE 'sqlite_%%'%s\n"
+        "ORDER BY 1 collate nocase";
+      zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : "");
+      zRevText = sqlite3_mprintf(
+          /* lower-case query is first run, producing upper-case query. */
+          "with tabcols as materialized(\n"
+          "select tname, cname\n"
+          "from ("
+          " select printf('\"%%w\"',ss.tname) as tname,"
+          " printf('\"%%w\"',ti.name) as cname\n"
+          " from (%z) ss\n inner join pragma_table_info(tname) ti))\n"
+          "select 'SELECT total(bad_text_count) AS bad_text_count\n"
+          "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n"
+          " from (select 'SELECT COUNT(*) AS bad_text_count\n"
+          "FROM '||tname||' WHERE '\n"
+          "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n"
+          "|| ' AND typeof('||cname||')=''text'' ',\n"
+          "' OR ') as query, tname from tabcols group by tname)"
+          , zRevText);
+      shell_check_oom(zRevText);
+      if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText);
+      lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
+      if( lrc!=SQLITE_OK ){
+        /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the
+        ** user does cruel and unnatural things like ".limit expr_depth 0". */
+        rc = 1;
+      }else{
+        if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC);
+        lrc = SQLITE_ROW==sqlite3_step(pStmt);
+        if( lrc ){
+          const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
+          sqlite3_stmt *pCheckStmt;
+          lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
+          if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery);
+          if( lrc!=SQLITE_OK ){
+            rc = 1;
+          }else{
+            if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){
+              double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
+              if( countIrreversible>0 ){
+                int sz = (int)(countIrreversible + 0.5);
+                sqlite3_fprintf(stderr,
+                      "Digest includes %d invalidly encoded text field%s.\n",
+                      sz, (sz>1)? "s": "");
+              }
+            }
+            sqlite3_finalize(pCheckStmt);
+          }
+          sqlite3_finalize(pStmt);
+        }
+      }
+      if( rc ) eputz(".sha3sum failed.\n");
+      sqlite3_free(zRevText);
+    }
+#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */
+    sqlite3_free(zSql);
+  }else
+
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
+  if( c=='s'
+   && (cli_strncmp(azArg[0], "shell", n)==0
+       || cli_strncmp(azArg[0],"system",n)==0)
+  ){
+    char *zCmd;
+    int i, x;
+    failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
+    if( nArg<2 ){
+      eputz("Usage: .system COMMAND\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    zCmd = sqlite3_mprintf(strchr(azArg[1],' ')==0?"%s":"\"%s\"", azArg[1]);
+    for(i=2; i<nArg && zCmd!=0; i++){
+      zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"",
+                             zCmd, azArg[i]);
+    }
+    /*consoleRestore();*/
+    x = zCmd!=0 ? system(zCmd) : 1;
+    /*consoleRenewSetup();*/
+    sqlite3_free(zCmd);
+    if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x);
+  }else
+#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */
+
+  if( c=='s' && cli_strncmp(azArg[0], "show", n)==0 ){
+    static const char *azBool[] = { "off", "on", "trigger", "full"};
+    const char *zOut;
+    int i;
+    if( nArg!=1 ){
+      eputz("Usage: .show\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","echo",
+          azBool[ShellHasFlag(p, SHFLG_Echo)]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","explain",
+          p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off");
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","headers",
+          azBool[p->showHeader!=0]);
+    if( p->mode==MODE_Column
+     || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
+    ){
+      sqlite3_fprintf(p->out,
+            "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode",
+            modeDescr[p->mode], p->cmOpts.iWrap,
+            p->cmOpts.bWordWrap ? "on" : "off",
+            p->cmOpts.bQuote ? "" : "no");
+    }else{
+      sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]);
+    }
+    sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue");
+    output_c_string(p->out, p->nullValue);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","output",
+          strlen30(p->outfile) ? p->outfile : "stdout");
+    sqlite3_fprintf(p->out, "%12.12s: ", "colseparator");
+    output_c_string(p->out, p->colSeparator);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator");
+    output_c_string(p->out, p->rowSeparator);
+    sqlite3_fputs("\n", p->out);
+    switch( p->statsOn ){
+      case 0:  zOut = "off";     break;
+      default: zOut = "on";      break;
+      case 2:  zOut = "stmt";    break;
+      case 3:  zOut = "vmstep";  break;
+    }
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut);
+    sqlite3_fprintf(p->out, "%12.12s: ", "width");
+    for (i=0;i<p->nWidth;i++) {
+      sqlite3_fprintf(p->out, "%d ", p->colWidth[i]);
+    }
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename",
+          p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : "");
+  }else
+
+  if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){
+    if( nArg==2 ){
+      if( cli_strcmp(azArg[1],"stmt")==0 ){
+        p->statsOn = 2;
+      }else if( cli_strcmp(azArg[1],"vmstep")==0 ){
+        p->statsOn = 3;
+      }else{
+        p->statsOn = (u8)booleanValue(azArg[1]);
+      }
+    }else if( nArg==1 ){
+      display_stats(p->db, p, 0);
+    }else{
+      eputz("Usage: .stats ?on|off|stmt|vmstep?\n");
+      rc = 1;
+    }
+  }else
+
+  if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0)
+   || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0
+                 || cli_strncmp(azArg[0], "indexes", n)==0) )
+  ){
+    sqlite3_stmt *pStmt;
+    char **azResult;
+    int nRow, nAlloc;
+    int ii;
+    ShellText s;
+    initText(&s);
+    open_db(p, 0);
+    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
+    if( rc ){
+      sqlite3_finalize(pStmt);
+      return shellDatabaseError(p->db);
+    }
+
+    if( nArg>2 && c=='i' ){
+      /* It is an historical accident that the .indexes command shows an error
+      ** when called with the wrong number of arguments whereas the .tables
+      ** command does not. */
+      eputz("Usage: .indexes ?LIKE-PATTERN?\n");
+      rc = 1;
+      sqlite3_finalize(pStmt);
+      goto meta_command_exit;
+    }
+    for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){
+      const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1);
+      if( zDbName==0 ) continue;
+      if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0);
+      if( sqlite3_stricmp(zDbName, "main")==0 ){
+        appendText(&s, "SELECT name FROM ", 0);
+      }else{
+        appendText(&s, "SELECT ", 0);
+        appendText(&s, zDbName, '\'');
+        appendText(&s, "||'.'||name FROM ", 0);
+      }
+      appendText(&s, zDbName, '"');
+      appendText(&s, ".sqlite_schema ", 0);
+      if( c=='t' ){
+        appendText(&s," WHERE type IN ('table','view')"
+                      "   AND name NOT LIKE 'sqlite_%'"
+                      "   AND name LIKE ?1", 0);
+      }else{
+        appendText(&s," WHERE type='index'"
+                      "   AND tbl_name LIKE ?1", 0);
+      }
+    }
+    rc = sqlite3_finalize(pStmt);
+    if( rc==SQLITE_OK ){
+      appendText(&s, " ORDER BY 1", 0);
+      rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0);
+    }
+    freeText(&s);
+    if( rc ) return shellDatabaseError(p->db);
+
+    /* Run the SQL statement prepared by the above block. Store the results
+    ** as an array of nul-terminated strings in azResult[].  */
+    nRow = nAlloc = 0;
+    azResult = 0;
+    if( nArg>1 ){
+      sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT);
+    }else{
+      sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC);
+    }
+    while( sqlite3_step(pStmt)==SQLITE_ROW ){
+      if( nRow>=nAlloc ){
+        char **azNew;
+        int n2 = nAlloc*2 + 10;
+        azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
+        shell_check_oom(azNew);
+        nAlloc = n2;
+        azResult = azNew;
+      }
+      azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
+      shell_check_oom(azResult[nRow]);
+      nRow++;
+    }
+    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
+      rc = shellDatabaseError(p->db);
+    }
+
+    /* Pretty-print the contents of array azResult[] to the output */
+    if( rc==0 && nRow>0 ){
+      int len, maxlen = 0;
+      int i, j;
+      int nPrintCol, nPrintRow;
+      for(i=0; i<nRow; i++){
+        len = strlen30(azResult[i]);
+        if( len>maxlen ) maxlen = len;
+      }
+      nPrintCol = 80/(maxlen+2);
+      if( nPrintCol<1 ) nPrintCol = 1;
+      nPrintRow = (nRow + nPrintCol - 1)/nPrintCol;
+      for(i=0; i<nPrintRow; i++){
+        for(j=i; j<nRow; j+=nPrintRow){
+          char *zSp = j<nPrintRow ? "" : "  ";
+          sqlite3_fprintf(p->out,
+               "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:"");
+        }
+        sqlite3_fputs("\n", p->out);
+      }
+    }
+
+    for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]);
+    sqlite3_free(azResult);
+  }else
+
+#ifndef SQLITE_SHELL_FIDDLE
+  /* Begin redirecting output to the file "testcase-out.txt" */
+  if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){
+    output_reset(p);
+    p->out = output_file_open("testcase-out.txt");
+    if( p->out==0 ){
+      eputz("Error: cannot open 'testcase-out.txt'\n");
+    }
+    if( nArg>=2 ){
+      sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]);
+    }else{
+      sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?");
+    }
+  }else
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
+
+#ifndef SQLITE_UNTESTABLE
+  if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){
+    static const struct {
+       const char *zCtrlName;   /* Name of a test-control option */
+       int ctrlCode;            /* Integer code for that option */
+       int unSafe;              /* Not valid unless --unsafe-testing */
+       const char *zUsage;      /* Usage notes */
+    } aCtrl[] = {
+    {"always",             SQLITE_TESTCTRL_ALWAYS, 1,     "BOOLEAN"         },
+    {"assert",             SQLITE_TESTCTRL_ASSERT, 1,     "BOOLEAN"         },
+  /*{"benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, ""        },*/
+  /*{"bitvec_test",        SQLITE_TESTCTRL_BITVEC_TEST, 1,  ""              },*/
+    {"byteorder",          SQLITE_TESTCTRL_BYTEORDER, 0,  ""                },
+    {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN"  },
+    {"fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..."       },
+    {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
+    {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
+    {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
+    {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
+    {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
+    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK ..."},
+#ifdef YYCOVERAGE
+    {"parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE,0,""             },
+#endif
+    {"pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,1, "OFFSET  "       },
+    {"prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,0, ""               },
+    {"prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,   0, ""               },
+    {"prng_seed",          SQLITE_TESTCTRL_PRNG_SEED,   0, "SEED ?db?"      },
+    {"seek_count",         SQLITE_TESTCTRL_SEEK_COUNT,  0, ""               },
+    {"sorter_mmap",        SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX"           },
+    {"tune",               SQLITE_TESTCTRL_TUNE,        1, "ID VALUE"       },
+    };
+    int testctrl = -1;
+    int iCtrl = -1;
+    int rc2 = 0;    /* 0: usage.  1: %d  2: %x  3: no-output */
+    int isOk = 0;
+    int i, n2;
+    const char *zCmd = 0;
+
+    open_db(p, 0);
+    zCmd = nArg>=2 ? azArg[1] : "help";
+
+    /* The argument can optionally begin with "-" or "--" */
+    if( zCmd[0]=='-' && zCmd[1] ){
+      zCmd++;
+      if( zCmd[0]=='-' && zCmd[1] ) zCmd++;
+    }
+
+    /* --help lists all test-controls */
+    if( cli_strcmp(zCmd,"help")==0 ){
+      sqlite3_fputs("Available test-controls:\n", p->out);
+      for(i=0; i<ArraySize(aCtrl); i++){
+        if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue;
+        sqlite3_fprintf(p->out, "  .testctrl %s %s\n",
+              aCtrl[i].zCtrlName, aCtrl[i].zUsage);
+      }
+      rc = 1;
+      goto meta_command_exit;
+    }
+
+    /* convert testctrl text option to value. allow any unique prefix
+    ** of the option name, or a numerical value. */
+    n2 = strlen30(zCmd);
+    for(i=0; i<ArraySize(aCtrl); i++){
+      if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue;
+      if( cli_strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){
+        if( testctrl<0 ){
+          testctrl = aCtrl[i].ctrlCode;
+          iCtrl = i;
+        }else{
+          sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n"
+                "Use \".testctrl --help\" for help\n", zCmd);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }
+    }
+    if( testctrl<0 ){
+      sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n"
+            "Use \".testctrl --help\" for help\n", zCmd);
+    }else{
+      switch(testctrl){
+
+        /* Special processing for .testctrl opt MASK ...
+        ** Each MASK argument can be one of:
+        **
+        **      +LABEL       Enable the named optimization 
+        **
+        **      -LABEL       Disable the named optimization
+        **
+        **      INTEGER      Mask of optimizations to disable
+        */
+        case SQLITE_TESTCTRL_OPTIMIZATIONS: {
+          static const struct {
+             unsigned int mask;    /* Mask for this optimization */
+             unsigned int bDsply;  /* Display this on output */
+             const char *zLabel;   /* Name of optimization */
+          } aLabel[] = {
+            { 0x00000001, 1, "QueryFlattener" },
+            { 0x00000001, 0, "Flatten" },
+            { 0x00000002, 1, "WindowFunc" },
+            { 0x00000004, 1, "GroupByOrder" },
+            { 0x00000008, 1, "FactorOutConst" },
+            { 0x00000010, 1, "DistinctOpt" },
+            { 0x00000020, 1, "CoverIdxScan" },
+            { 0x00000040, 1, "OrderByIdxJoin" },
+            { 0x00000080, 1, "Transitive" },
+            { 0x00000100, 1, "OmitNoopJoin" },
+            { 0x00000200, 1, "CountOfView" },
+            { 0x00000400, 1, "CurosrHints" },
+            { 0x00000800, 1, "Stat4" },
+            { 0x00001000, 1, "PushDown" },
+            { 0x00002000, 1, "SimplifyJoin" },
+            { 0x00004000, 1, "SkipScan" },
+            { 0x00008000, 1, "PropagateConst" },
+            { 0x00010000, 1, "MinMaxOpt" },
+            { 0x00020000, 1, "SeekScan" },
+            { 0x00040000, 1, "OmitOrderBy" },
+            { 0x00080000, 1, "BloomFilter" },
+            { 0x00100000, 1, "BloomPulldown" },
+            { 0x00200000, 1, "BalancedMerge" },
+            { 0x00400000, 1, "ReleaseReg" },
+            { 0x00800000, 1, "FlttnUnionAll" },
+            { 0x01000000, 1, "IndexedEXpr" },
+            { 0x02000000, 1, "Coroutines" },
+            { 0x04000000, 1, "NullUnusedCols" },
+            { 0x08000000, 1, "OnePass" },
+            { 0x10000000, 1, "OrderBySubq" },
+            { 0xffffffff, 0, "All" },
+          };
+          unsigned int curOpt;
+          unsigned int newOpt;
+          int ii;
+          sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt);
+          newOpt = curOpt;
+          for(ii=2; ii<nArg; ii++){
+            const char *z = azArg[ii];
+            int useLabel = 0;
+            const char *zLabel = 0;
+            if( (z[0]=='+'|| z[0]=='-') && !IsDigit(z[1]) ){
+              useLabel = z[0];
+              zLabel = &z[1];
+            }else if( !IsDigit(z[0]) && z[0]!=0 && !IsDigit(z[1]) ){
+              useLabel = '+';
+              zLabel = z;
+            }else{
+              newOpt = (unsigned int)strtol(z,0,0);
+            }
+            if( useLabel ){
+              int jj;
+              for(jj=0; jj<ArraySize(aLabel); jj++){
+                if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break;
+              }
+              if( jj>=ArraySize(aLabel) ){
+                sqlite3_fprintf(stderr,
+                    "Error: no such optimization: \"%s\"\n", zLabel);
+                sqlite3_fputs("Should be one of:", stderr);
+                for(jj=0; jj<ArraySize(aLabel); jj++){
+                  sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel);
+                }
+                sqlite3_fputs("\n", stderr);
+                rc = 1;
+                goto meta_command_exit;
+              }
+              if( useLabel=='+' ){
+                newOpt &= ~aLabel[jj].mask;
+              }else{
+                newOpt |= aLabel[jj].mask;
+              }
+            }
+          }
+          if( curOpt!=newOpt ){
+            sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,p->db,newOpt);
+          }else if( nArg<3 ){
+            curOpt = ~newOpt;
+          }
+          if( newOpt==0 ){
+            sqlite3_fputs("+All\n", p->out);
+          }else if( newOpt==0xffffffff ){
+            sqlite3_fputs("-All\n", p->out);
+          }else{
+            int jj;
+            for(jj=0; jj<ArraySize(aLabel); jj++){
+              unsigned int m = aLabel[jj].mask;
+              if( !aLabel[jj].bDsply  ) continue;
+              if( (curOpt&m)!=(newOpt&m) ){
+                sqlite3_fprintf(p->out, "%c%s\n", (newOpt & m)==0 ? '+' : '-',
+                      aLabel[jj].zLabel);
+              }
+            }
+          }
+          rc2 = isOk = 3;
+          break;
+        }
+
+        /* sqlite3_test_control(int, db, int) */
+        case SQLITE_TESTCTRL_FK_NO_ACTION:
+          if( nArg==3 ){
+            unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0);
+            rc2 = sqlite3_test_control(testctrl, p->db, opt);
+            isOk = 3;
+          }
+          break;
+
+        /* sqlite3_test_control(int) */
+        case SQLITE_TESTCTRL_PRNG_SAVE:
+        case SQLITE_TESTCTRL_PRNG_RESTORE:
+        case SQLITE_TESTCTRL_BYTEORDER:
+          if( nArg==2 ){
+            rc2 = sqlite3_test_control(testctrl);
+            isOk = testctrl==SQLITE_TESTCTRL_BYTEORDER ? 1 : 3;
+          }
+          break;
+
+        /* sqlite3_test_control(int, uint) */
+        case SQLITE_TESTCTRL_PENDING_BYTE:
+          if( nArg==3 ){
+            unsigned int opt = (unsigned int)integerValue(azArg[2]);
+            rc2 = sqlite3_test_control(testctrl, opt);
+            isOk = 3;
+          }
+          break;
+
+        /* sqlite3_test_control(int, int, sqlite3*) */
+        case SQLITE_TESTCTRL_PRNG_SEED:
+          if( nArg==3 || nArg==4 ){
+            int ii = (int)integerValue(azArg[2]);
+            sqlite3 *db;
+            if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){
+              sqlite3_randomness(sizeof(ii),&ii);
+              sqlite3_fprintf(stdout, "-- random seed: %d\n", ii);
+            }
+            if( nArg==3 ){
+              db = 0;
+            }else{
+              db = p->db;
+              /* Make sure the schema has been loaded */
+              sqlite3_table_column_metadata(db, 0, "x", 0, 0, 0, 0, 0, 0);
+            }
+            rc2 = sqlite3_test_control(testctrl, ii, db);
+            isOk = 3;
+          }
+          break;
+
+        /* sqlite3_test_control(int, int) */
+        case SQLITE_TESTCTRL_ASSERT:
+        case SQLITE_TESTCTRL_ALWAYS:
+          if( nArg==3 ){
+            int opt = booleanValue(azArg[2]);
+            rc2 = sqlite3_test_control(testctrl, opt);
+            isOk = 1;
+          }
+          break;
+
+        /* sqlite3_test_control(int, int) */
+        case SQLITE_TESTCTRL_LOCALTIME_FAULT:
+        case SQLITE_TESTCTRL_NEVER_CORRUPT:
+          if( nArg==3 ){
+            int opt = booleanValue(azArg[2]);
+            rc2 = sqlite3_test_control(testctrl, opt);
+            isOk = 3;
+          }
+          break;
+
+        /* sqlite3_test_control(sqlite3*) */
+        case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS:
+          rc2 = sqlite3_test_control(testctrl, p->db);
+          isOk = 3;
+          break;
+
+        case SQLITE_TESTCTRL_IMPOSTER:
+          if( nArg==5 ){
+            rc2 = sqlite3_test_control(testctrl, p->db,
+                          azArg[2],
+                          integerValue(azArg[3]),
+                          integerValue(azArg[4]));
+            isOk = 3;
+          }
+          break;
+
+        case SQLITE_TESTCTRL_SEEK_COUNT: {
+          u64 x = 0;
+          rc2 = sqlite3_test_control(testctrl, p->db, &x);
+          sqlite3_fprintf(p->out, "%llu\n", x);
+          isOk = 3;
+          break;
+        }
+
+#ifdef YYCOVERAGE
+        case SQLITE_TESTCTRL_PARSER_COVERAGE: {
+          if( nArg==2 ){
+            sqlite3_test_control(testctrl, p->out);
+            isOk = 3;
+          }
+          break;
+        }
+#endif
+#ifdef SQLITE_DEBUG
+        case SQLITE_TESTCTRL_TUNE: {
+          if( nArg==4 ){
+            int id = (int)integerValue(azArg[2]);
+            int val = (int)integerValue(azArg[3]);
+            sqlite3_test_control(testctrl, id, &val);
+            isOk = 3;
+          }else if( nArg==3 ){
+            int id = (int)integerValue(azArg[2]);
+            sqlite3_test_control(testctrl, -id, &rc2);
+            isOk = 1;
+          }else if( nArg==2 ){
+            int id = 1;
+            while(1){
+              int val = 0;
+              rc2 = sqlite3_test_control(testctrl, -id, &val);
+              if( rc2!=SQLITE_OK ) break;
+              if( id>1 ) sqlite3_fputs("  ", p->out);
+              sqlite3_fprintf(p->out, "%d: %d", id, val);
+              id++;
+            }
+            if( id>1 ) sqlite3_fputs("\n", p->out);
+            isOk = 3;
+          }
+          break;
+        }
+#endif
+        case SQLITE_TESTCTRL_SORTER_MMAP:
+          if( nArg==3 ){
+            int opt = (unsigned int)integerValue(azArg[2]);
+            rc2 = sqlite3_test_control(testctrl, p->db, opt);
+            isOk = 3;
+          }
+          break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
+        case SQLITE_TESTCTRL_FAULT_INSTALL: {
+          int kk;
+          int bShowHelp = nArg<=2;
+          isOk = 3;
+          for(kk=2; kk<nArg; kk++){
+            const char *z = azArg[kk];
+            if( z[0]=='-' && z[1]=='-' ) z++;
+            if( cli_strcmp(z,"off")==0 ){
+              sqlite3_test_control(testctrl, 0);
+            }else if( cli_strcmp(z,"on")==0 ){
+              faultsim_state.iCnt = faultsim_state.nSkip;
+              if( faultsim_state.iErr==0 ) faultsim_state.iErr = 1;
+              faultsim_state.nHit = 0;
+              sqlite3_test_control(testctrl, faultsim_callback);
+            }else if( cli_strcmp(z,"reset")==0 ){
+              faultsim_state.iCnt = faultsim_state.nSkip;
+              faultsim_state.nHit = 0;
+              sqlite3_test_control(testctrl, faultsim_callback);
+            }else if( cli_strcmp(z,"status")==0 ){
+              sqlite3_fprintf(p->out, "faultsim.iId:       %d\n",
+                              faultsim_state.iId);
+              sqlite3_fprintf(p->out, "faultsim.iErr:      %d\n",
+                              faultsim_state.iErr);
+              sqlite3_fprintf(p->out, "faultsim.iCnt:      %d\n",
+                              faultsim_state.iCnt);
+              sqlite3_fprintf(p->out, "faultsim.nHit:      %d\n",
+                              faultsim_state.nHit);
+              sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n",
+                              faultsim_state.iInterval);
+              sqlite3_fprintf(p->out, "faultsim.eVerbose:  %d\n",
+                              faultsim_state.eVerbose);
+              sqlite3_fprintf(p->out, "faultsim.nRepeat:   %d\n",
+                              faultsim_state.nRepeat);
+              sqlite3_fprintf(p->out, "faultsim.nSkip:     %d\n",
+                              faultsim_state.nSkip);
+            }else if( cli_strcmp(z,"-v")==0 ){
+              if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++;
+            }else if( cli_strcmp(z,"-q")==0 ){
+              if( faultsim_state.eVerbose>0 ) faultsim_state.eVerbose--;
+            }else if( cli_strcmp(z,"-id")==0 && kk+1<nArg ){
+              faultsim_state.iId = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-errcode")==0 && kk+1<nArg ){
+              faultsim_state.iErr = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-interval")==0 && kk+1<nArg ){
+              faultsim_state.iInterval = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-repeat")==0 && kk+1<nArg ){
+              faultsim_state.nRepeat = atoi(azArg[++kk]);
+           }else if( cli_strcmp(z,"-skip")==0 && kk+1<nArg ){
+              faultsim_state.nSkip = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){
+              bShowHelp = 1;
+            }else{
+              sqlite3_fprintf(stderr,
+                  "Unrecognized fault_install argument: \"%s\"\n",
+                  azArg[kk]);
+              rc = 1;
+              bShowHelp = 1;
+              break;
+            }
+          }
+          if( bShowHelp ){
+            sqlite3_fputs(
+               "Usage: .testctrl fault_install ARGS\n"
+               "Possible arguments:\n"
+               "   off               Disable faultsim\n"
+               "   on                Activate faultsim\n"
+               "   reset             Reset the trigger counter\n"
+               "   status            Show current status\n"
+               "   -v                Increase verbosity\n"
+               "   -q                Decrease verbosity\n"
+               "   --errcode N       When triggered, return N as error code\n"
+               "   --id ID           Trigger only for the ID specified\n"
+               "   --interval N      Trigger only after every N-th call\n"
+               "   --repeat N        Turn off after N hits.  0 means never\n"
+               "   --skip N          Skip the first N encounters\n"
+               ,p->out
+            );
+          }
+          break;
+        }
+      }
+    }
+    if( isOk==0 && iCtrl>=0 ){
+      sqlite3_fprintf(p->out,
+          "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
+      rc = 1;
+    }else if( isOk==1 ){
+      sqlite3_fprintf(p->out, "%d\n", rc2);
+    }else if( isOk==2 ){
+      sqlite3_fprintf(p->out, "0x%08x\n", rc2);
+    }
+  }else
+#endif /* !defined(SQLITE_UNTESTABLE) */
+
+  if( c=='t' && n>4 && cli_strncmp(azArg[0], "timeout", n)==0 ){
+    open_db(p, 0);
+    sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0);
+  }else
+
+  if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){
+    if( nArg==2 ){
+      enableTimer = booleanValue(azArg[1]);
+      if( enableTimer && !HAS_TIMER ){
+        eputz("Error: timer not available on this system.\n");
+        enableTimer = 0;
+      }
+    }else{
+      eputz("Usage: .timer on|off\n");
+      rc = 1;
+    }
+  }else
+
+#ifndef SQLITE_OMIT_TRACE
+  if( c=='t' && cli_strncmp(azArg[0], "trace", n)==0 ){
+    int mType = 0;
+    int jj;
+    open_db(p, 0);
+    for(jj=1; jj<nArg; jj++){
+      const char *z = azArg[jj];
+      if( z[0]=='-' ){
+        if( optionMatch(z, "expanded") ){
+          p->eTraceType = SHELL_TRACE_EXPANDED;
+        }
+#ifdef SQLITE_ENABLE_NORMALIZE
+        else if( optionMatch(z, "normalized") ){
+          p->eTraceType = SHELL_TRACE_NORMALIZED;
+        }
+#endif
+        else if( optionMatch(z, "plain") ){
+          p->eTraceType = SHELL_TRACE_PLAIN;
+        }
+        else if( optionMatch(z, "profile") ){
+          mType |= SQLITE_TRACE_PROFILE;
+        }
+        else if( optionMatch(z, "row") ){
+          mType |= SQLITE_TRACE_ROW;
+        }
+        else if( optionMatch(z, "stmt") ){
+          mType |= SQLITE_TRACE_STMT;
+        }
+        else if( optionMatch(z, "close") ){
+          mType |= SQLITE_TRACE_CLOSE;
+        }
+        else {
+          sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else{
+        output_file_close(p->traceOut);
+        p->traceOut = output_file_open(z);
+      }
+    }
+    if( p->traceOut==0 ){
+      sqlite3_trace_v2(p->db, 0, 0, 0);
+    }else{
+      if( mType==0 ) mType = SQLITE_TRACE_STMT;
+      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
+    }
+  }else
+#endif /* !defined(SQLITE_OMIT_TRACE) */
+
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+  if( c=='u' && cli_strncmp(azArg[0], "unmodule", n)==0 ){
+    int ii;
+    int lenOpt;
+    char *zOpt;
+    if( nArg<2 ){
+      eputz("Usage: .unmodule [--allexcept] NAME ...\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    zOpt = azArg[1];
+    if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++;
+    lenOpt = (int)strlen(zOpt);
+    if( lenOpt>=3 && cli_strncmp(zOpt, "-allexcept",lenOpt)==0 ){
+      assert( azArg[nArg]==0 );
+      sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0);
+    }else{
+      for(ii=1; ii<nArg; ii++){
+        sqlite3_create_module(p->db, azArg[ii], 0, 0);
+      }
+    }
+  }else
+#endif
+
+#if SQLITE_USER_AUTHENTICATION
+  if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){
+    if( nArg<2 ){
+      eputz("Usage: .user SUBCOMMAND ...\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    if( cli_strcmp(azArg[1],"login")==0 ){
+      if( nArg!=4 ){
+        eputz("Usage: .user login USER PASSWORD\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
+      rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3],
+                                     strlen30(azArg[3]));
+      if( rc ){
+        sqlite3_fprintf(stderr,"Authentication failed for user %s\n", azArg[2]);
+        rc = 1;
+      }
+    }else if( cli_strcmp(azArg[1],"add")==0 ){
+      if( nArg!=5 ){
+        eputz("Usage: .user add USER PASSWORD ISADMIN\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
+      rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
+                            booleanValue(azArg[4]));
+      if( rc ){
+        sqlite3_fprintf(stderr,"User-Add failed: %d\n", rc);
+        rc = 1;
+      }
+    }else if( cli_strcmp(azArg[1],"edit")==0 ){
+      if( nArg!=5 ){
+        eputz("Usage: .user edit USER PASSWORD ISADMIN\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
+      rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
+                              booleanValue(azArg[4]));
+      if( rc ){
+        sqlite3_fprintf(stderr,"User-Edit failed: %d\n", rc);
+        rc = 1;
+      }
+    }else if( cli_strcmp(azArg[1],"delete")==0 ){
+      if( nArg!=3 ){
+        eputz("Usage: .user delete USER\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
+      rc = sqlite3_user_delete(p->db, azArg[2]);
+      if( rc ){
+        sqlite3_fprintf(stderr,"User-Delete failed: %d\n", rc);
+        rc = 1;
+      }
+    }else{
+      eputz("Usage: .user login|add|edit|delete ...\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+  }else
+#endif /* SQLITE_USER_AUTHENTICATION */
+
+  if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){
+    char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit";
+    sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
+          sqlite3_libversion(), sqlite3_sourceid());
+#if SQLITE_HAVE_ZLIB
+    sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion());
+#endif
+#define CTIMEOPT_VAL_(opt) #opt
+#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt)
+#if defined(__clang__) && defined(__clang_major__)
+    sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "."
+          CTIMEOPT_VAL(__clang_minor__) "."
+          CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz);
+#elif defined(_MSC_VER)
+    sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz);
+#elif defined(__GNUC__) && defined(__VERSION__)
+    sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz);
+#endif
+  }else
+
+  if( c=='v' && cli_strncmp(azArg[0], "vfsinfo", n)==0 ){
+    const char *zDbName = nArg==2 ? azArg[1] : "main";
+    sqlite3_vfs *pVfs = 0;
+    if( p->db ){
+      sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs);
+      if( pVfs ){
+        sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
+        sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+        sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+        sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
+      }
+    }
+  }else
+
+  if( c=='v' && cli_strncmp(azArg[0], "vfslist", n)==0 ){
+    sqlite3_vfs *pVfs;
+    sqlite3_vfs *pCurrent = 0;
+    if( p->db ){
+      sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent);
+    }
+    for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){
+      sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
+            pVfs==pCurrent ? "  <--- CURRENT" : "");
+      sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+      sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+      sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
+      if( pVfs->pNext ){
+        sqlite3_fputs("-----------------------------------\n", p->out);
+      }
+    }
+  }else
+
+  if( c=='v' && cli_strncmp(azArg[0], "vfsname", n)==0 ){
+    const char *zDbName = nArg==2 ? azArg[1] : "main";
+    char *zVfsName = 0;
+    if( p->db ){
+      sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName);
+      if( zVfsName ){
+        sqlite3_fprintf(p->out, "%s\n", zVfsName);
+        sqlite3_free(zVfsName);
+      }
+    }
+  }else
+
+  if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){
+    unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff;
+    sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x);
+  }else
+
+  if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){
+    int j;
+    assert( nArg<=ArraySize(azArg) );
+    p->nWidth = nArg-1;
+    p->colWidth = realloc(p->colWidth, (p->nWidth+1)*sizeof(int)*2);
+    if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory();
+    if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth];
+    for(j=1; j<nArg; j++){
+      p->colWidth[j-1] = (int)integerValue(azArg[j]);
+    }
+  }else
+
+  {
+    sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: "
+          " \"%s\". Enter \".help\" for help\n", azArg[0]);
+    rc = 1;
+  }
+
+meta_command_exit:
+  if( p->outCount ){
+    p->outCount--;
+    if( p->outCount==0 ) output_reset(p);
+  }
+  p->bSafeMode = p->bSafeModePersist;
+  return rc;
+}
+
+/* Line scan result and intermediate states (supporting scan resumption)
+*/
+#ifndef CHAR_BIT
+# define CHAR_BIT 8
+#endif
+typedef enum {
+  QSS_HasDark = 1<<CHAR_BIT, QSS_EndingSemi = 2<<CHAR_BIT,
+  QSS_CharMask = (1<<CHAR_BIT)-1, QSS_ScanMask = 3<<CHAR_BIT,
+  QSS_Start = 0
+} QuickScanState;
+#define QSS_SETV(qss, newst) ((newst) | ((qss) & QSS_ScanMask))
+#define QSS_INPLAIN(qss) (((qss)&QSS_CharMask)==QSS_Start)
+#define QSS_PLAINWHITE(qss) (((qss)&~QSS_EndingSemi)==QSS_Start)
+#define QSS_PLAINDARK(qss) (((qss)&~QSS_EndingSemi)==QSS_HasDark)
+#define QSS_SEMITERM(qss) (((qss)&~QSS_HasDark)==QSS_EndingSemi)
+
+/*
+** Scan line for classification to guide shell's handling.
+** The scan is resumable for subsequent lines when prior
+** return values are passed as the 2nd argument.
+*/
+static QuickScanState quickscan(char *zLine, QuickScanState qss,
+                                SCAN_TRACKER_REFTYPE pst){
+  char cin;
+  char cWait = (char)qss; /* intentional narrowing loss */
+  if( cWait==0 ){
+  PlainScan:
+    assert( cWait==0 );
+    while( (cin = *zLine++)!=0 ){
+      if( IsSpace(cin) )
+        continue;
+      switch (cin){
+      case '-':
+        if( *zLine!='-' )
+          break;
+        while((cin = *++zLine)!=0 )
+          if( cin=='\n')
+            goto PlainScan;
+        return qss;
+      case ';':
+        qss |= QSS_EndingSemi;
+        continue;
+      case '/':
+        if( *zLine=='*' ){
+          ++zLine;
+          cWait = '*';
+          CONTINUE_PROMPT_AWAITS(pst, "/*");
+          qss = QSS_SETV(qss, cWait);
+          goto TermScan;
+        }
+        break;
+      case '[':
+        cin = ']';
+        deliberate_fall_through;
+      case '`': case '\'': case '"':
+        cWait = cin;
+        qss = QSS_HasDark | cWait;
+        CONTINUE_PROMPT_AWAITC(pst, cin);
+        goto TermScan;
+      case '(':
+        CONTINUE_PAREN_INCR(pst, 1);
+        break;
+      case ')':
+        CONTINUE_PAREN_INCR(pst, -1);
+        break;
+      default:
+        break;
+      }
+      qss = (qss & ~QSS_EndingSemi) | QSS_HasDark;
+    }
+  }else{
+  TermScan:
+    while( (cin = *zLine++)!=0 ){
+      if( cin==cWait ){
+        switch( cWait ){
+        case '*':
+          if( *zLine != '/' )
+            continue;
+          ++zLine;
+          cWait = 0;
+          CONTINUE_PROMPT_AWAITC(pst, 0);
+          qss = QSS_SETV(qss, 0);
+          goto PlainScan;
+        case '`': case '\'': case '"':
+          if(*zLine==cWait){
+            /* Swallow doubled end-delimiter.*/
+            ++zLine;
+            continue;
+          }
+          deliberate_fall_through;
+        case ']':
+          cWait = 0;
+          CONTINUE_PROMPT_AWAITC(pst, 0);
+          qss = QSS_SETV(qss, 0);
+          goto PlainScan;
+        default: assert(0);
+        }
+      }
+    }
+  }
+  return qss;
+}
+
+/*
+** Return TRUE if the line typed in is an SQL command terminator other
+** than a semi-colon.  The SQL Server style "go" command is understood
+** as is the Oracle "/".
+*/
+static int line_is_command_terminator(char *zLine){
+  while( IsSpace(zLine[0]) ){ zLine++; };
+  if( zLine[0]=='/' )
+    zLine += 1; /* Oracle */
+  else if ( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o' )
+    zLine += 2; /* SQL Server */
+  else
+    return 0;
+  return quickscan(zLine, QSS_Start, 0)==QSS_Start;
+}
+
+/*
+** The CLI needs a working sqlite3_complete() to work properly.  So error
+** out of the build if compiling with SQLITE_OMIT_COMPLETE.
+*/
+#ifdef SQLITE_OMIT_COMPLETE
+# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE.
+#endif
+
+/*
+** Return true if zSql is a complete SQL statement.  Return false if it
+** ends in the middle of a string literal or C-style comment.
+*/
+static int line_is_complete(char *zSql, int nSql){
+  int rc;
+  if( zSql==0 ) return 1;
+  zSql[nSql] = ';';
+  zSql[nSql+1] = 0;
+  rc = sqlite3_complete(zSql);
+  zSql[nSql] = 0;
+  return rc;
+}
+
+/*
+** This function is called after processing each line of SQL in the
+** runOneSqlLine() function. Its purpose is to detect scenarios where
+** defensive mode should be automatically turned off. Specifically, when
+**
+**   1. The first line of input is "PRAGMA foreign_keys=OFF;",
+**   2. The second line of input is "BEGIN TRANSACTION;",
+**   3. The database is empty, and
+**   4. The shell is not running in --safe mode.
+** 
+** The implementation uses the ShellState.eRestoreState to maintain state:
+**
+**    0: Have not seen any SQL.
+**    1: Have seen "PRAGMA foreign_keys=OFF;".
+**    2-6: Currently running .dump transaction. If the "2" bit is set,
+**         disable DEFENSIVE when done. If "4" is set, disable DQS_DDL.
+**    7: Nothing left to do. This function becomes a no-op.
+*/
+static int doAutoDetectRestore(ShellState *p, const char *zSql){
+  int rc = SQLITE_OK;
+
+  if( p->eRestoreState<7 ){
+    switch( p->eRestoreState ){
+      case 0: {
+        const char *zExpect = "PRAGMA foreign_keys=OFF;";
+        assert( strlen(zExpect)==24 );
+        if( p->bSafeMode==0
+         && strlen(zSql)>=24
+         && memcmp(zSql, zExpect, 25)==0
+        ){
+          p->eRestoreState = 1;
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      };
+  
+      case 1: {
+        int bIsDump = 0;
+        const char *zExpect = "BEGIN TRANSACTION;";
+        assert( strlen(zExpect)==18 );
+        if( memcmp(zSql, zExpect, 19)==0 ){
+          /* Now check if the database is empty. */
+          const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1";
+          sqlite3_stmt *pStmt = 0;
+  
+          bIsDump = 1;
+          shellPrepare(p->db, &rc, zQuery, &pStmt);
+          if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+            bIsDump = 0;
+          }
+          shellFinalize(&rc, pStmt);
+        }
+        if( bIsDump && rc==SQLITE_OK ){
+          int bDefense = 0;
+          int bDqsDdl = 0;
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, -1, &bDqsDdl);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 1, 0);
+          p->eRestoreState = (bDefense ? 2 : 0) + (bDqsDdl ? 4 : 0);
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+  
+      default: {
+        if( sqlite3_get_autocommit(p->db) ){
+          if( (p->eRestoreState & 2) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
+          }
+          if( (p->eRestoreState & 4) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 0, 0);
+          }
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Run a single line of SQL.  Return the number of errors.
+*/
+static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
+  int rc;
+  char *zErrMsg = 0;
+
+  open_db(p, 0);
+  if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql);
+  if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0;
+  BEGIN_TIMER;
+  rc = shell_exec(p, zSql, &zErrMsg);
+  END_TIMER(p->out);
+  if( rc || zErrMsg ){
+    char zPrefix[100];
+    const char *zErrorTail;
+    const char *zErrorType;
+    if( zErrMsg==0 ){
+      zErrorType = "Error";
+      zErrorTail = sqlite3_errmsg(p->db);
+    }else if( cli_strncmp(zErrMsg, "in prepare, ",12)==0 ){
+      zErrorType = "Parse error";
+      zErrorTail = &zErrMsg[12];
+    }else if( cli_strncmp(zErrMsg, "stepping, ", 10)==0 ){
+      zErrorType = "Runtime error";
+      zErrorTail = &zErrMsg[10];
+    }else{
+      zErrorType = "Error";
+      zErrorTail = zErrMsg;
+    }
+    if( in!=0 || !stdin_is_interactive ){
+      sqlite3_snprintf(sizeof(zPrefix), zPrefix,
+                       "%s near line %d:", zErrorType, startline);
+    }else{
+      sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType);
+    }
+    sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail);
+    sqlite3_free(zErrMsg);
+    zErrMsg = 0;
+    return 1;
+  }else if( ShellHasFlag(p, SHFLG_CountChanges) ){
+    char zLineBuf[2000];
+    sqlite3_snprintf(sizeof(zLineBuf), zLineBuf,
+            "changes: %lld   total_changes: %lld",
+            sqlite3_changes64(p->db), sqlite3_total_changes64(p->db));
+    sqlite3_fprintf(p->out, "%s\n", zLineBuf);
+  }
+
+  if( doAutoDetectRestore(p, zSql) ) return 1;
+  return 0;
+}
+
+static void echo_group_input(ShellState *p, const char *zDo){
+  if( ShellHasFlag(p, SHFLG_Echo) ) sqlite3_fprintf(p->out, "%s\n", zDo);
+}
+
+#ifdef SQLITE_SHELL_FIDDLE
+/*
+** Alternate one_input_line() impl for wasm mode. This is not in the primary
+** impl because we need the global shellState and cannot access it from that
+** function without moving lots of code around (creating a larger/messier diff).
+*/
+static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
+  /* Parse the next line from shellState.wasm.zInput. */
+  const char *zBegin = shellState.wasm.zPos;
+  const char *z = zBegin;
+  char *zLine = 0;
+  i64 nZ = 0;
+
+  UNUSED_PARAMETER(in);
+  UNUSED_PARAMETER(isContinuation);
+  if(!z || !*z){
+    return 0;
+  }
+  while(*z && isspace(*z)) ++z;
+  zBegin = z;
+  for(; *z && '\n'!=*z; ++nZ, ++z){}
+  if(nZ>0 && '\r'==zBegin[nZ-1]){
+    --nZ;
+  }
+  shellState.wasm.zPos = z;
+  zLine = realloc(zPrior, nZ+1);
+  shell_check_oom(zLine);
+  memcpy(zLine, zBegin, nZ);
+  zLine[nZ] = 0;
+  return zLine;
+}
+#endif /* SQLITE_SHELL_FIDDLE */
+
+/*
+** Read input from *in and process it.  If *in==0 then input
+** is interactive - the user is typing it it.  Otherwise, input
+** is coming from a file or device.  A prompt is issued and history
+** is saved only if input is interactive.  An interrupt signal will
+** cause this routine to exit immediately, unless input is interactive.
+**
+** Return the number of errors.
+*/
+static int process_input(ShellState *p){
+  char *zLine = 0;          /* A single input line */
+  char *zSql = 0;           /* Accumulated SQL text */
+  i64 nLine;                /* Length of current line */
+  i64 nSql = 0;             /* Bytes of zSql[] used */
+  i64 nAlloc = 0;           /* Allocated zSql[] space */
+  int rc;                   /* Error code */
+  int errCnt = 0;           /* Number of errors seen */
+  i64 startline = 0;        /* Line number for start of current input */
+  QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */
+
+  if( p->inputNesting==MAX_INPUT_NESTING ){
+    /* This will be more informative in a later version. */
+    sqlite3_fprintf(stderr,"Input nesting limit (%d) reached at line %d."
+          " Check recursion.\n", MAX_INPUT_NESTING, p->lineno);
+    return 1;
+  }
+  ++p->inputNesting;
+  p->lineno = 0;
+  CONTINUE_PROMPT_RESET;
+  while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){
+    fflush(p->out);
+    zLine = one_input_line(p->in, zLine, nSql>0);
+    if( zLine==0 ){
+      /* End of input */
+      if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out);
+      break;
+    }
+    if( seenInterrupt ){
+      if( p->in!=0 ) break;
+      seenInterrupt = 0;
+    }
+    p->lineno++;
+    if( QSS_INPLAIN(qss)
+        && line_is_command_terminator(zLine)
+        && line_is_complete(zSql, nSql) ){
+      memcpy(zLine,";",2);
+    }
+    qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE);
+    if( QSS_PLAINWHITE(qss) && nSql==0 ){
+      /* Just swallow single-line whitespace */
+      echo_group_input(p, zLine);
+      qss = QSS_Start;
+      continue;
+    }
+    if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){
+      CONTINUE_PROMPT_RESET;
+      echo_group_input(p, zLine);
+      if( zLine[0]=='.' ){
+        rc = do_meta_command(zLine, p);
+        if( rc==2 ){ /* exit requested */
+          break;
+        }else if( rc ){
+          errCnt++;
+        }
+      }
+      qss = QSS_Start;
+      continue;
+    }
+    /* No single-line dispositions remain; accumulate line(s). */
+    nLine = strlen(zLine);
+    if( nSql+nLine+2>=nAlloc ){
+      /* Grow buffer by half-again increments when big. */
+      nAlloc = nSql+(nSql>>1)+nLine+100;
+      zSql = realloc(zSql, nAlloc);
+      shell_check_oom(zSql);
+    }
+    if( nSql==0 ){
+      i64 i;
+      for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
+      assert( nAlloc>0 && zSql!=0 );
+      memcpy(zSql, zLine+i, nLine+1-i);
+      startline = p->lineno;
+      nSql = nLine-i;
+    }else{
+      zSql[nSql++] = '\n';
+      memcpy(zSql+nSql, zLine, nLine+1);
+      nSql += nLine;
+    }
+    if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){
+      echo_group_input(p, zSql);
+      errCnt += runOneSqlLine(p, zSql, p->in, startline);
+      CONTINUE_PROMPT_RESET;
+      nSql = 0;
+      if( p->outCount ){
+        output_reset(p);
+        p->outCount = 0;
+      }else{
+        clearTempFile(p);
+      }
+      p->bSafeMode = p->bSafeModePersist;
+      qss = QSS_Start;
+    }else if( nSql && QSS_PLAINWHITE(qss) ){
+      echo_group_input(p, zSql);
+      nSql = 0;
+      qss = QSS_Start;
+    }
+  }
+  if( nSql ){
+    /* This may be incomplete. Let the SQL parser deal with that. */
+    echo_group_input(p, zSql);
+    errCnt += runOneSqlLine(p, zSql, p->in, startline);
+    CONTINUE_PROMPT_RESET;
+  }
+  free(zSql);
+  free(zLine);
+  --p->inputNesting;
+  return errCnt>0;
+}
+
+/*
+** Return a pathname which is the user's home directory.  A
+** 0 return indicates an error of some kind.
+*/
+static char *find_home_dir(int clearFlag){
+  static char *home_dir = NULL;
+  if( clearFlag ){
+    free(home_dir);
+    home_dir = 0;
+    return 0;
+  }
+  if( home_dir ) return home_dir;
+
+#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \
+     && !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI)
+  {
+    struct passwd *pwent;
+    uid_t uid = getuid();
+    if( (pwent=getpwuid(uid)) != NULL) {
+      home_dir = pwent->pw_dir;
+    }
+  }
+#endif
+
+#if defined(_WIN32_WCE)
+  /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv()
+   */
+  home_dir = "/";
+#else
+
+#if defined(_WIN32) || defined(WIN32)
+  if (!home_dir) {
+    home_dir = getenv("USERPROFILE");
+  }
+#endif
+
+  if (!home_dir) {
+    home_dir = getenv("HOME");
+  }
+
+#if defined(_WIN32) || defined(WIN32)
+  if (!home_dir) {
+    char *zDrive, *zPath;
+    int n;
+    zDrive = getenv("HOMEDRIVE");
+    zPath = getenv("HOMEPATH");
+    if( zDrive && zPath ){
+      n = strlen30(zDrive) + strlen30(zPath) + 1;
+      home_dir = malloc( n );
+      if( home_dir==0 ) return 0;
+      sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath);
+      return home_dir;
+    }
+    home_dir = "c:\\";
+  }
+#endif
+
+#endif /* !_WIN32_WCE */
+
+  if( home_dir ){
+    i64 n = strlen(home_dir) + 1;
+    char *z = malloc( n );
+    if( z ) memcpy(z, home_dir, n);
+    home_dir = z;
+  }
+
+  return home_dir;
+}
+
+/*
+** On non-Windows platforms, look for $XDG_CONFIG_HOME.
+** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return
+** the path to it.  If there is no $(XDG_CONFIG_HOME) then
+** look for $(HOME)/.config/sqlite3/sqliterc and if found
+** return that.  If none of these are found, return 0.
+**
+** The string returned is obtained from sqlite3_malloc() and
+** should be freed by the caller.
+*/
+static char *find_xdg_config(void){
+#if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \
+     || defined(__RTP__) || defined(_WRS_KERNEL)
+  return 0;
+#else
+  char *zConfig = 0;
+  const char *zXdgHome;
+
+  zXdgHome = getenv("XDG_CONFIG_HOME");
+  if( zXdgHome==0 ){
+    const char *zHome = getenv("HOME");
+    if( zHome==0 ) return 0;
+    zConfig = sqlite3_mprintf("%s/.config/sqlite3/sqliterc", zHome);
+  }else{
+    zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome);
+  }
+  shell_check_oom(zConfig);
+  if( access(zConfig,0)!=0 ){
+    sqlite3_free(zConfig);
+    zConfig = 0;
+  }
+  return zConfig;
+#endif
+}
+
+/*
+** Read input from the file given by sqliterc_override.  Or if that
+** parameter is NULL, take input from the first of find_xdg_config()
+** or ~/.sqliterc which is found.
+**
+** Returns the number of errors.
+*/
+static void process_sqliterc(
+  ShellState *p,                  /* Configuration data */
+  const char *sqliterc_override   /* Name of config file. NULL to use default */
+){
+  char *home_dir = NULL;
+  const char *sqliterc = sqliterc_override;
+  char *zBuf = 0;
+  FILE *inSaved = p->in;
+  int savedLineno = p->lineno;
+
+  if( sqliterc == NULL ){
+    sqliterc = zBuf = find_xdg_config();
+  }
+  if( sqliterc == NULL ){
+    home_dir = find_home_dir(0);
+    if( home_dir==0 ){
+      eputz("-- warning: cannot find home directory;"
+            " cannot read ~/.sqliterc\n");
+      return;
+    }
+    zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
+    shell_check_oom(zBuf);
+    sqliterc = zBuf;
+  }
+  p->in = sqlite3_fopen(sqliterc,"rb");
+  if( p->in ){
+    if( stdin_is_interactive ){
+      sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc);
+    }
+    if( process_input(p) && bail_on_error ) exit(1);
+    fclose(p->in);
+  }else if( sqliterc_override!=0 ){
+    sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc);
+    if( bail_on_error ) exit(1);
+  }
+  p->in = inSaved;
+  p->lineno = savedLineno;
+  sqlite3_free(zBuf);
+}
+
+/*
+** Show available command line options
+*/
+static const char zOptions[] =
+  "   --                   treat no subsequent arguments as options\n"
+#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+  "   -A ARGS...           run \".archive ARGS\" and exit\n"
+#endif
+  "   -append              append the database to the end of the file\n"
+  "   -ascii               set output mode to 'ascii'\n"
+  "   -bail                stop after hitting an error\n"
+  "   -batch               force batch I/O\n"
+  "   -box                 set output mode to 'box'\n"
+  "   -column              set output mode to 'column'\n"
+  "   -cmd COMMAND         run \"COMMAND\" before reading stdin\n"
+  "   -csv                 set output mode to 'csv'\n"
+#if !defined(SQLITE_OMIT_DESERIALIZE)
+  "   -deserialize         open the database using sqlite3_deserialize()\n"
+#endif
+  "   -echo                print inputs before execution\n"
+  "   -init FILENAME       read/process named file\n"
+  "   -[no]header          turn headers on or off\n"
+#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
+  "   -heap SIZE           Size of heap for memsys3 or memsys5\n"
+#endif
+  "   -help                show this message\n"
+  "   -html                set output mode to HTML\n"
+  "   -interactive         force interactive I/O\n"
+  "   -json                set output mode to 'json'\n"
+  "   -line                set output mode to 'line'\n"
+  "   -list                set output mode to 'list'\n"
+  "   -lookaside SIZE N    use N entries of SZ bytes for lookaside memory\n"
+  "   -markdown            set output mode to 'markdown'\n"
+#if !defined(SQLITE_OMIT_DESERIALIZE)
+  "   -maxsize N           maximum size for a --deserialize database\n"
+#endif
+  "   -memtrace            trace all memory allocations and deallocations\n"
+  "   -mmap N              default mmap size set to N\n"
+#ifdef SQLITE_ENABLE_MULTIPLEX
+  "   -multiplex           enable the multiplexor VFS\n"
+#endif
+  "   -newline SEP         set output row separator. Default: '\\n'\n"
+  "   -nofollow            refuse to open symbolic links to database files\n"
+  "   -nonce STRING        set the safe-mode escape nonce\n"
+  "   -no-rowid-in-view    Disable rowid-in-view using sqlite3_config()\n"
+  "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
+  "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
+  "   -pcachetrace         trace all page cache operations\n"
+  "   -quote               set output mode to 'quote'\n"
+  "   -readonly            open the database read-only\n"
+  "   -safe                enable safe-mode\n"
+  "   -separator SEP       set output column separator. Default: '|'\n"
+#ifdef SQLITE_ENABLE_SORTER_REFERENCES
+  "   -sorterref SIZE      sorter references threshold size\n"
+#endif
+  "   -stats               print memory stats before each finalize\n"
+  "   -table               set output mode to 'table'\n"
+  "   -tabs                set output mode to 'tabs'\n"
+  "   -unsafe-testing      allow unsafe commands and modes for testing\n"
+  "   -version             show SQLite version\n"
+  "   -vfs NAME            use NAME as the default VFS\n"
+  "   -vfstrace            enable tracing of all VFS calls\n"
+#ifdef SQLITE_HAVE_ZLIB
+  "   -zip                 open the file as a ZIP Archive\n"
+#endif
+;
+static void usage(int showDetail){
+  sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL]]\n"
+       "FILENAME is the name of an SQLite database. A new database is created\n"
+       "if the file does not previously exist. Defaults to :memory:.\n", Argv0);
+  if( showDetail ){
+    sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions);
+  }else{
+    eputz("Use the -help option for additional information\n");
+  }
+  exit(0);
+}
+
+/*
+** Internal check:  Verify that the SQLite is uninitialized.  Print a
+** error message if it is initialized.
+*/
+static void verify_uninitialized(void){
+  if( sqlite3_config(-1)==SQLITE_MISUSE ){
+    sputz(stdout, "WARNING: attempt to configure SQLite after"
+          " initialization.\n");
+  }
+}
+
+/*
+** Initialize the state information in data
+*/
+static void main_init(ShellState *data) {
+  memset(data, 0, sizeof(*data));
+  data->normalMode = data->cMode = data->mode = MODE_List;
+  data->autoExplain = 1;
+#ifdef _WIN32
+  data->crlfMode = 1;
+#endif
+  data->pAuxDb = &data->aAuxDb[0];
+  memcpy(data->colSeparator,SEP_Column, 2);
+  memcpy(data->rowSeparator,SEP_Row, 2);
+  data->showHeader = 0;
+  data->shellFlgs = SHFLG_Lookaside;
+  sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
+#if !defined(SQLITE_SHELL_FIDDLE)
+  verify_uninitialized();
+#endif
+  sqlite3_config(SQLITE_CONFIG_URI, 1);
+  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
+  sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
+  sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
+}
+
+/*
+** Output text to the console in a font that attracts extra attention.
+*/
+#if defined(_WIN32) || defined(WIN32)
+static void printBold(const char *zText){
+#if !SQLITE_OS_WINRT
+  HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE);
+  CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo;
+  GetConsoleScreenBufferInfo(out, &defaultScreenInfo);
+  SetConsoleTextAttribute(out,
+         FOREGROUND_RED|FOREGROUND_INTENSITY
+  );
+#endif
+  sputz(stdout, zText);
+#if !SQLITE_OS_WINRT
+  SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
+#endif
+}
+#else
+static void printBold(const char *zText){
+  sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText);
+}
+#endif
+
+/*
+** Get the argument to an --option.  Throw an error and die if no argument
+** is available.
+*/
+static char *cmdline_option_value(int argc, char **argv, int i){
+  if( i==argc ){
+    sqlite3_fprintf(stderr,
+            "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]);
+    exit(1);
+  }
+  return argv[i];
+}
+
+static void sayAbnormalExit(void){
+  if( seenInterrupt ) eputz("Program interrupted.\n");
+}
+
+#ifndef SQLITE_SHELL_IS_UTF8
+#  if (defined(_WIN32) || defined(WIN32)) \
+   && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__)))
+#    define SQLITE_SHELL_IS_UTF8          (0)
+#  else
+#    define SQLITE_SHELL_IS_UTF8          (1)
+#  endif
+#endif
+
+#ifdef SQLITE_SHELL_FIDDLE
+#  define main fiddle_main
+#endif
+
+#if SQLITE_SHELL_IS_UTF8
+int SQLITE_CDECL main(int argc, char **argv){
+#else
+int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+  char **argv;
+#endif
+#ifdef SQLITE_DEBUG
+  sqlite3_int64 mem_main_enter = 0;
+#endif
+  char *zErrMsg = 0;
+#ifdef SQLITE_SHELL_FIDDLE
+#  define data shellState
+#else
+  ShellState data;
+#endif
+  const char *zInitFile = 0;
+  int i;
+  int rc = 0;
+  int warnInmemoryDb = 0;
+  int readStdin = 1;
+  int nCmd = 0;
+  int nOptsEnd = argc;
+  int bEnableVfstrace = 0;
+  char **azCmd = 0;
+  const char *zVfs = 0;           /* Value of -vfs command-line option */
+#if !SQLITE_SHELL_IS_UTF8
+  char **argvToFree = 0;
+  int argcToFree = 0;
+#endif
+  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
+
+#ifdef SQLITE_SHELL_FIDDLE
+  stdin_is_interactive = 0;
+  stdout_is_console = 1;
+  data.wasm.zDefaultDbName = "/fiddle.sqlite3";
+#else
+  stdin_is_interactive = isatty(0);
+  stdout_is_console = isatty(1);
+#endif
+  atexit(sayAbnormalExit);
+#ifdef SQLITE_DEBUG
+  mem_main_enter = sqlite3_memory_used();
+#endif
+#if !defined(_WIN32_WCE)
+  if( getenv("SQLITE_DEBUG_BREAK") ){
+    if( isatty(0) && isatty(2) ){
+      char zLine[100];
+      sqlite3_fprintf(stderr,
+            "attach debugger to process %d and press ENTER to continue...",
+            GETPID());
+      if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0
+       && cli_strcmp(zLine,"stop")==0
+      ){
+        exit(1);
+      }
+    }else{
+#if defined(_WIN32) || defined(WIN32)
+#if SQLITE_OS_WINRT
+      __debugbreak();
+#else
+      DebugBreak();
+#endif
+#elif defined(SIGTRAP)
+      raise(SIGTRAP);
+#endif
+    }
+  }
+#endif
+  /* Register a valid signal handler early, before much else is done. */
+#ifdef SIGINT
+  signal(SIGINT, interrupt_handler);
+#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
+  if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){
+    eputz("No ^C handler.\n");
+  }
+#endif
+
+#if USE_SYSTEM_SQLITE+0!=1
+  if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
+    sqlite3_fprintf(stderr,
+          "SQLite header and source version mismatch\n%s\n%s\n",
+          sqlite3_sourceid(), SQLITE_SOURCE_ID);
+    exit(1);
+  }
+#endif
+  main_init(&data);
+
+  /* On Windows, we must translate command-line arguments into UTF-8.
+  ** The SQLite memory allocator subsystem has to be enabled in order to
+  ** do this.  But we want to run an sqlite3_shutdown() afterwards so that
+  ** subsequent sqlite3_config() calls will work.  So copy all results into
+  ** memory that does not come from the SQLite memory allocator.
+  */
+#if !SQLITE_SHELL_IS_UTF8
+  sqlite3_initialize();
+  argvToFree = malloc(sizeof(argv[0])*argc*2);
+  shell_check_oom(argvToFree);
+  argcToFree = argc;
+  argv = argvToFree + argc;
+  for(i=0; i<argc; i++){
+    char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    i64 n;
+    shell_check_oom(z);
+    n = strlen(z);
+    argv[i] = malloc( n+1 );
+    shell_check_oom(argv[i]);
+    memcpy(argv[i], z, n+1);
+    argvToFree[i] = argv[i];
+    sqlite3_free(z);
+  }
+  sqlite3_shutdown();
+#endif
+
+  assert( argc>=1 && argv && argv[0] );
+  Argv0 = argv[0];
+
+#ifdef SQLITE_SHELL_DBNAME_PROC
+  {
+    /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name
+    ** of a C-function that will provide the name of the database file.  Use
+    ** this compile-time option to embed this shell program in larger
+    ** applications. */
+    extern void SQLITE_SHELL_DBNAME_PROC(const char**);
+    SQLITE_SHELL_DBNAME_PROC(&data.pAuxDb->zDbFilename);
+    warnInmemoryDb = 0;
+  }
+#endif
+
+  /* Do an initial pass through the command-line argument to locate
+  ** the name of the database file, the name of the initialization file,
+  ** the size of the alternative malloc heap, options affecting commands
+  ** or SQL run from the command line, and the first command to execute.
+  */
+#ifndef SQLITE_SHELL_FIDDLE
+  verify_uninitialized();
+#endif
+  for(i=1; i<argc; i++){
+    char *z;
+    z = argv[i];
+    if( z[0]!='-' || i>nOptsEnd ){
+      if( data.aAuxDb->zDbFilename==0 ){
+        data.aAuxDb->zDbFilename = z;
+      }else{
+        /* Excess arguments are interpreted as SQL (or dot-commands) and
+        ** mean that nothing is read from stdin */
+        readStdin = 0;
+        nCmd++;
+        azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd);
+        shell_check_oom(azCmd);
+        azCmd[nCmd-1] = z;
+      }
+      continue;
+    }
+    if( z[1]=='-' ) z++;
+    if( cli_strcmp(z, "-")==0 ){
+      nOptsEnd = i;
+      continue;
+    }else if( cli_strcmp(z,"-separator")==0
+     || cli_strcmp(z,"-nullvalue")==0
+     || cli_strcmp(z,"-newline")==0
+     || cli_strcmp(z,"-cmd")==0
+    ){
+      (void)cmdline_option_value(argc, argv, ++i);
+    }else if( cli_strcmp(z,"-init")==0 ){
+      zInitFile = cmdline_option_value(argc, argv, ++i);
+    }else if( cli_strcmp(z,"-interactive")==0 ){
+    }else if( cli_strcmp(z,"-batch")==0 ){
+      /* Need to check for batch mode here to so we can avoid printing
+      ** informational messages (like from process_sqliterc) before
+      ** we do the actual processing of arguments later in a second pass.
+      */
+      stdin_is_interactive = 0;
+    }else if( cli_strcmp(z,"-utf8")==0 ){
+    }else if( cli_strcmp(z,"-no-utf8")==0 ){
+    }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){
+      int val = 0;
+      sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val);
+      assert( val==0 );
+    }else if( cli_strcmp(z,"-heap")==0 ){
+#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
+      const char *zSize;
+      sqlite3_int64 szHeap;
+
+      zSize = cmdline_option_value(argc, argv, ++i);
+      szHeap = integerValue(zSize);
+      if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000;
+      verify_uninitialized();
+      sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64);
+#else
+      (void)cmdline_option_value(argc, argv, ++i);
+#endif
+    }else if( cli_strcmp(z,"-pagecache")==0 ){
+      sqlite3_int64 n, sz;
+      sz = integerValue(cmdline_option_value(argc,argv,++i));
+      if( sz>70000 ) sz = 70000;
+      if( sz<0 ) sz = 0;
+      n = integerValue(cmdline_option_value(argc,argv,++i));
+      if( sz>0 && n>0 && 0xffffffffffffLL/sz<n ){
+        n = 0xffffffffffffLL/sz;
+      }
+      verify_uninitialized();
+      sqlite3_config(SQLITE_CONFIG_PAGECACHE,
+                    (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
+      data.shellFlgs |= SHFLG_Pagecache;
+    }else if( cli_strcmp(z,"-lookaside")==0 ){
+      int n, sz;
+      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
+      if( sz<0 ) sz = 0;
+      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
+      if( n<0 ) n = 0;
+      verify_uninitialized();
+      sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n);
+      if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside;
+    }else if( cli_strcmp(z,"-threadsafe")==0 ){
+      int n;
+      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
+      verify_uninitialized();
+      switch( n ){
+         case 0:  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);  break;
+         case 2:  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);   break;
+         default: sqlite3_config(SQLITE_CONFIG_SERIALIZED);    break;
+      }
+    }else if( cli_strcmp(z,"-vfstrace")==0 ){
+      vfstrace_register("trace",0,(int(*)(const char*,void*))sqlite3_fputs,
+                        stderr,1);
+      bEnableVfstrace = 1;
+#ifdef SQLITE_ENABLE_MULTIPLEX
+    }else if( cli_strcmp(z,"-multiplex")==0 ){
+      extern int sqlite3_multiplex_initialize(const char*,int);
+      sqlite3_multiplex_initialize(0, 1);
+#endif
+    }else if( cli_strcmp(z,"-mmap")==0 ){
+      sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
+      verify_uninitialized();
+      sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
+#if defined(SQLITE_ENABLE_SORTER_REFERENCES)
+    }else if( cli_strcmp(z,"-sorterref")==0 ){
+      sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
+      verify_uninitialized();
+      sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
+#endif
+    }else if( cli_strcmp(z,"-vfs")==0 ){
+      zVfs = cmdline_option_value(argc, argv, ++i);
+#ifdef SQLITE_HAVE_ZLIB
+    }else if( cli_strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( cli_strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
+#ifndef SQLITE_OMIT_DESERIALIZE
+    }else if( cli_strcmp(z,"-deserialize")==0 ){
+      data.openMode = SHELL_OPEN_DESERIALIZE;
+    }else if( cli_strcmp(z,"-maxsize")==0 && i+1<argc ){
+      data.szMax = integerValue(argv[++i]);
+#endif
+    }else if( cli_strcmp(z,"-readonly")==0 ){
+      data.openMode = SHELL_OPEN_READONLY;
+    }else if( cli_strcmp(z,"-nofollow")==0 ){
+      data.openFlags = SQLITE_OPEN_NOFOLLOW;
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+    }else if( cli_strncmp(z, "-A",2)==0 ){
+      /* All remaining command-line arguments are passed to the ".archive"
+      ** command, so ignore them */
+      break;
+#endif
+    }else if( cli_strcmp(z, "-memtrace")==0 ){
+      sqlite3MemTraceActivate(stderr);
+    }else if( cli_strcmp(z, "-pcachetrace")==0 ){
+      sqlite3PcacheTraceActivate(stderr);
+    }else if( cli_strcmp(z,"-bail")==0 ){
+      bail_on_error = 1;
+    }else if( cli_strcmp(z,"-nonce")==0 ){
+      free(data.zNonce);
+      data.zNonce = strdup(cmdline_option_value(argc, argv, ++i));
+    }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
+      ShellSetFlag(&data,SHFLG_TestingMode);
+    }else if( cli_strcmp(z,"-safe")==0 ){
+      /* no-op - catch this on the second pass */
+    }
+  }
+#ifndef SQLITE_SHELL_FIDDLE
+  if( !bEnableVfstrace ) verify_uninitialized();
+#endif
+
+
+#ifdef SQLITE_SHELL_INIT_PROC
+  {
+    /* If the SQLITE_SHELL_INIT_PROC macro is defined, then it is the name
+    ** of a C-function that will perform initialization actions on SQLite that
+    ** occur just before or after sqlite3_initialize(). Use this compile-time
+    ** option to embed this shell program in larger applications. */
+    extern void SQLITE_SHELL_INIT_PROC(void);
+    SQLITE_SHELL_INIT_PROC();
+  }
+#else
+  /* All the sqlite3_config() calls have now been made. So it is safe
+  ** to call sqlite3_initialize() and process any command line -vfs option. */
+  sqlite3_initialize();
+#endif
+
+  if( zVfs ){
+    sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs);
+    if( pVfs ){
+      sqlite3_vfs_register(pVfs, 1);
+    }else{
+      sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs);
+      exit(1);
+    }
+  }
+
+  if( data.pAuxDb->zDbFilename==0 ){
+#ifndef SQLITE_OMIT_MEMORYDB
+    data.pAuxDb->zDbFilename = ":memory:";
+    warnInmemoryDb = argc==1;
+#else
+    sqlite3_fprintf(stderr,
+                    "%s: Error: no database filename specified\n", Argv0);
+    return 1;
+#endif
+  }
+  data.out = stdout;
+#ifndef SQLITE_SHELL_FIDDLE
+  sqlite3_appendvfs_init(0,0,0);
+#endif
+
+  /* Go ahead and open the database file if it already exists.  If the
+  ** file does not exist, delay opening it.  This prevents empty database
+  ** files from being created if a user mistypes the database name argument
+  ** to the sqlite command-line tool.
+  */
+  if( access(data.pAuxDb->zDbFilename, 0)==0 ){
+    open_db(&data, 0);
+  }
+
+  /* Process the initialization file if there is one.  If no -init option
+  ** is given on the command line, look for a file named ~/.sqliterc and
+  ** try to process it.
+  */
+  process_sqliterc(&data,zInitFile);
+
+  /* Make a second pass through the command-line argument and set
+  ** options.  This second pass is delayed until after the initialization
+  ** file is processed so that the command-line arguments will override
+  ** settings in the initialization file.
+  */
+  for(i=1; i<argc; i++){
+    char *z = argv[i];
+    if( z[0]!='-' || i>=nOptsEnd ) continue;
+    if( z[1]=='-' ){ z++; }
+    if( cli_strcmp(z,"-init")==0 ){
+      i++;
+    }else if( cli_strcmp(z,"-html")==0 ){
+      data.mode = MODE_Html;
+    }else if( cli_strcmp(z,"-list")==0 ){
+      data.mode = MODE_List;
+    }else if( cli_strcmp(z,"-quote")==0 ){
+      data.mode = MODE_Quote;
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row);
+    }else if( cli_strcmp(z,"-line")==0 ){
+      data.mode = MODE_Line;
+    }else if( cli_strcmp(z,"-column")==0 ){
+      data.mode = MODE_Column;
+    }else if( cli_strcmp(z,"-json")==0 ){
+      data.mode = MODE_Json;
+    }else if( cli_strcmp(z,"-markdown")==0 ){
+      data.mode = MODE_Markdown;
+    }else if( cli_strcmp(z,"-table")==0 ){
+      data.mode = MODE_Table;
+    }else if( cli_strcmp(z,"-box")==0 ){
+      data.mode = MODE_Box;
+    }else if( cli_strcmp(z,"-csv")==0 ){
+      data.mode = MODE_Csv;
+      memcpy(data.colSeparator,",",2);
+#ifdef SQLITE_HAVE_ZLIB
+    }else if( cli_strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( cli_strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
+#ifndef SQLITE_OMIT_DESERIALIZE
+    }else if( cli_strcmp(z,"-deserialize")==0 ){
+      data.openMode = SHELL_OPEN_DESERIALIZE;
+    }else if( cli_strcmp(z,"-maxsize")==0 && i+1<argc ){
+      data.szMax = integerValue(argv[++i]);
+#endif
+    }else if( cli_strcmp(z,"-readonly")==0 ){
+      data.openMode = SHELL_OPEN_READONLY;
+    }else if( cli_strcmp(z,"-nofollow")==0 ){
+      data.openFlags |= SQLITE_OPEN_NOFOLLOW;
+    }else if( cli_strcmp(z,"-ascii")==0 ){
+      data.mode = MODE_Ascii;
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record);
+    }else if( cli_strcmp(z,"-tabs")==0 ){
+      data.mode = MODE_List;
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row);
+    }else if( cli_strcmp(z,"-separator")==0 ){
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
+                       "%s",cmdline_option_value(argc,argv,++i));
+    }else if( cli_strcmp(z,"-newline")==0 ){
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,
+                       "%s",cmdline_option_value(argc,argv,++i));
+    }else if( cli_strcmp(z,"-nullvalue")==0 ){
+      sqlite3_snprintf(sizeof(data.nullValue), data.nullValue,
+                       "%s",cmdline_option_value(argc,argv,++i));
+    }else if( cli_strcmp(z,"-header")==0 ){
+      data.showHeader = 1;
+      ShellSetFlag(&data, SHFLG_HeaderSet);
+     }else if( cli_strcmp(z,"-noheader")==0 ){
+      data.showHeader = 0;
+      ShellSetFlag(&data, SHFLG_HeaderSet);
+    }else if( cli_strcmp(z,"-echo")==0 ){
+      ShellSetFlag(&data, SHFLG_Echo);
+    }else if( cli_strcmp(z,"-eqp")==0 ){
+      data.autoEQP = AUTOEQP_on;
+    }else if( cli_strcmp(z,"-eqpfull")==0 ){
+      data.autoEQP = AUTOEQP_full;
+    }else if( cli_strcmp(z,"-stats")==0 ){
+      data.statsOn = 1;
+    }else if( cli_strcmp(z,"-scanstats")==0 ){
+      data.scanstatsOn = 1;
+    }else if( cli_strcmp(z,"-backslash")==0 ){
+      /* Undocumented command-line option: -backslash
+      ** Causes C-style backslash escapes to be evaluated in SQL statements
+      ** prior to sending the SQL into SQLite.  Useful for injecting
+      ** crazy bytes in the middle of SQL statements for testing and debugging.
+      */
+      ShellSetFlag(&data, SHFLG_Backslash);
+    }else if( cli_strcmp(z,"-bail")==0 ){
+      /* No-op.  The bail_on_error flag should already be set. */
+    }else if( cli_strcmp(z,"-version")==0 ){
+      sqlite3_fprintf(stdout, "%s %s (%d-bit)\n",
+            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
+      return 0;
+    }else if( cli_strcmp(z,"-interactive")==0 ){
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
+      stdin_is_interactive = 1;
+    }else if( cli_strcmp(z,"-batch")==0 ){
+      /* already handled */
+    }else if( cli_strcmp(z,"-utf8")==0 ){
+      /* already handled */
+    }else if( cli_strcmp(z,"-no-utf8")==0 ){
+      /* already handled */
+    }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){
+      /* already handled */
+    }else if( cli_strcmp(z,"-heap")==0 ){
+      i++;
+    }else if( cli_strcmp(z,"-pagecache")==0 ){
+      i+=2;
+    }else if( cli_strcmp(z,"-lookaside")==0 ){
+      i+=2;
+    }else if( cli_strcmp(z,"-threadsafe")==0 ){
+      i+=2;
+    }else if( cli_strcmp(z,"-nonce")==0 ){
+      i += 2;
+    }else if( cli_strcmp(z,"-mmap")==0 ){
+      i++;
+    }else if( cli_strcmp(z,"-memtrace")==0 ){
+      i++;
+    }else if( cli_strcmp(z,"-pcachetrace")==0 ){
+      i++;
+#ifdef SQLITE_ENABLE_SORTER_REFERENCES
+    }else if( cli_strcmp(z,"-sorterref")==0 ){
+      i++;
+#endif
+    }else if( cli_strcmp(z,"-vfs")==0 ){
+      i++;
+    }else if( cli_strcmp(z,"-vfstrace")==0 ){
+      i++;
+#ifdef SQLITE_ENABLE_MULTIPLEX
+    }else if( cli_strcmp(z,"-multiplex")==0 ){
+      i++;
+#endif
+    }else if( cli_strcmp(z,"-help")==0 ){
+      usage(1);
+    }else if( cli_strcmp(z,"-cmd")==0 ){
+      /* Run commands that follow -cmd first and separately from commands
+      ** that simply appear on the command-line.  This seems goofy.  It would
+      ** be better if all commands ran in the order that they appear.  But
+      ** we retain the goofy behavior for historical compatibility. */
+      if( i==argc-1 ) break;
+      z = cmdline_option_value(argc,argv,++i);
+      if( z[0]=='.' ){
+        rc = do_meta_command(z, &data);
+        if( rc && bail_on_error ) return rc==2 ? 0 : rc;
+      }else{
+        open_db(&data, 0);
+        rc = shell_exec(&data, z, &zErrMsg);
+        if( zErrMsg!=0 ){
+          shellEmitError(zErrMsg);
+          if( bail_on_error ) return rc!=0 ? rc : 1;
+        }else if( rc!=0 ){
+          sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z);
+          if( bail_on_error ) return rc;
+        }
+      }
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+    }else if( cli_strncmp(z, "-A", 2)==0 ){
+      if( nCmd>0 ){
+        sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands"
+              " with \"%s\"\n", z);
+        return 1;
+      }
+      open_db(&data, OPEN_DB_ZIPFILE);
+      if( z[2] ){
+        argv[i] = &z[2];
+        arDotCommand(&data, 1, argv+(i-1), argc-(i-1));
+      }else{
+        arDotCommand(&data, 1, argv+i, argc-i);
+      }
+      readStdin = 0;
+      break;
+#endif
+    }else if( cli_strcmp(z,"-safe")==0 ){
+      data.bSafeMode = data.bSafeModePersist = 1;
+    }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
+      /* Acted upon in first pass. */
+    }else{
+      sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
+      eputz("Use -help for a list of options.\n");
+      return 1;
+    }
+    data.cMode = data.mode;
+  }
+
+  if( !readStdin ){
+    /* Run all arguments that do not begin with '-' as if they were separate
+    ** command-line inputs, except for the argToSkip argument which contains
+    ** the database filename.
+    */
+    for(i=0; i<nCmd; i++){
+      if( azCmd[i][0]=='.' ){
+        rc = do_meta_command(azCmd[i], &data);
+        if( rc ){
+          if( rc==2 ) rc = 0;
+          goto shell_main_exit;
+        }
+      }else{
+        open_db(&data, 0);
+        echo_group_input(&data, azCmd[i]);
+        rc = shell_exec(&data, azCmd[i], &zErrMsg);
+        if( zErrMsg || rc ){
+          if( zErrMsg!=0 ){
+            shellEmitError(zErrMsg);
+          }else{
+            sqlite3_fprintf(stderr,
+                            "Error: unable to process SQL: %s\n", azCmd[i]);
+          }
+          sqlite3_free(zErrMsg);
+          if( rc==0 ) rc = 1;
+          goto shell_main_exit;
+        }
+      }
+    }
+  }else{
+    /* Run commands received from standard input
+    */
+    if( stdin_is_interactive ){
+      char *zHome;
+      char *zHistory;
+      int nHistory;
+#if CIO_WIN_WC_XLATE
+# define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "")
+#else
+# define SHELL_CIO_CHAR_SET ""
+#endif
+      sqlite3_fprintf(stdout,
+            "SQLite version %s %.19s%s\n" /*extra-version-info*/
+            "Enter \".help\" for usage hints.\n",
+            sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET);
+      if( warnInmemoryDb ){
+        sputz(stdout, "Connected to a ");
+        printBold("transient in-memory database");
+        sputz(stdout, ".\nUse \".open FILENAME\" to reopen on a"
+              " persistent database.\n");
+      }
+      zHistory = getenv("SQLITE_HISTORY");
+      if( zHistory ){
+        zHistory = strdup(zHistory);
+      }else if( (zHome = find_home_dir(0))!=0 ){
+        nHistory = strlen30(zHome) + 20;
+        if( (zHistory = malloc(nHistory))!=0 ){
+          sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome);
+        }
+      }
+      if( zHistory ){ shell_read_history(zHistory); }
+#if HAVE_READLINE || HAVE_EDITLINE
+      rl_attempted_completion_function = readline_completion;
+#elif HAVE_LINENOISE
+      linenoiseSetCompletionCallback(linenoise_completion, NULL);
+#endif
+      data.in = 0;
+      rc = process_input(&data);
+      if( zHistory ){
+        shell_stifle_history(2000);
+        shell_write_history(zHistory);
+        free(zHistory);
+      }
+    }else{
+      data.in = stdin;
+      rc = process_input(&data);
+    }
+  }
+#ifndef SQLITE_SHELL_FIDDLE
+  /* In WASM mode we have to leave the db state in place so that
+  ** client code can "push" SQL into it after this call returns. */
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( data.expert.pExpert ){
+    expertFinish(&data, 1, 0);
+  }
+#endif
+ shell_main_exit:
+  free(azCmd);
+  set_table_name(&data, 0);
+  if( data.db ){
+    session_close_all(&data, -1);
+    close_db(data.db);
+  }
+  for(i=0; i<ArraySize(data.aAuxDb); i++){
+    sqlite3_free(data.aAuxDb[i].zFreeOnClose);
+    if( data.aAuxDb[i].db ){
+      session_close_all(&data, i);
+      close_db(data.aAuxDb[i].db);
+    }
+  }
+  find_home_dir(1);
+  output_reset(&data);
+  data.doXdgOpen = 0;
+  clearTempFile(&data);
+#if !SQLITE_SHELL_IS_UTF8
+  for(i=0; i<argcToFree; i++) free(argvToFree[i]);
+  free(argvToFree);
+#endif
+  free(data.colWidth);
+  free(data.zNonce);
+  /* Clear the global data structure so that valgrind will detect memory
+  ** leaks */
+  memset(&data, 0, sizeof(data));
+  if( bEnableVfstrace ){
+    vfstrace_unregister("trace");
+  }
+#ifdef SQLITE_DEBUG
+  if( sqlite3_memory_used()>mem_main_enter ){
+    sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n",
+          (unsigned int)(sqlite3_memory_used()-mem_main_enter));
+  }
+#endif
+#else /* SQLITE_SHELL_FIDDLE... */
+  shell_main_exit:
+#endif
+  return rc;
+}
+
+
+#ifdef SQLITE_SHELL_FIDDLE
+/* Only for emcc experimentation purposes. */
+int fiddle_experiment(int a,int b){
+  return a + b;
+}
+
+/*
+** Returns a pointer to the current DB handle.
+*/
+sqlite3 * fiddle_db_handle(){
+  return globalDb;
+}
+
+/*
+** Returns a pointer to the given DB name's VFS. If zDbName is 0 then
+** "main" is assumed. Returns 0 if no db with the given name is
+** open.
+*/
+sqlite3_vfs * fiddle_db_vfs(const char *zDbName){
+  sqlite3_vfs * pVfs = 0;
+  if(globalDb){
+    sqlite3_file_control(globalDb, zDbName ? zDbName : "main",
+                         SQLITE_FCNTL_VFS_POINTER, &pVfs);
+  }
+  return pVfs;
+}
+
+/* Only for emcc experimentation purposes. */
+sqlite3 * fiddle_db_arg(sqlite3 *arg){
+    sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg);
+    return arg;
+}
+
+/*
+** Intended to be called via a SharedWorker() while a separate
+** SharedWorker() (which manages the wasm module) is performing work
+** which should be interrupted. Unfortunately, SharedWorker is not
+** portable enough to make real use of.
+*/
+void fiddle_interrupt(void){
+  if( globalDb ) sqlite3_interrupt(globalDb);
+}
+
+/*
+** Returns the filename of the given db name, assuming "main" if
+** zDbName is NULL. Returns NULL if globalDb is not opened.
+*/
+const char * fiddle_db_filename(const char * zDbName){
+    return globalDb
+      ? sqlite3_db_filename(globalDb, zDbName ? zDbName : "main")
+      : NULL;
+}
+
+/*
+** Completely wipes out the contents of the currently-opened database
+** but leaves its storage intact for reuse. If any transactions are
+** active, they are forcibly rolled back.
+*/
+void fiddle_reset_db(void){
+  if( globalDb ){
+    int rc;
+    while( sqlite3_txn_state(globalDb,0)>0 ){
+      /*
+      ** Resolve problem reported in
+      ** https://sqlite.org/forum/forumpost/0b41a25d65
+      */
+      sqlite3_fputs("Rolling back in-progress transaction.\n", stdout);
+      sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0);
+    }
+    rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
+    if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0);
+    sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
+  }
+}
+
+/*
+** Uses the current database's VFS xRead to stream the db file's
+** contents out to the given callback. The callback gets a single
+** chunk of size n (its 2nd argument) on each call and must return 0
+** on success, non-0 on error. This function returns 0 on success,
+** SQLITE_NOTFOUND if no db is open, or propagates any other non-0
+** code from the callback. Note that this is not thread-friendly: it
+** expects that it will be the only thread reading the db file and
+** takes no measures to ensure that is the case.
+*/
+int fiddle_export_db( int (*xCallback)(unsigned const char *zOut, int n) ){
+  sqlite3_int64 nSize = 0;
+  sqlite3_int64 nPos = 0;
+  sqlite3_file * pFile = 0;
+  unsigned char buf[1024 * 8];
+  int nBuf = (int)sizeof(buf);
+  int rc = shellState.db
+    ? sqlite3_file_control(shellState.db, "main",
+                           SQLITE_FCNTL_FILE_POINTER, &pFile)
+    : SQLITE_NOTFOUND;
+  if( rc ) return rc;
+  rc = pFile->pMethods->xFileSize(pFile, &nSize);
+  if( rc ) return rc;
+  if(nSize % nBuf){
+    /* DB size is not an even multiple of the buffer size. Reduce
+    ** buffer size so that we do not unduly inflate the db size when
+    ** exporting. */
+    if(0 == nSize % 4096) nBuf = 4096;
+    else if(0 == nSize % 2048) nBuf = 2048;
+    else if(0 == nSize % 1024) nBuf = 1024;
+    else nBuf = 512;
+  }
+  for( ; 0==rc && nPos<nSize; nPos += nBuf ){
+    rc = pFile->pMethods->xRead(pFile, buf, nBuf, nPos);
+    if(SQLITE_IOERR_SHORT_READ == rc){
+      rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/;
+    }
+    if( 0==rc ) rc = xCallback(buf, nBuf);
+  }
+  return rc;
+}
+
+/*
+** Trivial exportable function for emscripten. It processes zSql as if
+** it were input to the sqlite3 shell and redirects all output to the
+** wasm binding. fiddle_main() must have been called before this
+** is called, or results are undefined.
+*/
+void fiddle_exec(const char * zSql){
+  if(zSql && *zSql){
+    if('.'==*zSql) puts(zSql);
+    shellState.wasm.zInput = zSql;
+    shellState.wasm.zPos = zSql;
+    process_input(&shellState);
+    shellState.wasm.zInput = shellState.wasm.zPos = 0;
+  }
+}
+#endif /* SQLITE_SHELL_FIDDLE */
diff --git a/deps/sqlite3/sqlite3.c b/deps/sqlite3/sqlite3.c
index 9808b3f1..ebbaf20c 100644
--- a/deps/sqlite3/sqlite3.c
+++ b/deps/sqlite3/sqlite3.c
@@ -18,7 +18,7 @@
 ** separate file. This file contains only code for the core SQLite library.
 **
 ** The content in this amalgamation comes from Fossil check-in
-** 03a9703e27c44437c39363d0baf82db4ebc9.
+** d2d954d43abe20a3cf10cc12f0f9922ba6b9.
 */
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
@@ -465,7 +465,7 @@ extern "C" {
 */
 #define SQLITE_VERSION        "3.47.0"
 #define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e"
+#define SQLITE_SOURCE_ID      "2024-10-21 17:10:05 d2d954d43abe20a3cf10cc12f0f9922ba6b94584e39b1ef1991f1d354058c99a"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -11032,6 +11032,31 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** CAPI3REF: Wal related information regarding the most recent COMMIT
+** EXPERIMENTAL
+**
+** This function reports on the state of the wal file (if any) for database
+** zDb, which should be "main", "temp", or the name of the attached database.
+** Its results - the values written to the output parameters - are only
+** defined if the most recent SQL command on the connection was a successful
+** COMMIT that wrote data to wal-mode database zDb.
+**
+** Assuming the above conditions are met, output parameter (*pnFrame) is set
+** to the total number of frames in the wal file. Parameter (*pnPrior) is
+** set to the number of frames that were present in the wal file before the
+** most recent transaction was committed. So that the number of frames written
+** by the most recent transaction is (*pnFrame)-(*pnPrior).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It
+** is not an error if this function is called at a time when the results
+** are undefined.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb,
+  unsigned int *pnPrior, unsigned int *pnFrame
+);
+
 /*
 ** CAPI3REF: Serialize a database
 **
@@ -11175,6 +11200,98 @@ SQLITE_API int sqlite3_deserialize(
 #define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
 #define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */
 
+/*
+** Access details of recent COMMIT commands. This function allows various
+** details related to the most recent COMMIT command to be accessed.
+** The requested value is always returned via output parameter (*piVal).
+** The specific value requested is identified by parameter op (see
+** below).
+**
+** SQLITE_OK is returned if successful, or SQLITE_ERROR if the "op" or
+** "zDb" paramters are unrecognized.
+*/
+SQLITE_API int sqlite3_commit_status(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Name of database - "main" etc. */
+  int op,                         /* SQLITE_COMMIT_XXX constant */
+  unsigned int *piVal             /* OUT: Write requested value here */
+);
+
+/*
+** The following describes the five requests supported by
+** sqlite3_commit_status(), each identified by an SQLITE_COMMIT_XXX
+** constant:
+**
+** SQLITE_COMMIT_FIRSTFRAME:
+**   In this case argument zDb must be "main", or "temp", or else the name of
+**   an attached database. If zDb does not correspond to any attached database,
+**   SQLITE_ERROR is returned.
+**
+**   The final value of (*piVal) for this request is only defined if (a) the
+**   most recent attempt to write to the database connection was successful,
+**   (b) the most recent attempt to write to the database did write to database
+**   zDb, and (c) zDb is a wal mode database.
+**
+**   If the above conditions are true, then output parameter (*piVal) is
+**   set to the frame number of the first frame written by the recent
+**   transaction. In wal mode, or in wal2 mode when a transaction is
+**   written into the *-wal file, the frame number indicates the frame's
+**   position in the wal file - frames are numbered starting from 1. In
+**   wal2 mode, when a transaction is written to the *-wal2 file, the frame
+**   number is the frame's position in the *-wal2 file, plus (1 << 31).
+**
+**   Note: Although the a database may have up to (1<<32) pages, each wal
+**   file may contain at most (1<<31) frames.
+**
+** SQLITE_COMMIT_NFRAME:
+**   zDb is interpreted in the same way as, and the final value of (*piVal)
+**   is undefined, for SQLITE_COMMIT_FIRSTFRAME.
+**
+**   Otherwise, (*piVal) is set to the number of frames written by the
+**   recent transaction.
+**
+** SQLITE_COMMIT_CONFLICT_DB:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the most recent attempt to write to
+**   the database handle was a BEGIN CONCURRENT transaction that
+**   failed with an SQLITE_BUSY_SNAPSHOT error.
+**
+**   In other cases, (*piVal) is set to the index of the database
+**   on which the SQLITE_BUSY_SNAPSHOT error occurred (0 for main,
+**   a value of 2 or greater for an attached database). This value
+**   may be used with the sqlite3_db_name() API to find the name
+**   of the conflicting database.
+**
+** SQLITE_COMMIT_CONFLICT_FRAME:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the most recent attempt to write to
+**   the database handle was a BEGIN CONCURRENT transaction that
+**   failed with an SQLITE_BUSY_SNAPSHOT error.
+**
+**   (*piVal) is set to the frame number of the conflicting frame for
+**   the recent SQLITE_BUSY_SNAPSHOT error. The conflicting transaction may
+**   be found by comparing this value with the FIRSTFRAME and
+**   NFRAME values for recent succesfully committed transactions on
+**   the same db. If the CONFLICT_FRAME value is F, then the conflicting
+**   transaction is the most recent successful commit for which
+**   (FIRSTFRAME <= F <= FIRSTFRAME+NFRAME) is true.
+**
+** SQLITE_COMMIT_CONFLICT_PGNO:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the previous attempt to write to
+**   the database using database handle db failed with
+**   SQLITE_BUSY_SNAPSHOT.
+**
+**   Return the page number of the conflicting page for the most
+**   recent SQLITE_BUSY_SNAPSHOT error.
+*/
+#define SQLITE_COMMIT_FIRSTFRAME     0
+#define SQLITE_COMMIT_NFRAME         1
+#define SQLITE_COMMIT_CONFLICT_DB    2
+#define SQLITE_COMMIT_CONFLICT_FRAME 3
+#define SQLITE_COMMIT_CONFLICT_PGNO  4
+
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -11688,6 +11805,32 @@ SQLITE_API int sqlite3session_changeset(
   void **ppChangeset              /* OUT: Buffer containing changeset */
 );
 
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+SQLITE_API int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+SQLITE_API int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
 /*
 ** CAPI3REF: Return An Upper-limit For The Size Of The Changeset
 ** METHOD: sqlite3_session
@@ -11888,12 +12031,16 @@ SQLITE_API int sqlite3changeset_start_v2(
 ** The following flags may passed via the 4th parameter to
 ** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]:
 **
-** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
+** <dt>SQLITE_CHANGESETSTART_INVERT <dd>
 **   Invert the changeset while iterating through it. This is equivalent to
 **   inverting a changeset using sqlite3changeset_invert() before applying it.
 **   It is an error to specify this flag with a patchset.
+**
+** <dt>SQLITE_CHANGESETSTART_FULL <dd>
+**   Do not trim extra fields added to fullchangeset changesets.
 */
 #define SQLITE_CHANGESETSTART_INVERT        0x0002
+#define SQLITE_CHANGESETSTART_FULL          0x0004
 
 
 /*
@@ -14648,7 +14795,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*);
 #define TK_TRANSACTION                      6
 #define TK_DEFERRED                         7
 #define TK_IMMEDIATE                        8
-#define TK_EXCLUSIVE                        9
+#define TK_ID                               9
 #define TK_COMMIT                          10
 #define TK_END                             11
 #define TK_ROLLBACK                        12
@@ -14681,25 +14828,25 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*);
 #define TK_DESC                            39
 #define TK_DETACH                          40
 #define TK_EACH                            41
-#define TK_FAIL                            42
-#define TK_OR                              43
-#define TK_AND                             44
-#define TK_IS                              45
-#define TK_ISNOT                           46
-#define TK_MATCH                           47
-#define TK_LIKE_KW                         48
-#define TK_BETWEEN                         49
-#define TK_IN                              50
-#define TK_ISNULL                          51
-#define TK_NOTNULL                         52
-#define TK_NE                              53
-#define TK_EQ                              54
-#define TK_GT                              55
-#define TK_LE                              56
-#define TK_LT                              57
-#define TK_GE                              58
-#define TK_ESCAPE                          59
-#define TK_ID                              60
+#define TK_EXCLUSIVE                       42
+#define TK_FAIL                            43
+#define TK_OR                              44
+#define TK_AND                             45
+#define TK_IS                              46
+#define TK_ISNOT                           47
+#define TK_MATCH                           48
+#define TK_LIKE_KW                         49
+#define TK_BETWEEN                         50
+#define TK_IN                              51
+#define TK_ISNULL                          52
+#define TK_NOTNULL                         53
+#define TK_NE                              54
+#define TK_EQ                              55
+#define TK_GT                              56
+#define TK_LE                              57
+#define TK_LT                              58
+#define TK_GE                              59
+#define TK_ESCAPE                          60
 #define TK_COLUMNKW                        61
 #define TK_DO                              62
 #define TK_FOR                             63
@@ -14816,15 +14963,16 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*);
 #define TK_UMINUS                         174
 #define TK_TRUTH                          175
 #define TK_REGISTER                       176
-#define TK_VECTOR                         177
-#define TK_SELECT_COLUMN                  178
-#define TK_IF_NULL_ROW                    179
-#define TK_ASTERISK                       180
-#define TK_SPAN                           181
-#define TK_ERROR                          182
-#define TK_QNUMBER                        183
-#define TK_SPACE                          184
-#define TK_ILLEGAL                        185
+#define TK_CONCURRENT                     177
+#define TK_VECTOR                         178
+#define TK_SELECT_COLUMN                  179
+#define TK_IF_NULL_ROW                    180
+#define TK_ASTERISK                       181
+#define TK_SPAN                           182
+#define TK_ERROR                          183
+#define TK_QNUMBER                        184
+#define TK_SPACE                          185
+#define TK_ILLEGAL                        186
 
 /************** End of parse.h ***********************************************/
 /************** Continuing where we left off in sqliteInt.h ******************/
@@ -15501,6 +15649,7 @@ typedef struct DbFixer DbFixer;
 typedef struct Schema Schema;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
+typedef struct FastPrng FastPrng;
 typedef struct FKey FKey;
 typedef struct FpDecode FpDecode;
 typedef struct FuncDestructor FuncDestructor;
@@ -16002,6 +16151,23 @@ typedef struct PgHdr DbPage;
 #define PAGER_JOURNALMODE_TRUNCATE    3   /* Commit by truncating journal */
 #define PAGER_JOURNALMODE_MEMORY      4   /* In-memory journal file */
 #define PAGER_JOURNALMODE_WAL         5   /* Use write-ahead logging */
+#define PAGER_JOURNALMODE_WAL2        6   /* Use write-ahead logging mode 2 */
+
+#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL || (x)==PAGER_JOURNALMODE_WAL2)
+
+/*
+** The argument to this macro is a file descriptor (type sqlite3_file*).
+** Return 0 if it is not open, or non-zero (but not 1) if it is.
+**
+** This is so that expressions can be written as:
+**
+**   if( isOpen(pPager->jfd) ){ ...
+**
+** instead of
+**
+**   if( pPager->jfd->pMethods ){ ...
+*/
+#define isOpen(pFd) ((pFd)->pMethods!=0)
 
 /*
 ** Flags that make up the mask passed to sqlite3PagerGet().
@@ -16083,7 +16249,7 @@ SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *);
 SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*);
 SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int);
 SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int);
-SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*);
+SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, u32*);
 SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper);
 SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*);
 SQLITE_PRIVATE int sqlite3PagerRollback(Pager*);
@@ -16091,11 +16257,12 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager);
 
+
 #ifndef SQLITE_OMIT_WAL
 SQLITE_PRIVATE   int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*);
 SQLITE_PRIVATE   int sqlite3PagerWalSupported(Pager *pPager);
 SQLITE_PRIVATE   int sqlite3PagerWalCallback(Pager *pPager);
-SQLITE_PRIVATE   int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen);
+SQLITE_PRIVATE   int sqlite3PagerOpenWal(Pager *pPager, int, int *pisOpen);
 SQLITE_PRIVATE   int sqlite3PagerCloseWal(Pager *pPager, sqlite3*);
 # ifdef SQLITE_ENABLE_SNAPSHOT
 SQLITE_PRIVATE   int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot);
@@ -16145,10 +16312,28 @@ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno);
 
 SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16);
 
+#ifndef SQLITE_OMIT_CONCURRENT
+SQLITE_PRIVATE int sqlite3PagerUsePage(Pager*, Pgno);
+SQLITE_PRIVATE void sqlite3PagerEndConcurrent(Pager*);
+SQLITE_PRIVATE int sqlite3PagerBeginConcurrent(Pager*);
+SQLITE_PRIVATE void sqlite3PagerDropExclusiveLock(Pager*);
+SQLITE_PRIVATE int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*);
+SQLITE_PRIVATE void sqlite3PagerSetDbsize(Pager *pPager, Pgno);
+SQLITE_PRIVATE int sqlite3PagerIsWal(Pager*);
+#else
+# define sqlite3PagerEndConcurrent(x)
+# define sqlite3PagerUsePage(x, y) SQLITE_OK
+#endif
+
+#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT)
+SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage*);
+#endif
+
+SQLITE_PRIVATE int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame);
+
 /* Functions to support testing and debugging. */
 #if !defined(NDEBUG) || defined(SQLITE_TEST)
 SQLITE_PRIVATE   Pgno sqlite3PagerPagenumber(DbPage*);
-SQLITE_PRIVATE   int sqlite3PagerIswriteable(DbPage*);
 #endif
 #ifdef SQLITE_TEST
 SQLITE_PRIVATE   int *sqlite3PagerStats(Pager*);
@@ -16532,6 +16717,8 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree*);
 # define sqlite3BtreeSeekCount(X) 0
 #endif
 
+SQLITE_PRIVATE int sqlite3BtreeExclusiveLock(Btree *pBt);
+
 #ifndef NDEBUG
 SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*);
 #endif
@@ -16592,7 +16779,6 @@ SQLITE_PRIVATE   int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*);
 # define sqlite3SchemaMutexHeld(X,Y,Z) 1
 #endif
 
-
 #endif /* SQLITE_BTREE_H */
 
 /************** End of btree.h ***********************************************/
@@ -16827,24 +17013,24 @@ typedef struct VdbeOpList VdbeOpList;
 #define OP_IdxLE          40 /* jump, synopsis: key=r[P3@P4]               */
 #define OP_IdxGT          41 /* jump, synopsis: key=r[P3@P4]               */
 #define OP_IdxLT          42 /* jump, synopsis: key=r[P3@P4]               */
-#define OP_Or             43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */
-#define OP_And            44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */
-#define OP_IdxGE          45 /* jump, synopsis: key=r[P3@P4]               */
+#define OP_IdxGE          43 /* jump, synopsis: key=r[P3@P4]               */
+#define OP_Or             44 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */
+#define OP_And            45 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */
 #define OP_RowSetRead     46 /* jump, synopsis: r[P3]=rowset(P1)           */
 #define OP_RowSetTest     47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */
 #define OP_Program        48 /* jump0                                      */
 #define OP_FkIfZero       49 /* jump, synopsis: if fkctr[P1]==0 goto P2    */
 #define OP_IfPos          50 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */
-#define OP_IsNull         51 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */
-#define OP_NotNull        52 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */
-#define OP_Ne             53 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */
-#define OP_Eq             54 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */
-#define OP_Gt             55 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */
-#define OP_Le             56 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */
-#define OP_Lt             57 /* jump, same as TK_LT, synopsis: IF r[P3]<r[P1] */
-#define OP_Ge             58 /* jump, same as TK_GE, synopsis: IF r[P3]>=r[P1] */
-#define OP_ElseEq         59 /* jump, same as TK_ESCAPE                    */
-#define OP_IfNotZero      60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */
+#define OP_IfNotZero      51 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */
+#define OP_IsNull         52 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */
+#define OP_NotNull        53 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */
+#define OP_Ne             54 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */
+#define OP_Eq             55 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */
+#define OP_Gt             56 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */
+#define OP_Le             57 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */
+#define OP_Lt             58 /* jump, same as TK_LT, synopsis: IF r[P3]<r[P1] */
+#define OP_Ge             59 /* jump, same as TK_GE, synopsis: IF r[P3]>=r[P1] */
+#define OP_ElseEq         60 /* jump, same as TK_ESCAPE                    */
 #define OP_DecrJumpZero   61 /* jump, synopsis: if (--r[P1])==0 goto P2    */
 #define OP_IncrVacuum     62 /* jump                                       */
 #define OP_VNext          63 /* jump                                       */
@@ -16993,9 +17179,9 @@ typedef struct VdbeOpList VdbeOpList;
 /*  16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\
 /*  24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\
 /*  32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\
-/*  40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\
-/*  48 */ 0x81, 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\
-/*  56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x41,\
+/*  40 */ 0x41, 0x41, 0x41, 0x41, 0x26, 0x26, 0x23, 0x0b,\
+/*  48 */ 0x81, 0x01, 0x03, 0x03, 0x03, 0x03, 0x0b, 0x0b,\
+/*  56 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x41,\
 /*  64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\
 /*  72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\
 /*  80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\
@@ -17575,6 +17761,14 @@ SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*);
 # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
 #endif
 
+/*
+** State of a simple PRNG used for the per-connection and per-pager
+** pseudo-random number generators.
+*/
+struct FastPrng {
+  unsigned int x, y;
+};
+
 /*
 ** Each database file to be accessed by the system is an instance
 ** of the following structure.  There are normally two of these structures
@@ -17826,6 +18020,7 @@ struct sqlite3 {
   u32 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
+  u8 eConcurrent;               /* CONCURRENT_* value */
   u8 temp_store;                /* 1: file 2: memory 0: default */
   u8 mallocFailed;              /* True if we have seen a malloc failure */
   u8 bBenignMalloc;             /* Do not require OOMs if true */
@@ -17839,6 +18034,7 @@ struct sqlite3 {
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   u8 eOpenState;                /* Current condition of the connection */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
+  FastPrng sPrng;               /* State of the per-connection PRNG */
   i64 nChange;                  /* Value returned by sqlite3_changes() */
   i64 nTotalChange;             /* Value returned by sqlite3_total_changes() */
   int aLimit[SQLITE_N_LIMIT];   /* Limits */
@@ -17947,8 +18143,21 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* Return values for sqlite3_commit_status() requests:
+  ** SQLITE_COMMIT_CONFLICT_DB, CONFLICT_FRAME and CONFLICT_PGNO.
+  */
+  u32 aCommit[5];
+#endif
 };
 
+/*
+** Candidate values for sqlite3.eConcurrent
+*/
+#define CONCURRENT_NONE   0
+#define CONCURRENT_OPEN   1
+#define CONCURRENT_SCHEMA 2
+
 /*
 ** A macro to discover the encoding of a database.
 */
@@ -18011,6 +18220,7 @@ struct sqlite3 {
 #define SQLITE_ReadUncommit   HI(0x00004) /* READ UNCOMMITTED in shared-cache */
 #define SQLITE_FkNoAction     HI(0x00008) /* Treat all FK as NO ACTION */
 
+#define SQLITE_NoopUpdate     0x01000000  /* UPDATE operations are no-ops */
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
 #define SQLITE_SqlTrace       HI(0x0100000) /* Debug print SQL as it executes */
@@ -21249,6 +21459,8 @@ SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*);
 SQLITE_PRIVATE void sqlite3PrngSaveState(void);
 SQLITE_PRIVATE void sqlite3PrngRestoreState(void);
 #endif
+SQLITE_PRIVATE void sqlite3FastPrngInit(FastPrng*);
+SQLITE_PRIVATE void sqlite3FastRandomness(FastPrng*, int N, void *P);
 SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int);
 SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int);
 SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb);
@@ -22868,6 +23080,7 @@ static const char * const sqlite3azCompileOpt[] = {
 #ifdef SQLITE_VDBE_COVERAGE
   "VDBE_COVERAGE",
 #endif
+"WAL2",
 #ifdef SQLITE_WIN32_MALLOC
   "WIN32_MALLOC",
 #endif
@@ -34270,6 +34483,28 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){
   sqlite3_mutex_leave(mutex);
 }
 
+/*
+** Initialize a fast PRNG.  A Fast PRNG is called "fast" because it does
+** not need a mutex to operate, though it does use a mutex to initialize.
+** The quality of the randomness is not as good as the global PRNG.
+*/
+SQLITE_PRIVATE void sqlite3FastPrngInit(FastPrng *pPrng){
+  sqlite3_randomness(sizeof(*pPrng), pPrng);
+  pPrng->x |= 1;
+}
+
+/*
+** Generate N bytes of pseudo-randomness using a FastPrng
+*/
+SQLITE_PRIVATE void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){
+  unsigned char *pOut = (unsigned char*)P;
+  while( N-->0 ){
+    pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001);
+    pPrng->y = (pPrng->y)*1103515245 + 12345;
+    *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff;
+  }
+}
+
 #ifndef SQLITE_UNTESTABLE
 /*
 ** For testing purposes, we sometimes want to preserve the state of
@@ -37326,24 +37561,24 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
     /*  40 */ "IdxLE"            OpHelp("key=r[P3@P4]"),
     /*  41 */ "IdxGT"            OpHelp("key=r[P3@P4]"),
     /*  42 */ "IdxLT"            OpHelp("key=r[P3@P4]"),
-    /*  43 */ "Or"               OpHelp("r[P3]=(r[P1] || r[P2])"),
-    /*  44 */ "And"              OpHelp("r[P3]=(r[P1] && r[P2])"),
-    /*  45 */ "IdxGE"            OpHelp("key=r[P3@P4]"),
+    /*  43 */ "IdxGE"            OpHelp("key=r[P3@P4]"),
+    /*  44 */ "Or"               OpHelp("r[P3]=(r[P1] || r[P2])"),
+    /*  45 */ "And"              OpHelp("r[P3]=(r[P1] && r[P2])"),
     /*  46 */ "RowSetRead"       OpHelp("r[P3]=rowset(P1)"),
     /*  47 */ "RowSetTest"       OpHelp("if r[P3] in rowset(P1) goto P2"),
     /*  48 */ "Program"          OpHelp(""),
     /*  49 */ "FkIfZero"         OpHelp("if fkctr[P1]==0 goto P2"),
     /*  50 */ "IfPos"            OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"),
-    /*  51 */ "IsNull"           OpHelp("if r[P1]==NULL goto P2"),
-    /*  52 */ "NotNull"          OpHelp("if r[P1]!=NULL goto P2"),
-    /*  53 */ "Ne"               OpHelp("IF r[P3]!=r[P1]"),
-    /*  54 */ "Eq"               OpHelp("IF r[P3]==r[P1]"),
-    /*  55 */ "Gt"               OpHelp("IF r[P3]>r[P1]"),
-    /*  56 */ "Le"               OpHelp("IF r[P3]<=r[P1]"),
-    /*  57 */ "Lt"               OpHelp("IF r[P3]<r[P1]"),
-    /*  58 */ "Ge"               OpHelp("IF r[P3]>=r[P1]"),
-    /*  59 */ "ElseEq"           OpHelp(""),
-    /*  60 */ "IfNotZero"        OpHelp("if r[P1]!=0 then r[P1]--, goto P2"),
+    /*  51 */ "IfNotZero"        OpHelp("if r[P1]!=0 then r[P1]--, goto P2"),
+    /*  52 */ "IsNull"           OpHelp("if r[P1]==NULL goto P2"),
+    /*  53 */ "NotNull"          OpHelp("if r[P1]!=NULL goto P2"),
+    /*  54 */ "Ne"               OpHelp("IF r[P3]!=r[P1]"),
+    /*  55 */ "Eq"               OpHelp("IF r[P3]==r[P1]"),
+    /*  56 */ "Gt"               OpHelp("IF r[P3]>r[P1]"),
+    /*  57 */ "Le"               OpHelp("IF r[P3]<=r[P1]"),
+    /*  58 */ "Lt"               OpHelp("IF r[P3]<r[P1]"),
+    /*  59 */ "Ge"               OpHelp("IF r[P3]>=r[P1]"),
+    /*  60 */ "ElseEq"           OpHelp(""),
     /*  61 */ "DecrJumpZero"     OpHelp("if (--r[P1])==0 goto P2"),
     /*  62 */ "IncrVacuum"       OpHelp(""),
     /*  63 */ "VNext"            OpHelp(""),
@@ -38510,6 +38745,16 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){
 /* #include "sqliteInt.h" */
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
+/* Turn this feature on in all builds for now */
+#define SQLITE_MUTEXFREE_SHMLOCK 1
+#define SQLITE_MFS_EXCLUSIVE     255
+#ifndef SQLITE_MFS_NSHARD
+# define SQLITE_MFS_NSHARD       8
+#endif
+#if SQLITE_MFS_NSHARD<1
+# error "SQLITE_MFS_NSHARD must be greater than 0"
+#endif
+
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
@@ -39688,6 +39933,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -39844,6 +40093,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       sqlite3_mutex_enter(pInode->pLockMutex);
       closePendingFds(pFile);
       sqlite3_mutex_leave(pInode->pLockMutex);
@@ -40707,6 +40963,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -42526,6 +42790,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -42795,8 +43062,41 @@ struct unixShmNode {
 #ifdef SQLITE_DEBUG
   u8 nextShmId;              /* Next available unixShm.id value */
 #endif
+
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks
+  ** are stored in the following 64-bit value. There are in total 8
+  ** shm-locking slots, each of which are assigned 8-bits from the 64-bit
+  ** value. The least-significant 8 bits correspond to shm-locking slot
+  ** 0, and so on.
+  **
+  ** If the 8-bits corresponding to a shm-locking locking slot are set to
+  ** 0xFF, then a write-lock is held on the slot. Or, if they are set to
+  ** a non-zero value smaller than 0xFF, then they represent the total
+  ** number of read-locks held on the slot. There is no way to distinguish
+  ** between a write-lock and 255 read-locks.  */
+  struct LockingSlot {
+    u32 nLock;
+    u64 aPadding[7];
+  } aMFSlot[3 + SQLITE_MFS_NSHARD*5];
+#endif
 };
 
+/*
+** Atomic CAS primitive used in multi-process mode. Equivalent to:
+**
+**   int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){
+**     if( *ptr==oldval ){
+**       *ptr = newval;
+**       return 1;
+**     }
+**     return 0;
+**   }
+*/
+#define unixCompareAndSwap(ptr,oldval,newval) \
+    __sync_bool_compare_and_swap(ptr,oldval,newval)
+
+
 /*
 ** Structure used internally by this VFS to record the state of an
 ** open shared memory connection.
@@ -42817,6 +43117,9 @@ struct unixShm {
   u8 id;                     /* Id of this connection within its unixShmNode */
   u16 sharedMask;            /* Mask of shared locks held */
   u16 exclMask;              /* Mask of exclusive locks held */
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  u8 aMFCurrent[8];          /* Current slot used for each shared lock */
+#endif
 };
 
 /*
@@ -43402,6 +43705,87 @@ static int unixShmMap(
   return rc;
 }
 
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+static int unixMutexFreeShmlock(
+  unixFile *pFd,             /* Database file holding the shared memory */
+  int ofst,                  /* First lock to acquire or release */
+  int n,                     /* Number of locks to acquire or release */
+  int flags                  /* What to do with the lock */
+){
+  struct LockMapEntry {
+    int iFirst;
+    int nSlot;
+  } aMap[9] = {
+    { 0, 1 },
+    { 1, 1 },
+    { 2, 1 },
+    { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+5*SQLITE_MFS_NSHARD, 0 },
+  };
+
+  unixShm *p = pFd->pShm;               /* The shared memory being locked */
+  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
+
+  if( flags & SQLITE_SHM_SHARED ){
+    /* SHARED locks */
+    u32 iOld, iNew, *ptr;
+    int iIncr = -1;
+    if( (flags & SQLITE_SHM_UNLOCK)==0 ){
+      p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot;
+      iIncr = 1;
+    }
+    ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock;
+    do {
+      iOld = *ptr;
+      iNew = iOld + iIncr;
+      if( iNew>SQLITE_MFS_EXCLUSIVE ){
+        return SQLITE_BUSY;
+      }
+    }while( 0==unixCompareAndSwap(ptr, iOld, iNew) );
+  }else{
+    /* EXCLUSIVE locks */
+    u16 mask = (1<<(ofst+n)) - (1<<ofst);
+    if( (flags & SQLITE_SHM_LOCK) || (mask & p->exclMask) ){
+      int iFirst = aMap[ofst].iFirst;
+      int iLast = aMap[ofst+n].iFirst;
+      int i;
+      for(i=iFirst; i<iLast; i++){
+        u32 *ptr = &pShmNode->aMFSlot[i].nLock;
+        if( flags & SQLITE_SHM_UNLOCK ){
+          assert( (*ptr)==SQLITE_MFS_EXCLUSIVE );
+          *ptr = 0;
+        }else{
+          u32 iOld;
+          do {
+            iOld = *ptr;
+            if( iOld>0 ){
+              while( i>iFirst ){
+                i--;
+                pShmNode->aMFSlot[i].nLock = 0;
+              }
+              return SQLITE_BUSY;
+            }
+          }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) );
+        }
+      }
+      if( flags & SQLITE_SHM_UNLOCK ){
+        p->exclMask &= ~mask;
+      }else{
+        p->exclMask |= mask;
+      }
+    }
+  }
+
+  return SQLITE_OK;
+}
+#else
+# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK
+#endif
+
 /*
 ** Check that the pShmNode->aLock[] array comports with the locking bitmasks
 ** held by each client. Return true if it does, or false otherwise. This
@@ -43476,6 +43860,11 @@ static int unixShmLock(
   assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 );
   assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 );
 
+  if( pDbFd->pInode->bProcessLock ){
+    return unixMutexFreeShmlock(pDbFd, ofst, n, flags);
+  }
+
+
   /* Check that, if this to be a blocking lock, no locks that occur later
   ** in the following list than the lock being obtained are already held:
   **
@@ -43650,12 +44039,16 @@ static void unixShmBarrier(
   sqlite3_file *fd                /* Database file holding the shared memory */
 ){
   UNUSED_PARAMETER(fd);
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  __sync_synchronize();
+#else
   sqlite3MemoryBarrier();         /* compiler-defined memory barrier */
   assert( fd->pMethods->xLock==nolockLock
        || unixFileMutexNotheld((unixFile*)fd)
   );
   unixEnterMutex();               /* Also mutex, for redundancy */
   unixLeaveMutex();
+#endif
 }
 
 /*
@@ -43724,6 +44117,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -43855,6 +44251,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -44297,6 +44715,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();
@@ -54056,6 +54477,12 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){
   if( p==0 ) return SQLITE_OK;
   assert( i>0 );
   assert( i<=p->iSize );
+  if( i>p->iSize || i==0 ){
+    sqlite3_log(SQLITE_ERROR,
+        "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize
+    );
+    abort();
+  }
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {
     u32 bin = i/p->iDivisor;
@@ -55871,6 +56298,14 @@ static void pcache1EnforceMaxPage(PCache1 *pCache){
     pcache1RemoveFromHash(p, 1);
   }
   if( pCache->nPage==0 && pCache->pBulk ){
+    if( pcache1.separateCache ){
+      PgHdr1 *p1 = pCache->pFree;
+      while( p1 ){
+        PgHdr1 *pNext = p1->pNext;
+        if( p1->isBulkLocal==0 ) pcache1Free(p1->page.pBuf);
+        p1 = pNext;
+      }
+    }
     sqlite3_free(pCache->pBulk);
     pCache->pBulk = pCache->pFree = 0;
   }
@@ -55916,7 +56351,13 @@ static void pcache1TruncateUnsafe(
         pCache->nPage--;
         *pp = pPage->pNext;
         if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage);
-        pcache1FreePage(pPage);
+        if( pcache1.separateCache ){
+          pPage->pNext = pCache->pFree;
+          pCache->pFree = pPage;
+          (*pCache->pnPurgeable)--;
+        }else{
+          pcache1FreePage(pPage);
+        }
       }else{
         pp = &pPage->pNext;
         TESTONLY( if( nPage>=0 ) nPage++; )
@@ -56414,6 +56855,15 @@ static void pcache1Destroy(sqlite3_pcache *p){
   assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) );
   pcache1EnterMutex(pGroup);
   if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0);
+  if( pcache1.separateCache ){
+    PgHdr1 *p1 = pCache->pFree;
+    while( p1 ){
+      PgHdr1 *pNext = p1->pNext;
+      if( p1->isBulkLocal==0 ) pcache1Free(p1->page.pBuf);
+      p1 = pNext;
+    }
+    pCache->pFree = 0;
+  }
   assert( pGroup->nMaxPage >= pCache->nMax );
   pGroup->nMaxPage -= pCache->nMax;
   assert( pGroup->nMinPage >= pCache->nMin );
@@ -57078,7 +57528,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64
 #define CKPT_SYNC_FLAGS(X)  (((X)>>2)&0x03)
 
 #ifdef SQLITE_OMIT_WAL
-# define sqlite3WalOpen(x,y,z)                   0
+# define sqlite3WalOpen(w,x,y,z)                 0
 # define sqlite3WalLimit(x,y)
 # define sqlite3WalClose(v,w,x,y,z)              0
 # define sqlite3WalBeginReadTransaction(y,z)     0
@@ -57086,7 +57536,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64
 # define sqlite3WalDbsize(y)                     0
 # define sqlite3WalBeginWriteTransaction(y)      0
 # define sqlite3WalEndWriteTransaction(x)        0
-# define sqlite3WalUndo(x,y,z)                   0
+# define sqlite3WalUndo(w,x,y,z)                 0
 # define sqlite3WalSavepoint(y,z)
 # define sqlite3WalSavepointUndo(y,z)            0
 # define sqlite3WalFrames(u,v,w,x,y,z)           0
@@ -57097,6 +57547,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64
 # define sqlite3WalFramesize(z)                  0
 # define sqlite3WalFindFrame(x,y,z)              0
 # define sqlite3WalFile(x)                       0
+# define sqlite3WalJournalMode(x)                0
 # undef SQLITE_USE_SEH
 #else
 
@@ -57108,7 +57559,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64
 typedef struct Wal Wal;
 
 /* Open and close a connection to a write-ahead log. */
-SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**);
+SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *,int,i64,int,Wal**);
 SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *);
 
 /* Set the limiting size of a WAL file. */
@@ -57136,7 +57587,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal);
 SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal);
 
 /* Undo any frames written (but not committed) to the log */
-SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);
+SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, int);
 
 /* Return an integer that records the current (uncommitted) write
 ** position in the WAL */
@@ -57189,6 +57640,15 @@ SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapsho
 SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* Tell the wal layer that we want to commit a concurrent transaction */
+SQLITE_PRIVATE int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, u32*);
+
+/* Upgrade the state of the client to take into account changes written
+** by other connections */
+SQLITE_PRIVATE int sqlite3WalUpgradeSnapshot(Wal *pWal);
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 #ifdef SQLITE_ENABLE_ZIPVFS
 /* If the WAL file is not empty, return the number of bytes of content
 ** stored in each frame (i.e. the db page-size when the WAL was created).
@@ -57199,6 +57659,9 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal);
 /* Return the sqlite3_file object for the WAL file */
 SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal);
 
+/* Return the journal mode (WAL or WAL2) used by this Wal object. */
+SQLITE_PRIVATE int sqlite3WalJournalMode(Wal *pWal);
+
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
 SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock);
 SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db);
@@ -57208,6 +57671,12 @@ SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db);
 SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal*);
 #endif
 
+/* sqlite3_wal_info() data */
+SQLITE_PRIVATE int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
+/* sqlite3_wal_info() data */
+SQLITE_PRIVATE int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */
 
@@ -57850,6 +58319,9 @@ struct Pager {
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   u32 nSubRec;                /* Number of records written to sub-journal */
   Bitvec *pInJournal;         /* One bit for each page in the database file */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Bitvec *pAllRead;           /* Pages read within current CONCURRENT trans. */
+#endif
   sqlite3_file *fd;           /* File descriptor for database */
   sqlite3_file *jfd;          /* File descriptor for main journal */
   sqlite3_file *sjfd;         /* File descriptor for sub-journal */
@@ -57981,20 +58453,6 @@ static const unsigned char aJournalMagic[] = {
 # define USEFETCH(x) 0
 #endif
 
-/*
-** The argument to this macro is a file descriptor (type sqlite3_file*).
-** Return 0 if it is not open, or non-zero (but not 1) if it is.
-**
-** This is so that expressions can be written as:
-**
-**   if( isOpen(pPager->jfd) ){ ...
-**
-** instead of
-**
-**   if( pPager->jfd->pMethods ){ ...
-*/
-#define isOpen(pFd) ((pFd)->pMethods!=0)
-
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
 /*
 ** Return true if page pgno can be read directly from the database file
@@ -58106,7 +58564,9 @@ static int assert_pager_state(Pager *p){
       if( !pagerUseWal(pPager) ){
         assert( p->eLock>=RESERVED_LOCK );
       }
-      assert( pPager->dbSize==pPager->dbOrigSize );
+#ifndef SQLITE_OMIT_CONCURRENT
+      assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead );
+#endif
       assert( pPager->dbOrigSize==pPager->dbFileSize );
       assert( pPager->dbOrigSize==pPager->dbHintSize );
       assert( pPager->setSuper==0 );
@@ -58125,6 +58585,7 @@ static int assert_pager_state(Pager *p){
         assert( isOpen(p->jfd)
              || p->journalMode==PAGER_JOURNALMODE_OFF
              || p->journalMode==PAGER_JOURNALMODE_WAL
+             || p->journalMode==PAGER_JOURNALMODE_WAL2
         );
       }
       assert( pPager->dbOrigSize==pPager->dbFileSize );
@@ -58139,6 +58600,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd)
            || p->journalMode==PAGER_JOURNALMODE_OFF
            || p->journalMode==PAGER_JOURNALMODE_WAL
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       assert( pPager->dbOrigSize<=pPager->dbHintSize );
@@ -58151,6 +58613,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd)
            || p->journalMode==PAGER_JOURNALMODE_OFF
            || p->journalMode==PAGER_JOURNALMODE_WAL
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       break;
@@ -59014,6 +59477,53 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If they are not already, begin recording all pages read from the pager layer
+** by the b-tree layer This is used by concurrent transactions. Return
+** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error
+** occurs.
+*/
+SQLITE_PRIVATE int sqlite3PagerBeginConcurrent(Pager *pPager){
+  int rc = SQLITE_OK;
+  if( pPager->pAllRead==0 ){
+    pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize);
+    pPager->dbOrigSize = pPager->dbSize;
+    if( pPager->pAllRead==0 ){
+      rc = SQLITE_NOMEM;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Stop recording all pages read from the pager layer by the b-tree layer
+** and discard any current records.
+*/
+SQLITE_PRIVATE void sqlite3PagerEndConcurrent(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pAllRead);
+  pPager->pAllRead = 0;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Return true if the database is in wal mode. False otherwise.
+*/
+SQLITE_PRIVATE int sqlite3PagerIsWal(Pager *pPager){
+  return pPager->pWal!=0;
+}
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+/*
+** Free the Pager.pInJournal and Pager.pAllRead bitvec objects.
+*/
+static void pagerFreeBitvecs(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pInJournal);
+  pPager->pInJournal = 0;
+  sqlite3PagerEndConcurrent(pPager);
+}
+
 /*
 ** This function is a no-op if the pager is in exclusive mode and not
 ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN
@@ -59038,8 +59548,7 @@ static void pager_unlock(Pager *pPager){
        || pPager->eState==PAGER_ERROR
   );
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   releaseAllSavepoints(pPager);
 
   if( pagerUseWal(pPager) ){
@@ -59272,7 +59781,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       }
       pPager->journalOff = 0;
     }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
-      || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
+      || (pPager->exclusiveMode && pPager->journalMode<PAGER_JOURNALMODE_WAL)
     ){
       rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile);
       pPager->journalOff = 0;
@@ -59287,6 +59796,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE
            || pPager->journalMode==PAGER_JOURNALMODE_MEMORY
            || pPager->journalMode==PAGER_JOURNALMODE_WAL
+           || pPager->journalMode==PAGER_JOURNALMODE_WAL2
       );
       sqlite3OsClose(pPager->jfd);
       if( bDelete ){
@@ -59306,8 +59816,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
   }
 #endif
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   pPager->nRec = 0;
   if( rc==SQLITE_OK ){
     if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){
@@ -60337,8 +60846,24 @@ static int pagerRollbackWal(Pager *pPager){
   **   + Reload page content from the database (if refcount>0).
   */
   pPager->dbSize = pPager->dbOrigSize;
-  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager);
+  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager,
+#ifdef SQLITE_OMIT_CONCURRENT
+      0
+#else
+      pPager->pAllRead!=0
+#endif
+  );
   pList = sqlite3PcacheDirtyList(pPager->pPCache);
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction, then page 1 must be reread from
+  ** the db file, even if it is not dirty. This is because the b-tree layer
+  ** may have already zeroed the nFree and iTrunk header fields.  */
+  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){
+    rc = pagerUndoCallback((void*)pPager, 1);
+  }
+#endif
+
   while( pList && rc==SQLITE_OK ){
     PgHdr *pNext = pList->pDirty;
     rc = pagerUndoCallback((void *)pPager, pList->pgno);
@@ -60388,6 +60913,8 @@ static int pagerWalFrames(
       if( p->pgno<=nTruncate ){
         ppNext = &p->pDirty;
         nList++;
+        PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n",
+                     PAGERID(pPager), p->pgno, pager_pagehash(p)));
       }
     }
     assert( pList );
@@ -60442,6 +60969,10 @@ static int pagerBeginReadTransaction(Pager *pPager){
   if( rc!=SQLITE_OK || changed ){
     pager_reset(pPager);
     if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
+    assert( pPager->journalMode==PAGER_JOURNALMODE_WAL
+         || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+    );
+    pPager->journalMode = sqlite3WalJournalMode(pPager->pWal);
   }
 
   return rc;
@@ -60537,9 +61068,9 @@ static int pagerOpenWalIfPresent(Pager *pPager){
           rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0);
         }else{
           testcase( sqlite3PcachePagecount(pPager->pPCache)==0 );
-          rc = sqlite3PagerOpenWal(pPager, 0);
+          rc = sqlite3PagerOpenWal(pPager, 0, 0);
         }
-      }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){
+      }else if( pPager->journalMode>=PAGER_JOURNALMODE_WAL ){
         pPager->journalMode = PAGER_JOURNALMODE_DELETE;
       }
     }
@@ -61458,7 +61989,7 @@ static int syncJournal(Pager *pPager, int newHdr){
   assert( assert_pager_state(pPager) );
   assert( !pagerUseWal(pPager) );
 
-  rc = sqlite3PagerExclusiveLock(pPager);
+  rc = sqlite3PagerExclusiveLock(pPager, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   if( !pPager->noSync ){
@@ -61809,6 +62340,12 @@ static int pagerStress(void *p, PgHdr *pPg){
   pPager->aStat[PAGER_STAT_SPILL]++;
   pPg->pDirty = 0;
   if( pagerUseWal(pPager) ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    /* If the transaction is a "BEGIN CONCURRENT" transaction, the page
+    ** cannot be flushed to disk. Return early in this case. */
+    if( pPager->pAllRead ) return SQLITE_OK;
+#endif
+
     /* Write a single frame for this page to the log. */
     rc = subjournalPageIfRequired(pPg);
     if( rc==SQLITE_OK ){
@@ -62044,6 +62581,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
     nPathname + 8 + 1 +                  /* Journal filename */
 #ifndef SQLITE_OMIT_WAL
     nPathname + 4 + 1 +                  /* WAL filename */
+    nPathname + 5 + 1 +                  /* Second WAL filename */
 #endif
     3                                    /* Terminator */
   );
@@ -62096,6 +62634,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
     sqlite3FileSuffix3(zFilename, pPager->zWal);
     pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1);
 #endif
+    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
+    memcpy(pPtr, "-wal2", 5);             pPtr += 5 + 1;
   }else{
     pPager->zWal = 0;
   }
@@ -62640,6 +63180,23 @@ static void pagerUnlockIfUnused(Pager *pPager){
   }
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If this pager is currently in a concurrent transaction (pAllRead!=0),
+** then set the bit in the pAllRead vector to indicate that the transaction
+** read from page pgno. Return SQLITE_OK if successful, or an SQLite error
+** code (i.e. SQLITE_NOMEM) if an error occurs.
+*/
+SQLITE_PRIVATE int sqlite3PagerUsePage(Pager *pPager, Pgno pgno){
+  int rc = SQLITE_OK;
+  if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
+    PAGERTRACE(("USING page %d\n", pgno));
+    rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
+  }
+  return rc;
+}
+#endif
+
 /*
 ** The page getter methods each try to acquire a reference to a
 ** page with page number pgno. If the requested reference is
@@ -62713,6 +63270,14 @@ static int getPageNormal(
   assert( assert_pager_state(pPager) );
   assert( pPager->hasHeldSharedLock==1 );
 
+  /* If this is an CONCURRENT transaction and the page being read was
+  ** present in the database file when the transaction was opened,
+  ** mark it as read in the pAllRead vector.  */
+  if( (rc = sqlite3PagerUsePage(pPager, pgno))!=SQLITE_OK ){
+    pPg = 0;
+    goto pager_acquire_err;
+  }
+
   if( pgno==0 ) return SQLITE_CORRUPT_BKPT;
   pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3);
   if( pBase==0 ){
@@ -63071,11 +63636,14 @@ static int pager_open_journal(Pager *pPager){
 ** Begin a write-transaction on the specified pager object. If a
 ** write-transaction has already been opened, this function is a no-op.
 **
-** If the exFlag argument is false, then acquire at least a RESERVED
-** lock on the database file. If exFlag is true, then acquire at least
+** If the exFlag argument is 0, then acquire at least a RESERVED
+** lock on the database file. If exFlag is >0, then acquire at least
 ** an EXCLUSIVE lock. If such a lock is already held, no locking
 ** functions need be called.
 **
+** If (exFlag<0) and the database is in WAL mode, do not take any locks.
+** The transaction will run in CONCURRENT mode instead.
+**
 ** If the subjInMemory argument is non-zero, then any sub-journal opened
 ** within this transaction will be opened as an in-memory file. This
 ** has no effect if the sub-journal is already opened (as it may be when
@@ -63093,7 +63661,6 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory
 
   if( pPager->eState==PAGER_READER ){
     assert( pPager->pInJournal==0 );
-
     if( pagerUseWal(pPager) ){
       /* If the pager is configured to use locking_mode=exclusive, and an
       ** exclusive lock on the database is not already held, obtain it now.
@@ -63109,9 +63676,10 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory
       /* Grab the write lock on the log file. If successful, upgrade to
       ** PAGER_RESERVED state. Otherwise, return an error code to the caller.
       ** The busy-handler is not invoked if another connection already
-      ** holds the write-lock. If possible, the upper layer will call it.
-      */
-      rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      ** holds the write-lock. If possible, the upper layer will call it.  */
+      if( exFlag>=0 ){
+        rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      }
     }else{
       /* Obtain a RESERVED lock on the database file. If the exFlag parameter
       ** is true, then immediately upgrade this to an EXCLUSIVE lock. The
@@ -63119,7 +63687,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory
       ** lock, but not when obtaining the RESERVED lock.
       */
       rc = pagerLockDb(pPager, RESERVED_LOCK);
-      if( rc==SQLITE_OK && exFlag ){
+      if( rc==SQLITE_OK && exFlag>0 ){
         rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
       }
     }
@@ -63419,7 +63987,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){
 ** to sqlite3PagerWrite().  In other words, return TRUE if it is ok
 ** to change the content of the page.
 */
-#ifndef NDEBUG
+#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG)
 SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){
   return pPg->flags & PGHDR_WRITEABLE;
 }
@@ -63575,17 +64143,26 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper){
 }
 
 /*
-** This function may only be called while a write-transaction is active in
-** rollback. If the connection is in WAL mode, this call is a no-op.
-** Otherwise, if the connection does not already have an EXCLUSIVE lock on
-** the database file, an attempt is made to obtain one.
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
 **
-** If the EXCLUSIVE lock is already held or the attempt to obtain it is
-** successful, or the connection is in WAL mode, SQLITE_OK is returned.
-** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is
-** returned.
+** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this
+** function is a no-op.
+**
+** If this is an CONCURRENT transaction on a wal-mode database, this function
+** attempts to obtain the WRITER lock on the wal file and also checks to
+** see that the transaction can be safely committed (does not commit with
+** any other transaction committed since it was opened).
+**
+** If the required locks are already held or successfully obtained and
+** the transaction can be committed, SQLITE_OK is returned. If a required lock
+** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction
+** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT
+** is returned. Otherwise, if some other error occurs (IO error, OOM etc.),
+** and SQLite error code is returned.
 */
-SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){
+SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, u32 *aConflict){
   int rc = pPager->errCode;
   assert( assert_pager_state(pPager) );
   if( rc==SQLITE_OK ){
@@ -63597,10 +64174,73 @@ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){
     if( 0==pagerUseWal(pPager) ){
       rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
     }
+#ifndef SQLITE_OMIT_CONCURRENT
+    else{
+      if( pPager->pAllRead ){
+        /* This is an CONCURRENT transaction. Attempt to lock the wal database
+        ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned,
+        ** invoke the busy-handler and try again for as long as it returns
+        ** non-zero.  */
+        do {
+          rc = sqlite3WalLockForCommit(
+              pPager->pWal, pPage1, pPager->pAllRead, aConflict
+          );
+        }while( rc==SQLITE_BUSY
+             && pPager->xBusyHandler(pPager->pBusyHandlerArg)
+        );
+      }
+    }
+#endif /* SQLITE_OMIT_CONCURRENT */
   }
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of committing an CONCURRENT transaction.
+** At this point the wal WRITER lock is held, and all pages in the cache
+** except for page 1 are compatible with the snapshot at the head of the
+** wal file.
+**
+** This function updates the in-memory data structures and reloads the
+** contents of page 1 so that the client is operating on the snapshot
+** at the head of the wal file.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+SQLITE_PRIVATE int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){
+  int rc;
+
+  assert( pPager->pWal && pPager->pAllRead );
+  rc = sqlite3WalUpgradeSnapshot(pPager->pWal);
+  if( rc==SQLITE_OK ){
+    rc = readDbPage(pPage1);
+  }
+
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Set the in-memory cache of the database file size to nSz pages.
+*/
+SQLITE_PRIVATE void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){
+  pPager->dbSize = nSz;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If this is a WAL mode connection and the WRITER lock is currently held,
+** relinquish it.
+*/
+SQLITE_PRIVATE void sqlite3PagerDropExclusiveLock(Pager *pPager){
+  if( pagerUseWal(pPager) ){
+    sqlite3WalEndWriteTransaction(pPager->pWal);
+  }
+}
+#endif  /* SQLITE_OMIT_CONCURRENT */
+
+
 /*
 ** Sync the database file for the pager pPager. zSuper points to the name
 ** of a super-journal file that should be written into the individual
@@ -64532,7 +65172,8 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
             || eMode==PAGER_JOURNALMODE_OFF       /* 2 */
             || eMode==PAGER_JOURNALMODE_TRUNCATE  /* 3 */
             || eMode==PAGER_JOURNALMODE_MEMORY    /* 4 */
-            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */ );
+            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */
+            || eMode==PAGER_JOURNALMODE_WAL2      /* 6 */ );
 
   /* This routine is only called from the OP_JournalMode opcode, and
   ** the logic there will never allow a temporary file to be changed
@@ -64566,9 +65207,12 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
     assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 );
     assert( (PAGER_JOURNALMODE_OFF & 5)==0 );
     assert( (PAGER_JOURNALMODE_WAL & 5)==5 );
+    assert( (PAGER_JOURNALMODE_WAL2 & 5)==4 );
 
     assert( isOpen(pPager->fd) || pPager->exclusiveMode );
-    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){
+    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0
+     && eMode!=PAGER_JOURNALMODE_WAL2       /* TODO: fix this if possible */
+    ){
       /* In this case we would like to delete the journal file. If it is
       ** not possible, then that is not a problem. Deleting the journal file
       ** here is an optimization only.
@@ -64743,7 +65387,7 @@ static int pagerExclusiveLock(Pager *pPager){
 ** lock on the database file and use heap-memory to store the wal-index
 ** in. Otherwise, use the normal shared-memory.
 */
-static int pagerOpenWal(Pager *pPager){
+static int pagerOpenWal(Pager *pPager, int bWal2){
   int rc = SQLITE_OK;
 
   assert( pPager->pWal==0 && pPager->tempFile==0 );
@@ -64764,7 +65408,7 @@ static int pagerOpenWal(Pager *pPager){
   if( rc==SQLITE_OK ){
     rc = sqlite3WalOpen(pPager->pVfs,
         pPager->fd, pPager->zWal, pPager->exclusiveMode,
-        pPager->journalSizeLimit, &pPager->pWal
+        pPager->journalSizeLimit, bWal2, &pPager->pWal
     );
   }
   pagerFixMaplimit(pPager);
@@ -64790,6 +65434,7 @@ static int pagerOpenWal(Pager *pPager){
 */
 SQLITE_PRIVATE int sqlite3PagerOpenWal(
   Pager *pPager,                  /* Pager object */
+  int bWal2,                      /* Open in wal2 mode if not already open */
   int *pbOpen                     /* OUT: Set to true if call is a no-op */
 ){
   int rc = SQLITE_OK;             /* Return code */
@@ -64806,9 +65451,9 @@ SQLITE_PRIVATE int sqlite3PagerOpenWal(
     /* Close any rollback journal previously open */
     sqlite3OsClose(pPager->jfd);
 
-    rc = pagerOpenWal(pPager);
+    rc = pagerOpenWal(pPager, bWal2);
     if( rc==SQLITE_OK ){
-      pPager->journalMode = PAGER_JOURNALMODE_WAL;
+      pPager->journalMode = bWal2?PAGER_JOURNALMODE_WAL2:PAGER_JOURNALMODE_WAL;
       pPager->eState = PAGER_OPEN;
     }
   }else{
@@ -64830,7 +65475,9 @@ SQLITE_PRIVATE int sqlite3PagerOpenWal(
 SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
   int rc = SQLITE_OK;
 
-  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL );
+  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL
+       || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+  );
 
   /* If the log file is not already open, but does exist in the file-system,
   ** it may need to be checkpointed before the connection can switch to
@@ -64845,7 +65492,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
       );
     }
     if( rc==SQLITE_OK && logexists ){
-      rc = pagerOpenWal(pPager);
+      rc = pagerOpenWal(pPager, 0);
     }
   }
 
@@ -64968,6 +65615,11 @@ SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){
 }
 
 #endif /* SQLITE_ENABLE_SNAPSHOT */
+
+SQLITE_PRIVATE int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){
+  return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame);
+}
+
 #endif /* !SQLITE_OMIT_WAL */
 
 #ifdef SQLITE_ENABLE_ZIPVFS
@@ -65097,7 +65749,7 @@ SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){
 **
 ** To read a page from the database (call it page number P), a reader
 ** first checks the WAL to see if it contains page P.  If so, then the
-** last valid instance of page P that is a followed by a commit frame
+** last valid instance of page P that is followed by a commit frame
 ** or is a commit frame itself becomes the value read.  If the WAL
 ** contains no copies of page P that are valid and which are a commit
 ** frame or are followed by a commit frame, then page P is read from
@@ -65232,7 +65884,7 @@ SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){
 ** and to the wal-index) might be using a different value K1, where K1>K0.
 ** Both readers can use the same hash table and mapping section to get
 ** the correct result.  There may be entries in the hash table with
-** K>K0 but to the first reader, those entries will appear to be unused
+** K>K0, but to the first reader those entries will appear to be unused
 ** slots in the hash table and so the first reader will get an answer as
 ** if no values greater than K0 had ever been inserted into the hash table
 ** in the first place - which is what reader one wants.  Meanwhile, the
@@ -65243,6 +65895,191 @@ SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){
 ** that correspond to frames greater than the new K value are removed
 ** from the hash table at this point.
 */
+
+/*
+** WAL2 NOTES
+**
+** This file also contains the implementation of "wal2" mode - activated
+** using "PRAGMA journal_mode = wal2". Wal2 mode is very similar to wal
+** mode, except that it uses two wal files instead of one. Under some
+** circumstances, wal2 mode provides more concurrency than legacy wal
+** mode.
+**
+** THE PROBLEM WAL2 SOLVES:
+**
+** In legacy wal mode, if a writer wishes to write to the database while
+** a checkpoint is ongoing, it may append frames to the existing wal file.
+** This means that after the checkpoint has finished, the wal file consists
+** of a large block of checkpointed frames, followed by a block of
+** uncheckpointed frames. In a deployment that features a high volume of
+** write traffic, this may mean that the wal file is never completely
+** checkpointed. And so grows indefinitely.
+**
+** An alternative is to use "PRAGMA wal_checkpoint=RESTART" or similar to
+** force a complete checkpoint of the wal file. But this must:
+**
+**   1) Wait on all existing readers to finish,
+**   2) Wait on any existing writer, and then block all new writers,
+**   3) Do the checkpoint,
+**   4) Wait on any new readers that started during steps 2 and 3. Writers
+**      are still blocked during this step.
+**
+** This means that in order to avoid the wal file growing indefinitely
+** in a busy system, writers must periodically pause to allow a checkpoint
+** to complete. In a system with long running readers, such pauses may be
+** for a non-trivial amount of time.
+**
+** OVERVIEW OF SOLUTION
+**
+** Wal2 mode uses two wal files. After writers have grown the first wal
+** file to a pre-configured size, they begin appending transactions to
+** the second wal file. Once all existing readers are reading snapshots
+** new enough to include the entire first wal file, a checkpointer can
+** checkpoint it.
+**
+** Meanwhile, writers are writing transactions to the second wal file.
+** Once that wal file has grown larger than the pre-configured size, each
+** new writer checks if:
+**
+**    * the first wal file has been checkpointed, and if so, if
+**    * there are no readers still reading from the first wal file (once
+**      it has been checkpointed, new readers read only from the second
+**      wal file).
+**
+** If both these conditions are true, the writer may switch back to the
+** first wal file. Eventually, a checkpointer can checkpoint the second
+** wal file, and so on.
+**
+** The wal file that writers are currently appending to (the one they
+** don't have to check the above two criteria before writing to) is called
+** the "current" wal file.
+**
+** The first wal file takes the same name as the wal file in legacy wal
+** mode systems - "<db>-wal". The second is named "<db>-wal2".
+
+**
+** CHECKPOINTS
+**
+** The "pre-configured size" mentioned above is the value set by
+** "PRAGMA journal_size_limit". Or, if journal_size_limit is not set,
+** 1000 pages.
+**
+** There is only a single type of checkpoint in wal2 mode (no "truncate",
+** "restart" etc.), and it always checkpoints the entire contents of a single
+** wal file. A wal file cannot be checkpointed until after a writer has written
+** the first transaction into the other wal file and all readers are reading a
+** snapshot that includes at least one transaction from the other wal file.
+**
+** The wal-hook, if one is registered, is invoked after a write-transaction
+** is committed, just as it is in legacy wal mode. The integer parameter
+** passed to the wal-hook is the total number of uncheckpointed frames in both
+** wal files. Except, the parameter is set to zero if there is no frames
+** that may be checkpointed. This happens in two scenarios:
+**
+**   1. The "other" wal file (the one that the writer did not just append to)
+**      is completely empty, or
+**
+**   2. The "other" wal file (the one that the writer did not just append to)
+**      has already been checkpointed.
+**
+**
+** WAL FILE FORMAT
+**
+** The file format used for each wal file in wal2 mode is the same as for
+** legacy wal mode.  Except, the file format field is set to 3021000
+** instead of 3007000.
+**
+** WAL-INDEX FORMAT
+**
+** The wal-index format is also very similar. Even though there are two
+** wal files, there is still a single wal-index shared-memory area (*-shm
+** file with the default unix or win32 VFS). The wal-index header is the
+** same size, with the following exceptions it has the same format:
+**
+**   * The version field is set to 3021000 instead of 3007000.
+**
+**   * An unused 32-bit field in the legacy wal-index header is
+**     now used to store (a) a single bit indicating which of the
+**     two wal files writers should append to and (b) the number
+**     of frames in the second wal file (31 bits).
+**
+** The first hash table in the wal-index contains entries corresponding
+** to the first HASHTABLE_NPAGE_ONE frames stored in the first wal file.
+** The second hash table in the wal-index contains entries indexing the
+** first HASHTABLE_NPAGE frames in the second wal file. The third hash
+** table contains the next HASHTABLE_NPAGE frames in the first wal file,
+** and so on.
+**
+** LOCKS
+**
+** Read-locks are simpler than for legacy wal mode. There are no locking
+** slots that contain frame numbers. Instead, there are four distinct
+** combinations of read locks a reader may hold:
+**
+**   WAL_LOCK_PART1:       "part" lock on first wal, none of second.
+**   WAL_LOCK_PART1_FULL2: "part" lock on first wal, "full" of second.
+**   WAL_LOCK_PART2: no lock on first wal, "part" lock on second.
+**   WAL_LOCK_PART2_FULL1: "full" lock on first wal, "part" lock on second.
+**
+** When a reader reads the wal-index header as part of opening a read
+** transaction, it takes a "part" lock on the current wal file. "Part"
+** because the wal file may grow while the read transaction is active, in
+** which case the reader would be reading only part of the wal file.
+** A part lock prevents a checkpointer from checkpointing the wal file
+** on which it is held.
+**
+** If there is data in the non-current wal file that has not been
+** checkpointed, the reader takes a "full" lock on that wal file. A
+** "full" lock indicates that the reader is using the entire wal file.
+** A full lock prevents a writer from overwriting the wal file on which
+** it is held, but does not prevent a checkpointer from checkpointing
+** it.
+**
+** There is still a single WRITER and a single CHECKPOINTER lock. The
+** recovery procedure still takes the same exclusive lock on the entire
+** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks
+** above use four of the six read-locking slots used by legacy wal mode.
+**
+** STARTUP/RECOVERY
+**
+** The read and write version fields of the database header in a wal2
+** database are set to 0x03, instead of 0x02 as in legacy wal mode.
+**
+** The wal file format used in wal2 mode is the same as the format used
+** in legacy wal mode. However, in order to support recovery, there are two
+** differences in the way wal file header fields are populated, as follows:
+**
+**   * When the first wal file is first created, the "nCkpt" field in
+**     the wal file header is set to 0. Thereafter, each time the writer
+**     switches wal file, it sets the nCkpt field in the new wal file
+**     header to ((nCkpt0 + 1) & 0x0F), where nCkpt0 is the value in
+**     the previous wal file header. This means that the first wal file
+**     always has an even value in the nCkpt field, and the second wal
+**     file always has an odd value.
+**
+**   * When a writer switches wal file, it sets the salt values in the
+**     new wal file to a copy of the checksum for the final frame in
+**     the previous wal file.
+**
+** Recovery proceeds as follows:
+**
+** 1. Each wal file is recovered separately. Except, if the first wal
+**    file does not exist or is zero bytes in size, the second wal file
+**    is truncated to zero bytes before it is "recovered".
+**
+** 2. If both wal files contain valid headers, then the nCkpt fields
+**    are compared to see which of the two wal files is older. If the
+**    salt keys in the second wal file match the final frame checksum
+**    in the older wal file, then both wal files are used. Otherwise,
+**    the newer wal file is ignored.
+**
+** 3. Or, if only one or neither of the wal files has a valid header,
+**    then only a single or no wal files are recovered into the
+**    reconstructed wal-index.
+**
+** Refer to header comments for walIndexRecover() for further details.
+*/
+
 #ifndef SQLITE_OMIT_WAL
 
 /* #include "wal.h" */
@@ -65258,20 +66095,20 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0;
 #endif
 
 /*
-** The maximum (and only) versions of the wal and wal-index formats
-** that may be interpreted by this version of SQLite.
+** Both the wal-file and the wal-index contain version fields
+** indicating the current version of the system. If a client
+** reads the header of a wal file (as part of recovery), or the
+** wal-index (as part of opening a read transaction) and (a) the
+** header checksum is correct but (b) the version field is not
+** recognized, the operation fails with SQLITE_CANTOPEN.
 **
-** If a client begins recovering a WAL file and finds that (a) the checksum
-** values in the wal-header are correct and (b) the version field is not
-** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN.
-**
-** Similarly, if a client successfully reads a wal-index header (i.e. the
-** checksum test is successful) and finds that the version field is not
-** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite
-** returns SQLITE_CANTOPEN.
+** Currently, clients support both version-1 ("journal_mode=wal") and
+** version-2 ("journal_mode=wal2"). Legacy clients may support version-1
+** only.
 */
-#define WAL_MAX_VERSION      3007000
-#define WALINDEX_MAX_VERSION 3007000
+#define WAL_VERSION1 3007000      /* For "journal_mode=wal" */
+#define WAL_VERSION2 3021000      /* For "journal_mode=wal2" */
+
 
 /*
 ** Index numbers for various locking bytes.   WAL_NREADER is the number
@@ -65294,6 +66131,39 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0;
 #define WAL_READ_LOCK(I)       (3+(I))
 #define WAL_NREADER            (SQLITE_SHM_NLOCK-3)
 
+/*
+** Values that may be stored in Wal.readLock in wal2 mode.
+**
+** In wal mode, the Wal.readLock member is set to -1 when no read-lock
+** is held, or else is the index of the read-mark on which a lock is
+** held.
+**
+** In wal2 mode, a value of -1 still indicates that no read-lock is held.
+** And a non-zero value still represents the index of the read-mark on
+** which a lock is held. There are two differences:
+**
+**   1. wal2 mode never uses read-mark 0.
+**
+**   2. locks on each read-mark have a different interpretation, as
+**      indicated by the symbolic names below.
+*/
+#define WAL_LOCK_NONE        -1
+#define WAL_LOCK_PART1        1
+#define WAL_LOCK_PART1_FULL2  2
+#define WAL_LOCK_PART2_FULL1  3
+#define WAL_LOCK_PART2        4
+
+/*
+** This constant is used in wal2 mode only.
+**
+** In wal2 mode, when committing a transaction, if the current wal file
+** is sufficiently large and there are no conflicting locks held, the
+** writer writes the new transaction into the start of the other wal
+** file. Usually, "sufficiently large" is defined by the value configured
+** using "PRAGMA journal_size_limit". However, if no such value has been
+** configured, sufficiently large defaults to WAL_DEFAULT_WALSIZE frames.
+*/
+#define WAL_DEFAULT_WALSIZE 1000
 
 /* Object declarations */
 typedef struct WalIndexHdr WalIndexHdr;
@@ -65313,21 +66183,63 @@ typedef struct WalCkptInfo WalCkptInfo;
 ** The szPage value can be any power of 2 between 512 and 32768, inclusive.
 ** Or it can be 1 to represent a 65536-byte page.  The latter case was
 ** added in 3.7.1 when support for 64K pages was added.
+**
+** WAL2 mode notes: Member variable mxFrame2 is only used in wal2 mode
+** (when iVersion is set to WAL_VERSION2). The lower 31 bits store
+** the maximum frame number in file *-wal2. The most significant bit
+** is a flag - set if clients are currently appending to *-wal2, clear
+** otherwise.
 */
 struct WalIndexHdr {
   u32 iVersion;                   /* Wal-index version */
-  u32 unused;                     /* Unused (padding) field */
+  u32 mxFrame2;                   /* See "WAL2 mode notes" above */
   u32 iChange;                    /* Counter incremented each transaction */
   u8 isInit;                      /* 1 when initialized */
   u8 bigEndCksum;                 /* True if checksums in WAL are big-endian */
   u16 szPage;                     /* Database page size in bytes. 1==64K */
-  u32 mxFrame;                    /* Index of last valid frame in the WAL */
+  u32 mxFrame;                    /* Index of last valid frame in each WAL */
   u32 nPage;                      /* Size of database in pages */
   u32 aFrameCksum[2];             /* Checksum of last frame in log */
   u32 aSalt[2];                   /* Two salt values copied from WAL header */
   u32 aCksum[2];                  /* Checksum over all prior fields */
 };
 
+/*
+** The following macros and functions are get/set methods for the maximum
+** frame numbers and current wal file values stored in the WalIndexHdr
+** structure. These are helpful because of the unorthodox way in which
+** the values are stored in wal2 mode (see above). They are equivalent
+** to functions with the following signatures.
+**
+**   u32  walidxGetMxFrame(WalIndexHdr*, int iWal);          // get mxFrame
+**   void walidxSetMxFrame(WalIndexHdr*, int iWal, u32 val); // set mxFrame
+**   int  walidxGetFile(WalIndexHdr*)                        // get file
+**   void walidxSetFile(WalIndexHdr*, int val);              // set file
+*/
+#define walidxGetMxFrame(pHdr, iWal) \
+  ((iWal) ? ((pHdr)->mxFrame2 & 0x7FFFFFFF) : (pHdr)->mxFrame)
+
+static void walidxSetMxFrame(WalIndexHdr *pHdr, int iWal, u32 mxFrame){
+  if( iWal ){
+    pHdr->mxFrame2 = (pHdr->mxFrame2 & 0x80000000) | mxFrame;
+  }else{
+    pHdr->mxFrame = mxFrame;
+  }
+  assert( walidxGetMxFrame(pHdr, iWal)==mxFrame );
+}
+
+#define walidxGetFile(pHdr) (int)((pHdr)->mxFrame2 >> 31)
+#define walidxSetFile(pHdr, iWal) (                                          \
+    (pHdr)->mxFrame2 = ((pHdr)->mxFrame2 & 0x7FFFFFFF) | (((u32)(iWal))<<31) \
+)
+
+/*
+** Argument is a pointer to a Wal structure. Return true if the current
+** cache of the wal-index header indicates "journal_mode=wal2" mode, or
+** false otherwise.
+*/
+#define isWalMode2(pWal) ((pWal)->hdr.iVersion==WAL_VERSION2)
+
 /*
 ** A copy of the following object occurs in the wal-index immediately
 ** following the second copy of the WalIndexHdr.  This object stores
@@ -65502,7 +66414,7 @@ struct WalCkptInfo {
 struct Wal {
   sqlite3_vfs *pVfs;         /* The VFS used to create pDbFd */
   sqlite3_file *pDbFd;       /* File handle for the database file */
-  sqlite3_file *pWalFd;      /* File handle for WAL file */
+  sqlite3_file *apWalFd[2];  /* File handle for "*-wal" and "*-wal2" */
   u32 iCallback;             /* Value to pass to log callback (or 0) */
   i64 mxWalSize;             /* Truncate WAL to this size upon reset */
   int nWiData;               /* Size of array apWiData */
@@ -65522,8 +66434,11 @@ struct Wal {
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
+  u32 nPriorFrame;           /* For sqlite3WalInfo() */
   const char *zWalName;      /* Name of WAL file */
+  const char *zWalName2;     /* Name of second WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
+  FastPrng sPrng;            /* Random number generator */
 #ifdef SQLITE_USE_SEH
   u32 lockMask;              /* Mask of locks held */
   void *pFree;               /* Pointer to sqlite3_free() if exception thrown */
@@ -65539,6 +66454,8 @@ struct Wal {
   WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
   int bGetSnapshot;          /* Transaction opened for sqlite3_get_snapshot() */
 #endif
+  int bClosing;              /* Set to true at start of sqlite3WalClose() */
+  int bWal2;                 /* bWal2 flag passed to WalOpen() */
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
   sqlite3 *db;
 #endif
@@ -65615,6 +66532,7 @@ struct WalIterator {
     sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \
 )
 
+
 /*
 ** Structured Exception Handling (SEH) is a Windows-specific technique
 ** for catching exceptions raised while accessing memory-mapped files.
@@ -65721,7 +66639,6 @@ static void sehInjectFault(Wal *pWal){
 # define SEH_SET_ON_ERROR(X,Y)
 #endif /* ifdef SQLITE_USE_SEH */
 
-
 /*
 ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index
 ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are
@@ -65815,7 +66732,6 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){
 */
 static volatile WalIndexHdr *walIndexHdr(Wal *pWal){
   assert( pWal->nWiData>0 && pWal->apWiData[0] );
-  SEH_INJECT_FAULT;
   return (volatile WalIndexHdr*)pWal->apWiData[0];
 }
 
@@ -65934,7 +66850,7 @@ static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){
 
   assert( pWal->writeLock );
   pWal->hdr.isInit = 1;
-  pWal->hdr.iVersion = WALINDEX_MAX_VERSION;
+  assert( pWal->hdr.iVersion==WAL_VERSION1||pWal->hdr.iVersion==WAL_VERSION2 );
   walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum);
   /* Possible TSAN false-positive.  See tag-20200519-1 */
   memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr));
@@ -66061,7 +66977,7 @@ static const char *walLockName(int lockIdx){
 /*
 ** Set or release locks on the WAL.  Locks are either shared or exclusive.
 ** A lock cannot be moved directly between shared and exclusive - it must go
-** through the unlocked state first.
+** through the concurrent state first.
 **
 ** In locking_mode=EXCLUSIVE, all of these routines become no-ops.
 */
@@ -66177,6 +67093,38 @@ static int walHashGet(
   return rc;
 }
 
+static u32 walExternalEncode(int iWal, u32 iFrame){
+  u32 iRet;
+  if( iWal ){
+    iRet = HASHTABLE_NPAGE_ONE + iFrame;
+    iRet += ((iFrame-1) / HASHTABLE_NPAGE) * HASHTABLE_NPAGE;
+  }else{
+    iRet = iFrame;
+    iFrame += HASHTABLE_NPAGE - HASHTABLE_NPAGE_ONE;
+    iRet += ((iFrame-1) / HASHTABLE_NPAGE) * HASHTABLE_NPAGE;
+  }
+  return iRet;
+}
+
+/*
+** Parameter iExternal is an external frame identifier. This function
+** transforms it to a wal file number (0 or 1) and frame number within
+** this wal file (reported via output parameter *piRead).
+*/
+static int walExternalDecode(u32 iExternal, u32 *piRead){
+  int iHash = (iExternal+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1)/HASHTABLE_NPAGE;
+
+  if( 0==(iHash & 0x01) ){
+    /* A frame in wal file 0 */
+    *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal :
+      iExternal - (iHash/2) * HASHTABLE_NPAGE;
+    return 0;
+  }
+
+  *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
+  return 1;
+}
+
 /*
 ** Return the number of the wal-index page that contains the hash-table
 ** and page-number array that contain entries corresponding to WAL frame
@@ -66195,6 +67143,22 @@ static int walFramePage(u32 iFrame){
   return iHash;
 }
 
+/*
+** Return the index of the hash-table corresponding to frame iFrame of wal
+** file iWal.
+*/
+static int walFramePage2(int iWal, u32 iFrame){
+  int iRet;
+  assert( iWal==0 || iWal==1 );
+  assert( iFrame>0 );
+  if( iWal==0 ){
+    iRet = 2*((iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1)/HASHTABLE_NPAGE);
+  }else{
+    iRet = 1 + 2 * ((iFrame-1) / HASHTABLE_NPAGE);
+  }
+  return iRet;
+}
+
 /*
 ** Return the page number associated with frame iFrame in this WAL.
 */
@@ -66207,6 +67171,10 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){
   return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE];
 }
 
+static u32 walFramePgno2(Wal *pWal, int iWal, u32 iFrame){
+  return walFramePgno(pWal, walExternalEncode(iWal, iFrame));
+}
+
 /*
 ** Remove entries from the hash table that point to WAL slots greater
 ** than pWal->hdr.mxFrame.
@@ -66224,27 +67192,37 @@ static void walCleanupHash(Wal *pWal){
   int iLimit = 0;                 /* Zero values greater than this */
   int nByte;                      /* Number of bytes to zero in aPgno[] */
   int i;                          /* Used to iterate through aHash[] */
+  int iWal = walidxGetFile(&pWal->hdr);
+  u32 mxFrame = walidxGetMxFrame(&pWal->hdr, iWal);
+
+  u32 iExternal;
+  if( isWalMode2(pWal) ){
+    iExternal = walExternalEncode(iWal, mxFrame);
+  }else{
+    assert( iWal==0 );
+    iExternal = mxFrame;
+  }
 
   assert( pWal->writeLock );
-  testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 );
-  testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE );
-  testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 );
+  testcase( mxFrame==HASHTABLE_NPAGE_ONE-1 );
+  testcase( mxFrame==HASHTABLE_NPAGE_ONE );
+  testcase( mxFrame==HASHTABLE_NPAGE_ONE+1 );
 
-  if( pWal->hdr.mxFrame==0 ) return;
+  if( mxFrame==0 ) return;
 
   /* Obtain pointers to the hash-table and page-number array containing
   ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed
   ** that the page said hash-table and array reside on is already mapped.(1)
   */
-  assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) );
-  assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] );
-  i = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc);
+  assert( pWal->nWiData>walFramePage(iExternal) );
+  assert( pWal->apWiData[walFramePage(iExternal)] );
+  i = walHashGet(pWal, walFramePage(iExternal), &sLoc);
   if( NEVER(i) ) return; /* Defense-in-depth, in case (1) above is wrong */
 
   /* Zero all hash-table entries that correspond to frame numbers greater
   ** than pWal->hdr.mxFrame.
   */
-  iLimit = pWal->hdr.mxFrame - sLoc.iZero;
+  iLimit = iExternal - sLoc.iZero;
   assert( iLimit>0 );
   for(i=0; i<HASHTABLE_NSLOT; i++){
     if( sLoc.aHash[i]>iLimit ){
@@ -66276,16 +67254,23 @@ static void walCleanupHash(Wal *pWal){
 #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */
 }
 
-
 /*
 ** Set an entry in the wal-index that will map database page number
 ** pPage into WAL frame iFrame.
 */
-static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
+static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){
   int rc;                         /* Return code */
   WalHashLoc sLoc;                /* Wal-index hash table location */
+  u32 iExternal;
 
-  rc = walHashGet(pWal, walFramePage(iFrame), &sLoc);
+  if( isWalMode2(pWal) ){
+    iExternal = walExternalEncode(iWal, iFrame);
+  }else{
+    assert( iWal==0 );
+    iExternal = iFrame;
+  }
+
+  rc = walHashGet(pWal, walFramePage(iExternal), &sLoc);
 
   /* Assuming the wal-index file was successfully mapped, populate the
   ** page number array and hash table entry.
@@ -66295,7 +67280,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
     int idx;                      /* Value to write to hash-table slot */
     int nCollide;                 /* Number of hash collisions */
 
-    idx = iFrame - sLoc.iZero;
+    idx = iExternal - sLoc.iZero;
     assert( idx <= HASHTABLE_NSLOT/2 + 1 );
 
     /* If this is the first entry to be added to this hash-table, zero the
@@ -66359,6 +67344,217 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
   return rc;
 }
 
+/*
+** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1.
+*/
+static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){
+  i64 nSize;                      /* Size of log file */
+  u32 aFrameCksum[2] = {0, 0};
+  int rc;
+  sqlite3_file *pWalFd = pWal->apWalFd[iWal];
+
+  assert( iWal==0 || iWal==1 );
+
+  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
+  sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt);
+
+  rc = sqlite3OsFileSize(pWalFd, &nSize);
+  if( rc==SQLITE_OK ){
+    if( nSize>WAL_HDRSIZE ){
+      u8 aBuf[WAL_HDRSIZE];         /* Buffer to load WAL header into */
+      u32 *aPrivate = 0;            /* Heap copy of *-shm pg being populated */
+      u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
+      int szFrame;                  /* Number of bytes in buffer aFrame[] */
+      u8 *aData;                    /* Pointer to data part of aFrame buffer */
+      int szPage;                   /* Page size according to the log */
+      u32 magic;                    /* Magic value read from WAL header */
+      u32 version;                  /* Magic value read from WAL header */
+      int isValid;                  /* True if this frame is valid */
+      int iPg;                      /* Current 32KB wal-index page */
+      int iLastFrame;               /* Last frame in wal, based on size alone */
+      int iLastPg;                  /* Last shm page used by this wal */
+
+      /* Read in the WAL header. */
+      rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+
+      /* If the database page size is not a power of two, or is greater than
+      ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
+      ** data. Similarly, if the 'magic' value is invalid, ignore the whole
+      ** WAL file.
+      */
+      magic = sqlite3Get4byte(&aBuf[0]);
+      szPage = sqlite3Get4byte(&aBuf[8]);
+      if( (magic&0xFFFFFFFE)!=WAL_MAGIC
+       || szPage&(szPage-1)
+       || szPage>SQLITE_MAX_PAGE_SIZE
+       || szPage<512
+      ){
+        return SQLITE_OK;
+      }
+      pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
+      pWal->szPage = szPage;
+
+      /* Verify that the WAL header checksum is correct */
+      walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
+          aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
+      );
+      if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
+       || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
+      ){
+        return SQLITE_OK;
+      }
+
+      memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
+      *pnCkpt = sqlite3Get4byte(&aBuf[12]);
+
+      /* Verify that the version number on the WAL format is one that
+      ** are able to understand */
+      version = sqlite3Get4byte(&aBuf[4]);
+      if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){
+        return SQLITE_CANTOPEN_BKPT;
+      }
+      pWal->hdr.iVersion = version;
+
+      /* Malloc a buffer to read frames into. */
+      szFrame = szPage + WAL_FRAME_HDRSIZE;
+      aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ);
+      SEH_FREE_ON_ERROR(0, aFrame);
+      if( !aFrame ){
+        return SQLITE_NOMEM_BKPT;
+      }
+      aData = &aFrame[WAL_FRAME_HDRSIZE];
+      aPrivate = (u32*)&aData[szPage];
+
+      /* Read all frames from the log file. */
+      iLastFrame = (nSize - WAL_HDRSIZE) / szFrame;
+      if( version==WAL_VERSION2 ){
+        iLastPg = walFramePage2(iWal, iLastFrame);
+      }else{
+        iLastPg = walFramePage(iLastFrame);
+      }
+      for(iPg=iWal; iPg<=iLastPg; iPg+=(version==WAL_VERSION2 ? 2 : 1)){
+        u32 *aShare;
+        int iFrame;                 /* Index of last frame read */
+        int iLast;
+        int iFirst;
+        int nHdr, nHdr32;
+
+        rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare);
+        assert( aShare!=0 || rc!=SQLITE_OK );
+        if( aShare==0 ) break;
+        SEH_SET_ON_ERROR(iPg, aShare);
+        pWal->apWiData[iPg] = aPrivate;
+
+        if( iWal ){
+          assert( version==WAL_VERSION2 );
+          iFirst = 1 + (iPg/2)*HASHTABLE_NPAGE;
+          iLast = iFirst + HASHTABLE_NPAGE - 1;
+        }else{
+          int i2 = (version==WAL_VERSION2) ? (iPg/2) : iPg;
+          iLast = HASHTABLE_NPAGE_ONE+i2*HASHTABLE_NPAGE;
+          iFirst = 1 + (i2==0?0:HASHTABLE_NPAGE_ONE+(i2-1)*HASHTABLE_NPAGE);
+        }
+        iLast = MIN(iLast, iLastFrame);
+
+        for(iFrame=iFirst; iFrame<=iLast; iFrame++){
+          i64 iOffset = walFrameOffset(iFrame, szPage);
+          u32 pgno;                 /* Database page number for frame */
+          u32 nTruncate;            /* dbsize field from frame header */
+
+          /* Read and decode the next log frame. */
+          rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset);
+          if( rc!=SQLITE_OK ) break;
+          isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
+          if( !isValid ) break;
+          rc = walIndexAppend(pWal, iWal, iFrame, pgno);
+          if( NEVER(rc!=SQLITE_OK) ) break;
+
+          /* If nTruncate is non-zero, this is a commit record. */
+          if( nTruncate ){
+            pWal->hdr.mxFrame = iFrame;
+            pWal->hdr.nPage = nTruncate;
+            pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16));
+            testcase( szPage<=32768 );
+            testcase( szPage>=65536 );
+            aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
+            aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
+          }
+        }
+        pWal->apWiData[iPg] = aShare;
+        SEH_SET_ON_ERROR(0, 0);
+        nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0);
+        nHdr32 = nHdr / sizeof(u32);
+#ifndef SQLITE_SAFER_WALINDEX_RECOVERY
+        /* Memcpy() should work fine here, on all reasonable implementations.
+        ** Technically, memcpy() might change the destination to some
+        ** intermediate value before setting to the final value, and that might
+        ** cause a concurrent reader to malfunction.  Memcpy() is allowed to
+        ** do that, according to the spec, but no memcpy() implementation that
+        ** we know of actually does that, which is why we say that memcpy()
+        ** is safe for this.  Memcpy() is certainly a lot faster.
+        */
+        memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr);
+#else
+        /* In the event that some platform is found for which memcpy()
+        ** changes the destination to some intermediate value before
+        ** setting the final value, this alternative copy routine is
+        ** provided.
+        */
+        {
+          int i;
+          for(i=nHdr32; i<WALINDEX_PGSZ/sizeof(u32); i++){
+            if( aShare[i]!=aPrivate[i] ){
+              /* Atomic memory operations are not required here because if
+              ** the value needs to be changed, that means it is not being
+              ** accessed concurrently. */
+              aShare[i] = aPrivate[i];
+            }
+          }
+        }
+#endif
+        SEH_INJECT_FAULT;
+        if( iFrame<=iLast ) break;
+      }
+
+      SEH_FREE_ON_ERROR(aFrame, 0);
+      sqlite3_free(aFrame);
+    }else if( pbZero ){
+      *pbZero = 1;
+    }
+  }
+
+  pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
+  pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
+
+  return rc;
+}
+
+static int walOpenWal2(Wal *pWal){
+  int rc = SQLITE_OK;
+  if( !isOpen(pWal->apWalFd[1]) ){
+    int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
+    rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f);
+  }
+  return rc;
+}
+
+static int walTruncateWal2(Wal *pWal){
+  int bIs;
+  int rc;
+  assert( !isOpen(pWal->apWalFd[1]) );
+  rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs);
+  if( rc==SQLITE_OK && bIs ){
+    rc = walOpenWal2(pWal);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3OsTruncate(pWal->apWalFd[1], 0);
+      sqlite3OsClose(pWal->apWalFd[1]);
+    }
+  }
+  return rc;
+}
 
 /*
 ** Recover the wal-index by reading the write-ahead log file.
@@ -66372,14 +67568,16 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
 */
 static int walIndexRecover(Wal *pWal){
   int rc;                         /* Return Code */
-  i64 nSize;                      /* Size of log file */
-  u32 aFrameCksum[2] = {0, 0};
   int iLock;                      /* Lock offset to lock for checkpoint */
+  u32 nCkpt1 = 0xFFFFFFFF;
+  u32 nCkpt2 = 0xFFFFFFFF;
+  int bZero = 0;
+  WalIndexHdr hdr;
 
   /* Obtain an exclusive lock on all byte in the locking range not already
   ** locked by the caller. The caller is guaranteed to have locked the
   ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte.
-  ** If successful, the same bytes that are locked here are unlocked before
+  ** If successful, the same bytes that are locked here are concurrent before
   ** this function returns.
   */
   assert( pWal->ckptLock==1 || pWal->ckptLock==0 );
@@ -66394,164 +67592,92 @@ static int walIndexRecover(Wal *pWal){
 
   WALTRACE(("WAL%p: recovery begin...\n", pWal));
 
-  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
-
-  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
-  if( rc!=SQLITE_OK ){
-    goto recovery_error;
-  }
-
-  if( nSize>WAL_HDRSIZE ){
-    u8 aBuf[WAL_HDRSIZE];         /* Buffer to load WAL header into */
-    u32 *aPrivate = 0;            /* Heap copy of *-shm hash being populated */
-    u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
-    int szFrame;                  /* Number of bytes in buffer aFrame[] */
-    u8 *aData;                    /* Pointer to data part of aFrame buffer */
-    int szPage;                   /* Page size according to the log */
-    u32 magic;                    /* Magic value read from WAL header */
-    u32 version;                  /* Magic value read from WAL header */
-    int isValid;                  /* True if this frame is valid */
-    u32 iPg;                      /* Current 32KB wal-index page */
-    u32 iLastFrame;               /* Last frame in wal, based on nSize alone */
-
-    /* Read in the WAL header. */
-    rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
-    if( rc!=SQLITE_OK ){
-      goto recovery_error;
+  /* Recover the *-wal file. If a valid version-1 header is recovered
+  ** from it, do not open the *-wal2 file. Even if it exists.
+  **
+  ** Otherwise, if the *-wal2 file exists or if the "wal2" flag was
+  ** specified when sqlite3WalOpen() was called, open and recover
+  ** the *-wal2 file. Except, if the *-wal file was zero bytes in size,
+  ** truncate the *-wal2 to zero bytes in size.
+  **
+  ** After this block has run, if the *-wal2 file is open the system
+  ** starts up in VERSION2 mode. In this case pWal->hdr contains the
+  ** wal-index header considering only *-wal2. Stack variable hdr
+  ** contains the wal-index header considering only *-wal. The hash
+  ** tables are populated for both.
+  **
+  ** Or, if the *-wal2 file is not open, start up in VERSION1 mode.
+  ** pWal->hdr is already populated.
+  */
+  rc = walIndexRecoverOne(pWal, 0, &nCkpt1, &bZero);
+  assert( pWal->hdr.iVersion==0
+      || pWal->hdr.iVersion==WAL_VERSION1
+      || pWal->hdr.iVersion==WAL_VERSION2
+  );
+  if( rc==SQLITE_OK && bZero ){
+    rc = walTruncateWal2(pWal);
+  }
+  if( rc==SQLITE_OK && pWal->hdr.iVersion!=WAL_VERSION1 ){
+    int bOpen = 1;
+    sqlite3_vfs *pVfs = pWal->pVfs;
+    if( pWal->hdr.iVersion==0 && pWal->bWal2==0 ){
+      rc = sqlite3OsAccess(pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bOpen);
     }
-
-    /* If the database page size is not a power of two, or is greater than
-    ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
-    ** data. Similarly, if the 'magic' value is invalid, ignore the whole
-    ** WAL file.
-    */
-    magic = sqlite3Get4byte(&aBuf[0]);
-    szPage = sqlite3Get4byte(&aBuf[8]);
-    if( (magic&0xFFFFFFFE)!=WAL_MAGIC
-     || szPage&(szPage-1)
-     || szPage>SQLITE_MAX_PAGE_SIZE
-     || szPage<512
-    ){
-      goto finished;
+    if( rc==SQLITE_OK && bOpen ){
+      rc = walOpenWal2(pWal);
+      if( rc==SQLITE_OK ){
+        hdr = pWal->hdr;
+        rc = walIndexRecoverOne(pWal, 1, &nCkpt2, 0);
+      }
     }
-    pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
-    pWal->szPage = szPage;
-    pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
-    memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
+  }
 
-    /* Verify that the WAL header checksum is correct */
-    walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
-        aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
-    );
-    if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
-     || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
-    ){
-      goto finished;
-    }
+  if( rc==SQLITE_OK ){
+    volatile WalCkptInfo *pInfo;
 
-    /* Verify that the version number on the WAL format is one that
-    ** are able to understand */
-    version = sqlite3Get4byte(&aBuf[4]);
-    if( version!=WAL_MAX_VERSION ){
-      rc = SQLITE_CANTOPEN_BKPT;
-      goto finished;
-    }
+    if( isOpen(pWal->apWalFd[1]) ){
+      /* The case where *-wal2 may follow *-wal */
+      if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){
+        if( pWal->hdr.mxFrame
+         && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0]
+         && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1]
+        ){
+          walidxSetFile(&pWal->hdr, 1);
+          walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame);
+          walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame);
+        }else{
+          pWal->hdr = hdr;
+        }
+      }else
 
-    /* Malloc a buffer to read frames into. */
-    szFrame = szPage + WAL_FRAME_HDRSIZE;
-    aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ);
-    SEH_FREE_ON_ERROR(0, aFrame);
-    if( !aFrame ){
-      rc = SQLITE_NOMEM_BKPT;
-      goto recovery_error;
-    }
-    aData = &aFrame[WAL_FRAME_HDRSIZE];
-    aPrivate = (u32*)&aData[szPage];
-
-    /* Read all frames from the log file. */
-    iLastFrame = (nSize - WAL_HDRSIZE) / szFrame;
-    for(iPg=0; iPg<=(u32)walFramePage(iLastFrame); iPg++){
-      u32 *aShare;
-      u32 iFrame;                 /* Index of last frame read */
-      u32 iLast = MIN(iLastFrame, HASHTABLE_NPAGE_ONE+iPg*HASHTABLE_NPAGE);
-      u32 iFirst = 1 + (iPg==0?0:HASHTABLE_NPAGE_ONE+(iPg-1)*HASHTABLE_NPAGE);
-      u32 nHdr, nHdr32;
-      rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare);
-      assert( aShare!=0 || rc!=SQLITE_OK );
-      if( aShare==0 ) break;
-      SEH_SET_ON_ERROR(iPg, aShare);
-      pWal->apWiData[iPg] = aPrivate;
-
-      for(iFrame=iFirst; iFrame<=iLast; iFrame++){
-        i64 iOffset = walFrameOffset(iFrame, szPage);
-        u32 pgno;                 /* Database page number for frame */
-        u32 nTruncate;            /* dbsize field from frame header */
-
-        /* Read and decode the next log frame. */
-        rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset);
-        if( rc!=SQLITE_OK ) break;
-        isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
-        if( !isValid ) break;
-        rc = walIndexAppend(pWal, iFrame, pgno);
-        if( NEVER(rc!=SQLITE_OK) ) break;
-
-        /* If nTruncate is non-zero, this is a commit record. */
-        if( nTruncate ){
-          pWal->hdr.mxFrame = iFrame;
-          pWal->hdr.nPage = nTruncate;
-          pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16));
-          testcase( szPage<=32768 );
-          testcase( szPage>=65536 );
-          aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
-          aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
-        }
-      }
-      pWal->apWiData[iPg] = aShare;
-      SEH_SET_ON_ERROR(0,0);
-      nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0);
-      nHdr32 = nHdr / sizeof(u32);
-#ifndef SQLITE_SAFER_WALINDEX_RECOVERY
-      /* Memcpy() should work fine here, on all reasonable implementations.
-      ** Technically, memcpy() might change the destination to some
-      ** intermediate value before setting to the final value, and that might
-      ** cause a concurrent reader to malfunction.  Memcpy() is allowed to
-      ** do that, according to the spec, but no memcpy() implementation that
-      ** we know of actually does that, which is why we say that memcpy()
-      ** is safe for this.  Memcpy() is certainly a lot faster.
-      */
-      memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr);
-#else
-      /* In the event that some platform is found for which memcpy()
-      ** changes the destination to some intermediate value before
-      ** setting the final value, this alternative copy routine is
-      ** provided.
-      */
-      {
-        int i;
-        for(i=nHdr32; i<WALINDEX_PGSZ/sizeof(u32); i++){
-          if( aShare[i]!=aPrivate[i] ){
-            /* Atomic memory operations are not required here because if
-            ** the value needs to be changed, that means it is not being
-            ** accessed concurrently. */
-            aShare[i] = aPrivate[i];
-          }
+      /* When *-wal may follow *-wal2 */
+      if( (nCkpt2==0x0F && nCkpt1==0) || (nCkpt2<0x0F && nCkpt2==nCkpt1-1) ){
+        if( hdr.mxFrame
+         && sqlite3Get4byte((u8*)(&hdr.aSalt[0]))==pWal->hdr.aFrameCksum[0]
+         && sqlite3Get4byte((u8*)(&hdr.aSalt[1]))==pWal->hdr.aFrameCksum[1]
+        ){
+          SWAP(WalIndexHdr, pWal->hdr, hdr);
+          walidxSetMxFrame(&pWal->hdr, 1, hdr.mxFrame);
+        }else{
+          walidxSetFile(&pWal->hdr, 1);
+          walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame);
+          walidxSetMxFrame(&pWal->hdr, 0, 0);
         }
+      }else
+
+      /* Fallback */
+      if( nCkpt1<=nCkpt2 ){
+        pWal->hdr = hdr;
+      }else{
+        walidxSetFile(&pWal->hdr, 1);
+        walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame);
+        walidxSetMxFrame(&pWal->hdr, 0, 0);
       }
-#endif
-      SEH_INJECT_FAULT;
-      if( iFrame<=iLast ) break;
+      pWal->hdr.iVersion = WAL_VERSION2;
+    }else{
+      pWal->hdr.iVersion = WAL_VERSION1;
     }
 
-    SEH_FREE_ON_ERROR(aFrame, 0);
-    sqlite3_free(aFrame);
-  }
-
-finished:
-  if( rc==SQLITE_OK ){
-    volatile WalCkptInfo *pInfo;
-    int i;
-    pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
-    pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
     walIndexWriteHdr(pWal);
 
     /* Reset the checkpoint-header. This is safe because this thread is
@@ -66559,45 +67685,53 @@ static int walIndexRecover(Wal *pWal){
     ** checkpointers. Then set the values of read-mark slots 1 through N.
     */
     pInfo = walCkptInfo(pWal);
-    pInfo->nBackfill = 0;
-    pInfo->nBackfillAttempted = pWal->hdr.mxFrame;
-    pInfo->aReadMark[0] = 0;
-    for(i=1; i<WAL_NREADER; i++){
-      rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      if( rc==SQLITE_OK ){
-        if( i==1 && pWal->hdr.mxFrame ){
-          pInfo->aReadMark[i] = pWal->hdr.mxFrame;
-        }else{
-          pInfo->aReadMark[i] = READMARK_NOT_USED;
+    memset((void*)pInfo, 0, sizeof(WalCkptInfo));
+    if( 0==isWalMode2(pWal) ){
+      int i;
+      pInfo->nBackfillAttempted = pWal->hdr.mxFrame;
+      pInfo->aReadMark[0] = 0;
+      for(i=1; i<WAL_NREADER; i++){
+        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        if( rc==SQLITE_OK ){
+          if( i==1 && pWal->hdr.mxFrame ){
+            pInfo->aReadMark[i] = pWal->hdr.mxFrame;
+          }else{
+            pInfo->aReadMark[i] = READMARK_NOT_USED;
+          }
+          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        }else if( rc!=SQLITE_BUSY ){
+          break;
         }
-        SEH_INJECT_FAULT;
-        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      }else if( rc!=SQLITE_BUSY ){
-        goto recovery_error;
       }
     }
 
     /* If more than one frame was recovered from the log file, report an
     ** event via sqlite3_log(). This is to help with identifying performance
     ** problems caused by applications routinely shutting down without
-    ** checkpointing the log file.
-    */
+    ** checkpointing the log file.  */
     if( pWal->hdr.nPage ){
-      sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
-          "recovered %d frames from WAL file %s",
-          pWal->hdr.mxFrame, pWal->zWalName
-      );
+      if( isWalMode2(pWal) ){
+        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
+            "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)",
+            walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1),
+            pWal->zWalName
+        );
+      }else{
+        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
+            "recovered %d frames from WAL file %s",
+            pWal->hdr.mxFrame, pWal->zWalName
+        );
+      }
     }
   }
 
-recovery_error:
   WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
   walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
   return rc;
 }
 
 /*
-** Close an open wal-index.
+** Close an open wal-index and wal files.
 */
 static void walIndexClose(Wal *pWal, int isDelete){
   if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){
@@ -66610,6 +67744,8 @@ static void walIndexClose(Wal *pWal, int isDelete){
   if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
     sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
   }
+  sqlite3OsClose(pWal->apWalFd[0]);
+  sqlite3OsClose(pWal->apWalFd[1]);
 }
 
 /*
@@ -66633,11 +67769,13 @@ SQLITE_PRIVATE int sqlite3WalOpen(
   const char *zWalName,           /* Name of the WAL file */
   int bNoShm,                     /* True to run in heap-memory mode */
   i64 mxWalSize,                  /* Truncate WAL to this size on reset */
+  int bWal2,                      /* True to open in wal2 mode */
   Wal **ppWal                     /* OUT: Allocated Wal handle */
 ){
   int rc;                         /* Return Code */
   Wal *pRet;                      /* Object to allocate and return */
   int flags;                      /* Flags passed to OsOpen() */
+  int nByte;                      /* Bytes of space to allocate */
 
   assert( zWalName && zWalName[0] );
   assert( pDbFd );
@@ -66686,34 +67824,38 @@ SQLITE_PRIVATE int sqlite3WalOpen(
   assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET );
 #endif
 
+  nByte = sizeof(Wal) + pVfs->szOsFile*2;
 
   /* Allocate an instance of struct Wal to return. */
   *ppWal = 0;
-  pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile);
+  pRet = (Wal*)sqlite3MallocZero(nByte);
   if( !pRet ){
     return SQLITE_NOMEM_BKPT;
   }
 
   pRet->pVfs = pVfs;
-  pRet->pWalFd = (sqlite3_file *)&pRet[1];
+  pRet->apWalFd[0] = (sqlite3_file*)((char*)pRet+sizeof(Wal));
+  pRet->apWalFd[1] = (sqlite3_file*)((char*)pRet+sizeof(Wal)+pVfs->szOsFile);
   pRet->pDbFd = pDbFd;
-  pRet->readLock = -1;
+  pRet->readLock = WAL_LOCK_NONE;
   pRet->mxWalSize = mxWalSize;
   pRet->zWalName = zWalName;
   pRet->syncHeader = 1;
   pRet->padToSectorBoundary = 1;
   pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
+  sqlite3FastPrngInit(&pRet->sPrng);
+  pRet->bWal2 = bWal2;
+  pRet->zWalName2 = &zWalName[sqlite3Strlen30(zWalName)+1];
 
-  /* Open file handle on the write-ahead log file. */
+  /* Open a file handle on the first write-ahead log file. */
   flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
-  rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags);
+  rc = sqlite3OsOpen(pVfs, zWalName, pRet->apWalFd[0], flags, &flags);
   if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){
     pRet->readOnly = WAL_RDONLY;
   }
 
   if( rc!=SQLITE_OK ){
     walIndexClose(pRet, 0);
-    sqlite3OsClose(pRet->pWalFd);
     sqlite3_free(pRet);
   }else{
     int iDC = sqlite3OsDeviceCharacteristics(pDbFd);
@@ -66923,34 +68065,50 @@ static void walIteratorFree(WalIterator *p){
 
 /*
 ** Construct a WalInterator object that can be used to loop over all
-** pages in the WAL following frame nBackfill in ascending order. Frames
+** pages in wal file iWal following frame nBackfill in ascending order. Frames
 ** nBackfill or earlier may be included - excluding them is an optimization
 ** only. The caller must hold the checkpoint lock.
 **
-** On success, make *pp point to the newly allocated WalInterator object
-** return SQLITE_OK. Otherwise, return an error code. If this routine
-** returns an error, the value of *pp is undefined.
+** On success, make *pp point to the newly allocated WalIterator object
+** and return SQLITE_OK. Otherwise, return an error code. If this routine
+** returns an error, the final value of *pp is undefined.
 **
 ** The calling routine should invoke walIteratorFree() to destroy the
 ** WalIterator object when it has finished with it.
 */
-static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
+static int walIteratorInit(
+  Wal *pWal,
+  int iWal,
+  u32 nBackfill,
+  WalIterator **pp
+){
   WalIterator *p;                 /* Return value */
   int nSegment;                   /* Number of segments to merge */
   u32 iLast;                      /* Last frame in log */
   sqlite3_int64 nByte;            /* Number of bytes to allocate */
   int i;                          /* Iterator variable */
+  int iLastSeg;                   /* Last hash table to iterate though */
   ht_slot *aTmp;                  /* Temp space used by merge-sort */
   int rc = SQLITE_OK;             /* Return Code */
+  int iMode = isWalMode2(pWal) ? 2 : 1;
+
+  assert( isWalMode2(pWal) || iWal==0 );
+  assert( 0==isWalMode2(pWal) || nBackfill==0 );
 
   /* This routine only runs while holding the checkpoint lock. And
   ** it only runs if there is actually content in the log (mxFrame>0).
   */
-  assert( pWal->ckptLock && pWal->hdr.mxFrame>0 );
-  iLast = pWal->hdr.mxFrame;
+  iLast = walidxGetMxFrame(&pWal->hdr, iWal);
+  assert( pWal->ckptLock && iLast>0 );
+
+  if( iMode==2 ){
+    iLastSeg = walFramePage2(iWal, iLast);
+  }else{
+    iLastSeg = walFramePage(iLast);
+  }
+  nSegment = 1 + (iLastSeg/iMode);
 
   /* Allocate space for the WalIterator object. */
-  nSegment = walFramePage(iLast) + 1;
   nByte = sizeof(WalIterator)
         + (nSegment-1)*sizeof(struct WalSegment)
         + iLast*sizeof(ht_slot);
@@ -66964,7 +68122,8 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
   p->nSegment = nSegment;
   aTmp = (ht_slot*)&(((u8*)p)[nByte]);
   SEH_FREE_ON_ERROR(0, p);
-  for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && i<nSegment; i++){
+  i = iMode==2 ? iWal : walFramePage(nBackfill+1);
+  for(; rc==SQLITE_OK && i<=iLastSeg; i+=iMode){
     WalHashLoc sLoc;
 
     rc = walHashGet(pWal, i, &sLoc);
@@ -66972,23 +68131,32 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
       int j;                      /* Counter variable */
       int nEntry;                 /* Number of entries in this segment */
       ht_slot *aIndex;            /* Sorted index for this segment */
+      u32 iZero;
 
-      if( (i+1)==nSegment ){
-        nEntry = (int)(iLast - sLoc.iZero);
+      if( iMode==2 ){
+        walExternalDecode(sLoc.iZero+1, &iZero);
+        iZero--;
+        assert( iZero==0 || i>=2 );
+      }else{
+        iZero = sLoc.iZero;
+      }
+
+      if( i==iLastSeg ){
+        nEntry = (int)(iLast - iZero);
       }else{
         nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno);
       }
-      aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero];
-      sLoc.iZero++;
+      aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[iZero];
+      iZero++;
 
       for(j=0; j<nEntry; j++){
         aIndex[j] = (ht_slot)j;
       }
-      walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry);
-      p->aSegment[i].iZero = sLoc.iZero;
-      p->aSegment[i].nEntry = nEntry;
-      p->aSegment[i].aIndex = aIndex;
-      p->aSegment[i].aPgno = (u32 *)sLoc.aPgno;
+      walMergesort((u32*)sLoc.aPgno, aTmp, aIndex, &nEntry);
+      p->aSegment[i/iMode].iZero = iZero;
+      p->aSegment[i/iMode].nEntry = nEntry;
+      p->aSegment[i/iMode].aIndex = aIndex;
+      p->aSegment[i/iMode].aPgno = (u32*)sLoc.aPgno;
     }
   }
   if( rc!=SQLITE_OK ){
@@ -67137,6 +68305,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){
   volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
   int i;                          /* Loop counter */
   u32 *aSalt = pWal->hdr.aSalt;   /* Big-endian salt values */
+  assert( isWalMode2(pWal)==0 );
   pWal->nCkpt++;
   pWal->hdr.mxFrame = 0;
   sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0]));
@@ -67149,6 +68318,68 @@ static void walRestartHdr(Wal *pWal, u32 salt1){
   assert( pInfo->aReadMark[0]==0 );
 }
 
+/*
+** This function is used in wal2 mode.
+**
+** This function is called when writer pWal is just about to start
+** writing out frames. Parameter iApp is the current wal file. The "other" wal
+** file (wal file !iApp) has been fully checkpointed. This function returns
+** SQLITE_OK if there are no readers preventing the writer from switching to
+** the other wal file. Or SQLITE_BUSY if there are.
+*/
+static int wal2RestartOk(Wal *pWal, int iApp){
+  /* The other wal file (wal file !iApp) can be overwritten if there
+  ** are no readers reading from it - no "full" or "partial" locks.
+  ** Technically speaking it is not possible for any reader to hold
+  ** a "part" lock, as this would have prevented the file from being
+  ** checkpointed. But checking anyway doesn't hurt. The following
+  ** is equivalent to:
+  **
+  **   if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2;
+  **   if( iApp==1 ) eLock = WAL_LOCK_PART1;
+  */
+  int eLock = 1 + (iApp==0);
+
+  assert( WAL_LOCK_PART1==1 );
+  assert( WAL_LOCK_PART1_FULL2==2 );
+  assert( WAL_LOCK_PART2_FULL1==3 );
+  assert( WAL_LOCK_PART2==4 );
+
+  assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 );
+  assert( iApp!=1 || eLock==WAL_LOCK_PART1 );
+
+  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3);
+}
+static void wal2RestartFinished(Wal *pWal, int iApp){
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3);
+}
+
+/*
+** This function is used in wal2 mode.
+**
+** This function is called when a checkpointer wishes to checkpoint wal
+** file iCkpt. It takes the required lock and, if successful, returns
+** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this
+** function returns SQLITE_OK, it is the responsibility of the caller
+** to invoke wal2CheckpointFinished() to release the lock.
+*/
+static int wal2CheckpointOk(Wal *pWal, int iCkpt){
+  int eLock = 1 + (iCkpt*2);
+
+  assert( WAL_LOCK_PART1==1 );
+  assert( WAL_LOCK_PART1_FULL2==2 );
+  assert( WAL_LOCK_PART2_FULL1==3 );
+  assert( WAL_LOCK_PART2==4 );
+
+  assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 );
+  assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 );
+
+  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2);
+}
+static void wal2CheckpointFinished(Wal *pWal, int iCkpt){
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2);
+}
+
 /*
 ** Copy as much content as we can from the WAL back into the database file
 ** in response to an sqlite3_wal_checkpoint() request or the equivalent.
@@ -67198,94 +68429,121 @@ static int walCheckpoint(
   u32 mxPage;                     /* Max database page to write */
   int i;                          /* Loop counter */
   volatile WalCkptInfo *pInfo;    /* The checkpoint status information */
+  int bWal2 = isWalMode2(pWal);   /* True for wal2 connections */
+  int iCkpt = bWal2 ? !walidxGetFile(&pWal->hdr) : 0;
 
+  mxSafeFrame = walidxGetMxFrame(&pWal->hdr, iCkpt);
   szPage = walPagesize(pWal);
   testcase( szPage<=32768 );
   testcase( szPage>=65536 );
   pInfo = walCkptInfo(pWal);
-  if( pInfo->nBackfill<pWal->hdr.mxFrame ){
+  if( (bWal2==1 && pInfo->nBackfill==0 && mxSafeFrame)
+   || (bWal2==0 && pInfo->nBackfill<mxSafeFrame)
+  ){
+    sqlite3_file *pWalFd = pWal->apWalFd[iCkpt];
+    mxPage = pWal->hdr.nPage;
+
+    /* If this is a wal2 system, check for a reader holding a lock
+    ** preventing this checkpoint operation. If one is found, return
+    ** early.  */
+    if( bWal2 ){
+      rc = wal2CheckpointOk(pWal, iCkpt);
+      if( rc!=SQLITE_OK ) return rc;
+    }
 
     /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
     ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
     assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
 
-    /* Compute in mxSafeFrame the index of the last frame of the WAL that is
-    ** safe to write into the database.  Frames beyond mxSafeFrame might
-    ** overwrite database pages that are in use by active readers and thus
-    ** cannot be backfilled from the WAL.
+    /* If this is a wal system (not wal2), compute in mxSafeFrame the index
+    ** of the last frame of the WAL that is safe to write into the database.
+    ** Frames beyond mxSafeFrame might overwrite database pages that are in
+    ** use by active readers and thus cannot be backfilled from the WAL.
     */
-    mxSafeFrame = pWal->hdr.mxFrame;
-    mxPage = pWal->hdr.nPage;
-    for(i=1; i<WAL_NREADER; i++){
-      u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
-      if( mxSafeFrame>y ){
-        assert( y<=pWal->hdr.mxFrame );
-        rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
-        if( rc==SQLITE_OK ){
-          u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
-          AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT;
-          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-        }else if( rc==SQLITE_BUSY ){
-          mxSafeFrame = y;
-          xBusy = 0;
-        }else{
-          goto walcheckpoint_out;
+    if( bWal2==0 ){
+      mxSafeFrame = pWal->hdr.mxFrame;
+      mxPage = pWal->hdr.nPage;
+      for(i=1; i<WAL_NREADER; i++){
+        u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
+        if( mxSafeFrame>y ){
+          assert( y<=pWal->hdr.mxFrame );
+          rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
+          if( rc==SQLITE_OK ){
+            u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
+            AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT;
+            walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+          }else if( rc==SQLITE_BUSY ){
+            mxSafeFrame = y;
+            xBusy = 0;
+          }else{
+            goto walcheckpoint_out;
+          }
         }
       }
     }
 
     /* Allocate the iterator */
-    if( pInfo->nBackfill<mxSafeFrame ){
-      rc = walIteratorInit(pWal, pInfo->nBackfill, &pIter);
+    if( bWal2 || pInfo->nBackfill<mxSafeFrame ){
+      assert( bWal2==0 || pInfo->nBackfill==0 );
+      rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter);
       assert( rc==SQLITE_OK || pIter==0 );
     }
 
-    if( pIter
-     && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
-    ){
+    if( pIter && (bWal2
+     || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
+    )){
       u32 nBackfill = pInfo->nBackfill;
+
+      assert( bWal2==0 || nBackfill==0 );
       pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
 
-      /* Sync the WAL to disk */
-      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
+      /* Sync the wal file being checkpointed to disk */
+      rc = sqlite3OsSync(pWalFd, CKPT_SYNC_FLAGS(sync_flags));
 
       /* If the database may grow as a result of this checkpoint, hint
-      ** about the eventual size of the db file to the VFS layer.
-      */
+      ** about the eventual size of the db file to the VFS layer.  */
       if( rc==SQLITE_OK ){
         i64 nReq = ((i64)mxPage * szPage);
         i64 nSize;                    /* Current size of database file */
         sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0);
         rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
         if( rc==SQLITE_OK && nSize<nReq ){
-          if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){
+          i64 mx = pWal->hdr.mxFrame + (bWal2?walidxGetMxFrame(&pWal->hdr,1):0);
+          if( (nSize+65536+mx*szPage)<nReq ){
             /* If the size of the final database is larger than the current
             ** database plus the amount of data in the wal file, plus the
             ** maximum size of the pending-byte page (65536 bytes), then
-            ** must be corruption somewhere.  */
+            ** must be corruption somewhere.  Or in the case of wal2 mode,
+            ** plus the amount of data in both wal files. */
             rc = SQLITE_CORRUPT_BKPT;
           }else{
             sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq);
           }
         }
-
       }
 
       /* Iterate through the contents of the WAL, copying data to the db file */
       while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
         i64 iOffset;
-        assert( walFramePgno(pWal, iFrame)==iDbpage );
+
+        assert( bWal2==1 || walFramePgno(pWal, iFrame)==iDbpage );
+        assert( bWal2==0 || walFramePgno2(pWal, iCkpt, iFrame)==iDbpage );
+
         SEH_INJECT_FAULT;
         if( AtomicLoad(&db->u1.isInterrupted) ){
           rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
           break;
         }
         if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
+          assert( bWal2==0 || iDbpage>mxPage );
           continue;
         }
         iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
+        WALTRACE(("WAL%p: checkpoint frame %d of wal %d to db page %d\n",
+              pWal, (int)iFrame, iCkpt, (int)iDbpage
+        ));
         /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
-        rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
+        rc = sqlite3OsRead(pWalFd, zBuf, szPage, iOffset);
         if( rc!=SQLITE_OK ) break;
         iOffset = (iDbpage-1)*(i64)szPage;
         testcase( IS_BIG_INT(iOffset) );
@@ -67294,23 +68552,27 @@ static int walCheckpoint(
       }
       sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0);
 
-      /* If work was actually accomplished... */
-      if( rc==SQLITE_OK ){
-        if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
+      /* If work was actually accomplished, truncate the db file, sync the wal
+      ** file and set WalCkptInfo.nBackfill to indicate so. */
+      if( rc==SQLITE_OK && (bWal2 || mxSafeFrame==walIndexHdr(pWal)->mxFrame) ){
+        if( !bWal2 ){
           i64 szDb = pWal->hdr.nPage*(i64)szPage;
           testcase( IS_BIG_INT(szDb) );
           rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
-          if( rc==SQLITE_OK ){
-            rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
-          }
         }
         if( rc==SQLITE_OK ){
-          AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
+          rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
         }
       }
+      if( rc==SQLITE_OK ){
+        AtomicStore(&pInfo->nBackfill, (bWal2 ? 1 : mxSafeFrame));
+        SEH_INJECT_FAULT;
+      }
 
       /* Release the reader lock held while backfilling */
-      walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
+      if( bWal2==0 ){
+        walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
+      }
     }
 
     if( rc==SQLITE_BUSY ){
@@ -67318,6 +68580,7 @@ static int walCheckpoint(
       ** just because there are active readers.  */
       rc = SQLITE_OK;
     }
+    if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt);
   }
 
   /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
@@ -67325,14 +68588,14 @@ static int walCheckpoint(
   ** until all readers have finished using the wal file. This ensures that
   ** the next process to write to the database restarts the wal file.
   */
-  if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
+  if( bWal2==0 && rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
     assert( pWal->writeLock );
     SEH_INJECT_FAULT;
     if( pInfo->nBackfill<pWal->hdr.mxFrame ){
       rc = SQLITE_BUSY;
     }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       assert( pInfo->nBackfill==pWal->hdr.mxFrame );
       rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
@@ -67351,7 +68614,7 @@ static int walCheckpoint(
           ** file-system. To avoid this, update the wal-index header to
           ** indicate that the log file contains zero valid frames.  */
           walRestartHdr(pWal, salt1);
-          rc = sqlite3OsTruncate(pWal->pWalFd, 0);
+          rc = sqlite3OsTruncate(pWal->apWalFd[0], 0);
         }
         walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
       }
@@ -67369,16 +68632,18 @@ static int walCheckpoint(
 ** it to exactly nMax bytes. If an error occurs while doing so, ignore it.
 */
 static void walLimitSize(Wal *pWal, i64 nMax){
-  i64 sz;
-  int rx;
-  sqlite3BeginBenignMalloc();
-  rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
-  if( rx==SQLITE_OK && (sz > nMax ) ){
-    rx = sqlite3OsTruncate(pWal->pWalFd, nMax);
-  }
-  sqlite3EndBenignMalloc();
-  if( rx ){
-    sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
+  if( isWalMode2(pWal)==0 ){
+    i64 sz;
+    int rx;
+    sqlite3BeginBenignMalloc();
+    rx = sqlite3OsFileSize(pWal->apWalFd[0], &sz);
+    if( rx==SQLITE_OK && (sz > nMax ) ){
+      rx = sqlite3OsTruncate(pWal->apWalFd[0], nMax);
+    }
+    sqlite3EndBenignMalloc();
+    if( rx ){
+      sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
+    }
   }
 }
 
@@ -67484,6 +68749,7 @@ SQLITE_PRIVATE int sqlite3WalClose(
     int isDelete = 0;             /* True to unlink wal and wal-index files */
 
     assert( walAssertLockmask(pWal) );
+    pWal->bClosing = 1;
 
     /* If an EXCLUSIVE lock can be obtained on the database file (using the
     ** ordinary, rollback-mode locking methods, this guarantees that the
@@ -67496,39 +68762,53 @@ SQLITE_PRIVATE int sqlite3WalClose(
     if( zBuf!=0
      && SQLITE_OK==(rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE))
     ){
+      int i;
       if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = sqlite3WalCheckpoint(pWal, db,
-          SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
-      );
-      if( rc==SQLITE_OK ){
-        int bPersist = -1;
-        sqlite3OsFileControlHint(
-            pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
+      for(i=0; rc==SQLITE_OK && i<2; i++){
+        rc = sqlite3WalCheckpoint(pWal, db,
+            SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
         );
-        if( bPersist!=1 ){
-          /* Try to delete the WAL file if the checkpoint completed and
-          ** fsynced (rc==SQLITE_OK) and if we are not in persistent-wal
-          ** mode (!bPersist) */
-          isDelete = 1;
-        }else if( pWal->mxWalSize>=0 ){
-          /* Try to truncate the WAL file to zero bytes if the checkpoint
-          ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
-          ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
-          ** non-negative value (pWal->mxWalSize>=0).  Note that we truncate
-          ** to zero bytes as truncating to the journal_size_limit might
-          ** leave a corrupt WAL file on disk. */
-          walLimitSize(pWal, 0);
+        if( rc==SQLITE_OK ){
+          int bPersist = -1;
+          sqlite3OsFileControlHint(
+              pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
+          );
+          if( bPersist!=1 ){
+            /* Try to delete the WAL file if the checkpoint completed and
+            ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
+            ** mode (!bPersist) */
+            isDelete = 1;
+          }else if( pWal->mxWalSize>=0 ){
+            /* Try to truncate the WAL file to zero bytes if the checkpoint
+            ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
+            ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
+            ** non-negative value (pWal->mxWalSize>=0).  Note that we truncate
+            ** to zero bytes as truncating to the journal_size_limit might
+            ** leave a corrupt WAL file on disk. */
+            walLimitSize(pWal, 0);
+          }
+        }
+
+        if( isWalMode2(pWal)==0 ) break;
+
+        SEH_TRY {
+          walCkptInfo(pWal)->nBackfill = 0;
+          walidxSetFile(&pWal->hdr, !walidxGetFile(&pWal->hdr));
+          pWal->writeLock = 1;
+          walIndexWriteHdr(pWal);
+          pWal->writeLock = 0;
         }
+        SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
       }
     }
 
     walIndexClose(pWal, isDelete);
-    sqlite3OsClose(pWal->pWalFd);
     if( isDelete ){
       sqlite3BeginBenignMalloc();
       sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0);
+      sqlite3OsDelete(pWal->pVfs, pWal->zWalName2, 0);
       sqlite3EndBenignMalloc();
     }
     WALTRACE(("WAL%p: closed\n", pWal));
@@ -67539,25 +68819,14 @@ SQLITE_PRIVATE int sqlite3WalClose(
 }
 
 /*
-** Try to read the wal-index header.  Return 0 on success and 1 if
-** there is a problem.
-**
-** The wal-index is in shared memory.  Another thread or process might
-** be writing the header at the same time this procedure is trying to
-** read it, which might result in inconsistency.  A dirty read is detected
-** by verifying that both copies of the header are the same and also by
-** a checksum on the header.
-**
-** If and only if the read is consistent and the header is different from
-** pWal->hdr, then pWal->hdr is updated to the content of the new header
-** and *pChanged is set to 1.
-**
-** If the checksum cannot be verified return non-zero. If the header
-** is read successfully and the checksum verified, return zero.
+** Try to copy the wal-index header from shared-memory into (*pHdr). Return
+** zero if successful or non-zero otherwise. If the header is corrupted
+** (either because the two copies are inconsistent or because the checksum
+** values are incorrect), the read fails and non-zero is returned.
 */
-static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
+static int walIndexLoadHdr(Wal *pWal, WalIndexHdr *pHdr){
   u32 aCksum[2];                  /* Checksum on the header content */
-  WalIndexHdr h1, h2;             /* Two copies of the header content */
+  WalIndexHdr h2;                 /* Second copy of the header content */
   WalIndexHdr volatile *aHdr;     /* Header in shared memory */
 
   /* The first page of the wal-index must be mapped at this point. */
@@ -67568,33 +68837,54 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
   ** meaning it is possible that an inconsistent snapshot is read
   ** from the file. If this happens, return non-zero.
   **
-  ** tag-20200519-1:
   ** There are two copies of the header at the beginning of the wal-index.
   ** When reading, read [0] first then [1].  Writes are in the reverse order.
   ** Memory barriers are used to prevent the compiler or the hardware from
-  ** reordering the reads and writes.  TSAN and similar tools can sometimes
-  ** give false-positive warnings about these accesses because the tools do not
-  ** account for the double-read and the memory barrier. The use of mutexes
-  ** here would be problematic as the memory being accessed is potentially
-  ** shared among multiple processes and not all mutex implementations work
-  ** reliably in that environment.
+  ** reordering the reads and writes.
   */
   aHdr = walIndexHdr(pWal);
-  memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */
+  memcpy(pHdr, (void *)&aHdr[0], sizeof(h2));
   walShmBarrier(pWal);
   memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
 
-  if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
+  if( memcmp(&h2, pHdr, sizeof(h2))!=0 ){
     return 1;   /* Dirty read */
   }
-  if( h1.isInit==0 ){
+  if( h2.isInit==0 ){
     return 1;   /* Malformed header - probably all zeros */
   }
-  walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
-  if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
+  walChecksumBytes(1, (u8*)&h2, sizeof(h2)-sizeof(h2.aCksum), 0, aCksum);
+  if( aCksum[0]!=h2.aCksum[0] || aCksum[1]!=h2.aCksum[1] ){
     return 1;   /* Checksum does not match */
   }
 
+  return 0;
+}
+
+/*
+** Try to read the wal-index header.  Return 0 on success and 1 if
+** there is a problem.
+**
+** The wal-index is in shared memory.  Another thread or process might
+** be writing the header at the same time this procedure is trying to
+** read it, which might result in inconsistency.  A dirty read is detected
+** by verifying that both copies of the header are the same and also by
+** a checksum on the header.
+**
+** If and only if the read is consistent and the header is different from
+** pWal->hdr, then pWal->hdr is updated to the content of the new header
+** and *pChanged is set to 1.
+**
+** If the checksum cannot be verified return non-zero. If the header
+** is read successfully and the checksum verified, return zero.
+*/
+static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
+  WalIndexHdr h1;                 /* Copy of the header content */
+
+  if( walIndexLoadHdr(pWal, &h1) ){
+    return 1;
+  }
+
   if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){
     *pChanged = 1;
     memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr));
@@ -67706,7 +68996,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   ** sure the wal-index was not constructed with some future format that
   ** this version of SQLite cannot understand.
   */
-  if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){
+  if( badHdr==0
+   && pWal->hdr.iVersion!=WAL_VERSION1 && pWal->hdr.iVersion!=WAL_VERSION2
+  ){
     rc = SQLITE_CANTOPEN_BKPT;
   }
   if( pWal->bShmUnreliable ){
@@ -67798,7 +69090,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   ** even if some external agent does a "chmod" to make the shared-memory
   ** writable by us, until sqlite3OsShmUnmap() has been called.
   ** This is a requirement on the VFS implementation.
-   */
+  */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
   assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
   if( rc!=SQLITE_READONLY_CANTINIT ){
@@ -67815,7 +69107,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   /* Make sure some writer hasn't come in and changed the WAL file out
   ** from under us, then disconnected, while we were not looking.
   */
-  rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
+  rc = sqlite3OsFileSize(pWal->apWalFd[0], &szWal);
   if( rc!=SQLITE_OK ){
     goto begin_unreliable_shm_out;
   }
@@ -67832,7 +69124,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   }
 
   /* Check the salt keys at the start of the wal file still match. */
-  rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
+  rc = sqlite3OsRead(pWal->apWalFd[0], aBuf, WAL_HDRSIZE, 0);
   if( rc!=SQLITE_OK ){
     goto begin_unreliable_shm_out;
   }
@@ -67869,7 +69161,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
     u32 nTruncate;              /* dbsize field from frame header */
 
     /* Read and decode the next log frame. */
-    rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset);
+    rc = sqlite3OsRead(pWal->apWalFd[0], aFrame, szFrame, iOffset);
     if( rc!=SQLITE_OK ) break;
     if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break;
 
@@ -67982,16 +69274,12 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
 */
 static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){
   volatile WalCkptInfo *pInfo;    /* Checkpoint information in wal-index */
-  u32 mxReadMark;                 /* Largest aReadMark[] value */
-  int mxI;                        /* Index of largest aReadMark[] value */
-  int i;                          /* Loop counter */
   int rc = SQLITE_OK;             /* Return code  */
-  u32 mxFrame;                    /* Wal frame to lock to */
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
   int nBlockTmout = 0;
 #endif
 
-  assert( pWal->readLock<0 );     /* Not currently locked */
+  assert( pWal->readLock==WAL_LOCK_NONE );     /* Not currently locked */
 
   /* useWal may only be set for read/write connections */
   assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 );
@@ -68092,141 +69380,175 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){
   assert( pWal->apWiData[0]!=0 );
   pInfo = walCkptInfo(pWal);
   SEH_INJECT_FAULT;
-  if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame
-#ifdef SQLITE_ENABLE_SNAPSHOT
-   && ((pWal->bGetSnapshot==0 && pWal->pSnapshot==0) || pWal->hdr.mxFrame==0)
-#endif
-  ){
-    /* The WAL has been completely backfilled (or it is empty).
-    ** and can be safely ignored.
-    */
-    rc = walLockShared(pWal, WAL_READ_LOCK(0));
+  if( isWalMode2(pWal) ){
+    /* This connection needs a "part" lock on the current wal file and,
+    ** unless pInfo->nBackfill is set to indicate that it has already been
+    ** checkpointed, a "full" lock on the other wal file.  */
+    int iWal = walidxGetFile(&pWal->hdr);
+    int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0;
+    int eLock = 1 + (iWal*2) + (nBackfill==iWal);
+
+    assert( nBackfill==0 || nBackfill==1 );
+    assert( iWal==0 || iWal==1 );
+    assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 );
+    assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 );
+    assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 );
+    assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 );
+
+    rc = walLockShared(pWal, WAL_READ_LOCK(eLock));
+    if( rc!=SQLITE_OK ){
+      return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
+    }
     walShmBarrier(pWal);
-    if( rc==SQLITE_OK ){
-      if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
-        /* It is not safe to allow the reader to continue here if frames
-        ** may have been appended to the log before READ_LOCK(0) was obtained.
-        ** When holding READ_LOCK(0), the reader ignores the entire log file,
-        ** which implies that the database file contains a trustworthy
-        ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from
-        ** happening, this is usually correct.
-        **
-        ** However, if frames have been appended to the log (or if the log
-        ** is wrapped and written for that matter) before the READ_LOCK(0)
-        ** is obtained, that is not necessarily true. A checkpointer may
-        ** have started to backfill the appended frames but crashed before
-        ** it finished. Leaving a corrupt image in the database file.
-        */
-        walUnlockShared(pWal, WAL_READ_LOCK(0));
-        return WAL_RETRY;
+    if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
+      walUnlockShared(pWal, WAL_READ_LOCK(eLock));
+      return WAL_RETRY;
+    }else{
+      pWal->readLock = eLock;
+    }
+    assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 );
+  }else{
+    u32 mxReadMark;               /* Largest aReadMark[] value */
+    int mxI;                      /* Index of largest aReadMark[] value */
+    int i;                        /* Loop counter */
+    u32 mxFrame;                  /* Wal frame to lock to */
+    if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame
+  #ifdef SQLITE_ENABLE_SNAPSHOT
+     && ((pWal->bGetSnapshot==0 && pWal->pSnapshot==0) || pWal->hdr.mxFrame==0)
+  #endif
+    ){
+      /* The WAL has been completely backfilled (or it is empty).
+      ** and can be safely ignored.
+      */
+      rc = walLockShared(pWal, WAL_READ_LOCK(0));
+      walShmBarrier(pWal);
+      if( rc==SQLITE_OK ){
+        if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr,sizeof(WalIndexHdr)) ){
+          /* It is not safe to allow the reader to continue here if frames
+          ** may have been appended to the log before READ_LOCK(0) was obtained.
+          ** When holding READ_LOCK(0), the reader ignores the entire log file,
+          ** which implies that the database file contains a trustworthy
+          ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from
+          ** happening, this is usually correct.
+          **
+          ** However, if frames have been appended to the log (or if the log
+          ** is wrapped and written for that matter) before the READ_LOCK(0)
+          ** is obtained, that is not necessarily true. A checkpointer may
+          ** have started to backfill the appended frames but crashed before
+          ** it finished. Leaving a corrupt image in the database file.
+          */
+          walUnlockShared(pWal, WAL_READ_LOCK(0));
+          return WAL_RETRY;
+        }
+        pWal->readLock = 0;
+        return SQLITE_OK;
+      }else if( rc!=SQLITE_BUSY ){
+        return rc;
       }
-      pWal->readLock = 0;
-      return SQLITE_OK;
-    }else if( rc!=SQLITE_BUSY ){
-      return rc;
     }
-  }
 
-  /* If we get this far, it means that the reader will want to use
-  ** the WAL to get at content from recent commits.  The job now is
-  ** to select one of the aReadMark[] entries that is closest to
-  ** but not exceeding pWal->hdr.mxFrame and lock that entry.
-  */
-  mxReadMark = 0;
-  mxI = 0;
-  mxFrame = pWal->hdr.mxFrame;
-#ifdef SQLITE_ENABLE_SNAPSHOT
-  if( pWal->pSnapshot && pWal->pSnapshot->mxFrame<mxFrame ){
-    mxFrame = pWal->pSnapshot->mxFrame;
-  }
-#endif
-  for(i=1; i<WAL_NREADER; i++){
-    u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
-    if( mxReadMark<=thisMark && thisMark<=mxFrame ){
-      assert( thisMark!=READMARK_NOT_USED );
-      mxReadMark = thisMark;
-      mxI = i;
+    /* If we get this far, it means that the reader will want to use
+    ** the WAL to get at content from recent commits.  The job now is
+    ** to select one of the aReadMark[] entries that is closest to
+    ** but not exceeding pWal->hdr.mxFrame and lock that entry.
+    */
+    mxReadMark = 0;
+    mxI = 0;
+    mxFrame = pWal->hdr.mxFrame;
+  #ifdef SQLITE_ENABLE_SNAPSHOT
+    if( pWal->pSnapshot && pWal->pSnapshot->mxFrame<mxFrame ){
+      mxFrame = pWal->pSnapshot->mxFrame;
     }
-  }
-  if( (pWal->readOnly & WAL_SHM_RDONLY)==0
-   && (mxReadMark<mxFrame || mxI==0)
-  ){
+  #endif
     for(i=1; i<WAL_NREADER; i++){
-      rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      if( rc==SQLITE_OK ){
-        AtomicStore(pInfo->aReadMark+i,mxFrame);
-        mxReadMark = mxFrame;
+      u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
+      if( mxReadMark<=thisMark && thisMark<=mxFrame ){
+        assert( thisMark!=READMARK_NOT_USED );
+        mxReadMark = thisMark;
         mxI = i;
-        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-        break;
-      }else if( rc!=SQLITE_BUSY ){
-        return rc;
       }
     }
-  }
-  if( mxI==0 ){
-    assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
-    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
-  }
+    if( (pWal->readOnly & WAL_SHM_RDONLY)==0
+     && (mxReadMark<mxFrame || mxI==0)
+    ){
+      for(i=1; i<WAL_NREADER; i++){
+        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        if( rc==SQLITE_OK ){
+          AtomicStore(pInfo->aReadMark+i,mxFrame);
+          mxReadMark = mxFrame;
+          mxI = i;
+          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+          break;
+        }else if( rc!=SQLITE_BUSY ){
+          return rc;
+        }
+      }
+    }
+    if( mxI==0 ){
+      assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
+      return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
+    }
 
-  (void)walEnableBlockingMs(pWal, nBlockTmout);
-  rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
-  walDisableBlocking(pWal);
-  if( rc ){
+    (void)walEnableBlockingMs(pWal, nBlockTmout);
+    rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
+    walDisableBlocking(pWal);
+    if( rc ){
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
-    if( rc==SQLITE_BUSY_TIMEOUT ){
-      *pCnt |= WAL_RETRY_BLOCKED_MASK;
-    }
+      if( rc==SQLITE_BUSY_TIMEOUT ){
+        *pCnt |= WAL_RETRY_BLOCKED_MASK;
+      }
 #else
-    assert( rc!=SQLITE_BUSY_TIMEOUT );
+      assert( rc!=SQLITE_BUSY_TIMEOUT );
 #endif
-    assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT );
-    return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc;
-  }
-  /* Now that the read-lock has been obtained, check that neither the
-  ** value in the aReadMark[] array or the contents of the wal-index
-  ** header have changed.
-  **
-  ** It is necessary to check that the wal-index header did not change
-  ** between the time it was read and when the shared-lock was obtained
-  ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility
-  ** that the log file may have been wrapped by a writer, or that frames
-  ** that occur later in the log than pWal->hdr.mxFrame may have been
-  ** copied into the database by a checkpointer. If either of these things
-  ** happened, then reading the database with the current value of
-  ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry
-  ** instead.
-  **
-  ** Before checking that the live wal-index header has not changed
-  ** since it was read, set Wal.minFrame to the first frame in the wal
-  ** file that has not yet been checkpointed. This client will not need
-  ** to read any frames earlier than minFrame from the wal file - they
-  ** can be safely read directly from the database file.
-  **
-  ** Because a ShmBarrier() call is made between taking the copy of
-  ** nBackfill and checking that the wal-header in shared-memory still
-  ** matches the one cached in pWal->hdr, it is guaranteed that the
-  ** checkpointer that set nBackfill was not working with a wal-index
-  ** header newer than that cached in pWal->hdr. If it were, that could
-  ** cause a problem. The checkpointer could omit to checkpoint
-  ** a version of page X that lies before pWal->minFrame (call that version
-  ** A) on the basis that there is a newer version (version B) of the same
-  ** page later in the wal file. But if version B happens to like past
-  ** frame pWal->hdr.mxFrame - then the client would incorrectly assume
-  ** that it can read version A from the database file. However, since
-  ** we can guarantee that the checkpointer that set nBackfill could not
-  ** see any pages past pWal->hdr.mxFrame, this problem does not come up.
-  */
-  pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT;
-  walShmBarrier(pWal);
-  if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark
-   || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
-  ){
-    walUnlockShared(pWal, WAL_READ_LOCK(mxI));
-    return WAL_RETRY;
-  }else{
-    assert( mxReadMark<=pWal->hdr.mxFrame );
-    pWal->readLock = (i16)mxI;
+      assert((rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT);
+      return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc;
+
+    }
+    /* Now that the read-lock has been obtained, check that neither the
+    ** value in the aReadMark[] array or the contents of the wal-index
+    ** header have changed.
+    **
+    ** It is necessary to check that the wal-index header did not change
+    ** between the time it was read and when the shared-lock was obtained
+    ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility
+    ** that the log file may have been wrapped by a writer, or that frames
+    ** that occur later in the log than pWal->hdr.mxFrame may have been
+    ** copied into the database by a checkpointer. If either of these things
+    ** happened, then reading the database with the current value of
+    ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry
+    ** instead.
+    **
+    ** Before checking that the live wal-index header has not changed
+    ** since it was read, set Wal.minFrame to the first frame in the wal
+    ** file that has not yet been checkpointed. This client will not need
+    ** to read any frames earlier than minFrame from the wal file - they
+    ** can be safely read directly from the database file.
+    **
+    ** Because a ShmBarrier() call is made between taking the copy of
+    ** nBackfill and checking that the wal-header in shared-memory still
+    ** matches the one cached in pWal->hdr, it is guaranteed that the
+    ** checkpointer that set nBackfill was not working with a wal-index
+    ** header newer than that cached in pWal->hdr. If it were, that could
+    ** cause a problem. The checkpointer could omit to checkpoint
+    ** a version of page X that lies before pWal->minFrame (call that version
+    ** A) on the basis that there is a newer version (version B) of the same
+    ** page later in the wal file. But if version B happens to like past
+    ** frame pWal->hdr.mxFrame - then the client would incorrectly assume
+    ** that it can read version A from the database file. However, since
+    ** we can guarantee that the checkpointer that set nBackfill could not
+    ** see any pages past pWal->hdr.mxFrame, this problem does not come up.
+    */
+    pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT;
+    walShmBarrier(pWal);
+    if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark
+     || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
+    ){
+      walUnlockShared(pWal, WAL_READ_LOCK(mxI));
+      return WAL_RETRY;
+    }else{
+      assert( mxReadMark<=pWal->hdr.mxFrame );
+      pWal->readLock = (i16)mxI;
+    }
   }
   return rc;
 }
@@ -68262,7 +69584,7 @@ static int walSnapshotRecover(
 
       if( iDbOff+szPage<=szDb ){
         iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
-        rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);
+        rc = sqlite3OsRead(pWal->apWalFd[0], pBuf1, szPage, iWalOff);
 
         if( rc==SQLITE_OK ){
           rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
@@ -68302,6 +69624,9 @@ static int walSnapshotRecover(
 SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){
   int rc;
 
+  /* Snapshots may not be used with wal2 mode databases. */
+  if( isWalMode2(pWal) ) return SQLITE_ERROR;
+
   assert( pWal->readLock>=0 );
   rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
   if( rc==SQLITE_OK ){
@@ -68327,6 +69652,15 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+/*
+** Return the current last frame in the wal-index - mxFrame for *-wal,
+** or mxFrame2 for *-wal2. If the last frame is current in wal2, return
+** mxFrame2 without clearing the 0x80000000 bit.
+*/
+static u32 walGetPriorFrame(WalIndexHdr *pHdr){
+  return (walidxGetFile(pHdr) ? pHdr->mxFrame2 : pHdr->mxFrame);
+}
+
 /*
 ** This function does the work of sqlite3WalBeginReadTransaction() (see
 ** below). That function simply calls this one inside an SEH_TRY{...} block.
@@ -68345,6 +69679,7 @@ static int walBeginReadTransaction(Wal *pWal, int *pChanged){
 
 #ifdef SQLITE_ENABLE_SNAPSHOT
   if( pSnapshot ){
+    if( isWalMode2(pWal) ) return SQLITE_ERROR;
     if( memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){
       bChanged = 1;
     }
@@ -68376,6 +69711,11 @@ static int walBeginReadTransaction(Wal *pWal, int *pChanged){
   testcase( rc==SQLITE_PROTOCOL );
   testcase( rc==SQLITE_OK );
 
+  if( rc==SQLITE_OK && pWal->hdr.iVersion==WAL_VERSION2 ){
+    rc = walOpenWal2(pWal);
+  }
+
+  pWal->nPriorFrame = walGetPriorFrame(&pWal->hdr);
 #ifdef SQLITE_ENABLE_SNAPSHOT
   if( rc==SQLITE_OK ){
     if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){
@@ -68459,16 +69799,103 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
   return rc;
 }
 
+
 /*
 ** Finish with a read transaction.  All this does is release the
 ** read-lock.
 */
 SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){
   sqlite3WalEndWriteTransaction(pWal);
-  if( pWal->readLock>=0 ){
+  if( pWal->readLock!=WAL_LOCK_NONE ){
     walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-    pWal->readLock = -1;
+    pWal->readLock = WAL_LOCK_NONE;
+  }
+}
+
+/* Search hash table iHash for an entry matching page number
+** pgno. Each call to this function searches a single hash table
+** (each hash table indexes up to HASHTABLE_NPAGE frames).
+**
+** This code might run concurrently to the code in walIndexAppend()
+** that adds entries to the wal-index (and possibly to this hash
+** table). This means the value just read from the hash
+** slot (aHash[iKey]) may have been added before or after the
+** current read transaction was opened. Values added after the
+** read transaction was opened may have been written incorrectly -
+** i.e. these slots may contain garbage data. However, we assume
+** that any slots written before the current read transaction was
+** opened remain unmodified.
+**
+** For the reasons above, the if(...) condition featured in the inner
+** loop of the following block is more stringent that would be required
+** if we had exclusive access to the hash-table:
+**
+**   (aPgno[iFrame]==pgno):
+**     This condition filters out normal hash-table collisions.
+**
+**   (iFrame<=iLast):
+**     This condition filters out entries that were added to the hash
+**     table after the current read-transaction had started.
+*/
+static int walSearchHash(
+  Wal *pWal,
+  u32 iLast,
+  int iHash,
+  Pgno pgno,
+  u32 *piRead
+){
+  WalHashLoc sLoc;                /* Hash table location */
+  int iKey;                       /* Hash slot index */
+  int nCollide;                   /* Number of hash collisions remaining */
+  int rc;                         /* Error code */
+  u32 iH;
+
+  rc = walHashGet(pWal, iHash, &sLoc);
+  if( rc!=SQLITE_OK ){
+    return rc;
   }
+  nCollide = HASHTABLE_NSLOT;
+  iKey = walHash(pgno);
+  SEH_INJECT_FAULT;
+  while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){
+    u32 iFrame = iH + sLoc.iZero;
+    if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){
+      assert( iFrame>*piRead || CORRUPT_DB );
+      *piRead = iFrame;
+    }
+    if( (nCollide--)==0 ){
+      *piRead = 0;
+      return SQLITE_CORRUPT_BKPT;
+    }
+    iKey = walNextHash(iKey);
+  }
+
+  return SQLITE_OK;
+}
+
+static int walSearchWal(
+  Wal *pWal,
+  int iWal,
+  Pgno pgno,
+  u32 *piRead
+){
+  int rc = SQLITE_OK;
+  int bWal2 = isWalMode2(pWal);
+  u32 iLast = walidxGetMxFrame(&pWal->hdr, iWal);
+  if( iLast ){
+    int iHash;
+    int iMinHash = walFramePage(pWal->minFrame);
+    u32 iExternal = bWal2 ? walExternalEncode(iWal, iLast) : iLast;
+    assert( bWal2==0 || pWal->minFrame==0 );
+    for(iHash=walFramePage(iExternal);
+        iHash>=iMinHash && *piRead==0;
+        iHash-=(1+bWal2)
+    ){
+      rc = walSearchHash(pWal, iExternal, iHash, pgno, piRead);
+      if( rc!=SQLITE_OK ) break;
+    }
+  }
+  return rc;
 }
 
 /*
@@ -68484,85 +69911,77 @@ static int walFindFrame(
   Pgno pgno,                      /* Database page number to read data for */
   u32 *piRead                     /* OUT: Frame number (or zero) */
 ){
+  int bWal2 = isWalMode2(pWal);
+  int iApp = walidxGetFile(&pWal->hdr);
+  int rc = SQLITE_OK;
   u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
-  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
-  int iHash;                      /* Used to loop through N hash tables */
-  int iMinHash;
 
-  /* This routine is only be called from within a read transaction. */
-  assert( pWal->readLock>=0 || pWal->lockError );
+  /* This routine is only be called from within a read transaction. Or,
+  ** sometimes, as part of a rollback that occurs after an error reaquiring
+  ** a read-lock in walRestartLog().  */
+  assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError );
+
+  /* If this is a regular wal system, then iApp must be set to 0 (there is
+  ** only one wal file, after all). Or, if this is a wal2 system and the
+  ** write-lock is not held, the client must have a partial-wal lock on wal
+  ** file iApp. This is not always true if the write-lock is held and this
+  ** function is being called after WalLockForCommit() as part of committing
+  ** a CONCURRENT transaction.  */
+#ifdef SQLITE_DEBUG
+  if( bWal2 ){
+    if( pWal->writeLock==0 ){
+      int l = pWal->readLock;
+      assert( iApp==1 || l==WAL_LOCK_PART1 || l==WAL_LOCK_PART1_FULL2 );
+      assert( iApp==0 || l==WAL_LOCK_PART2 || l==WAL_LOCK_PART2_FULL1 );
+    }
+  }else{
+    assert( iApp==0 );
+  }
+#endif
 
-  /* If the "last page" field of the wal-index header snapshot is 0, then
-  ** no data will be read from the wal under any circumstances. Return early
-  ** in this case as an optimization.  Likewise, if pWal->readLock==0,
-  ** then the WAL is ignored by the reader so return early, as if the
-  ** WAL were empty.
-  */
-  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
+  /* Return early if read-lock 0 is held. */
+  if( (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
+    assert( !bWal2 );
     *piRead = 0;
     return SQLITE_OK;
   }
 
-  /* Search the hash table or tables for an entry matching page number
-  ** pgno. Each iteration of the following for() loop searches one
-  ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
-  **
-  ** This code might run concurrently to the code in walIndexAppend()
-  ** that adds entries to the wal-index (and possibly to this hash
-  ** table). This means the value just read from the hash
-  ** slot (aHash[iKey]) may have been added before or after the
-  ** current read transaction was opened. Values added after the
-  ** read transaction was opened may have been written incorrectly -
-  ** i.e. these slots may contain garbage data. However, we assume
-  ** that any slots written before the current read transaction was
-  ** opened remain unmodified.
-  **
-  ** For the reasons above, the if(...) condition featured in the inner
-  ** loop of the following block is more stringent that would be required
-  ** if we had exclusive access to the hash-table:
-  **
-  **   (aPgno[iFrame]==pgno):
-  **     This condition filters out normal hash-table collisions.
-  **
-  **   (iFrame<=iLast):
-  **     This condition filters out entries that were added to the hash
-  **     table after the current read-transaction had started.
-  */
-  iMinHash = walFramePage(pWal->minFrame);
-  for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){
-    WalHashLoc sLoc;              /* Hash table location */
-    int iKey;                     /* Hash slot index */
-    int nCollide;                 /* Number of hash collisions remaining */
-    int rc;                       /* Error code */
-    u32 iH;
-
-    rc = walHashGet(pWal, iHash, &sLoc);
-    if( rc!=SQLITE_OK ){
-      return rc;
-    }
-    nCollide = HASHTABLE_NSLOT;
-    iKey = walHash(pgno);
-    SEH_INJECT_FAULT;
-    while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){
-      u32 iFrame = iH + sLoc.iZero;
-      if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){
-        assert( iFrame>iRead || CORRUPT_DB );
-        iRead = iFrame;
-      }
-      if( (nCollide--)==0 ){
-        *piRead = 0;
-        return SQLITE_CORRUPT_BKPT;
-      }
-      iKey = walNextHash(iKey);
-    }
-    if( iRead ) break;
+  /* Search the wal file that the client holds a partial lock on first. */
+  rc = walSearchWal(pWal, iApp, pgno, &iRead);
+
+  /* If the requested page was not found, no error has occured, and
+  ** the client holds a full-wal lock on the other wal file, search it
+  ** too.  */
+  if( rc==SQLITE_OK && bWal2 && iRead==0 && (
+        pWal->readLock==WAL_LOCK_PART1_FULL2
+     || pWal->readLock==WAL_LOCK_PART2_FULL1
+#ifndef SQLITE_OMIT_CONCURRENT
+     || (pWal->readLock==WAL_LOCK_PART1 && iApp==1)
+     || (pWal->readLock==WAL_LOCK_PART2 && iApp==0)
+#endif
+  )){
+    rc = walSearchWal(pWal, !iApp, pgno, &iRead);
   }
+  if( rc!=SQLITE_OK ) return rc;
 
-#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
+#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
+  if( iRead ){
+    u32 iFrame;
+    int iWal = walExternalDecode(iRead, &iFrame);
+    WALTRACE(("WAL%p: page %d @ frame %d wal %d\n",pWal,(int)pgno,iFrame,iWal));
+  }else{
+    WALTRACE(("WAL%p: page %d not found\n", pWal, (int)pgno));
+  }
+#endif
+
+#if  defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) && /*TODO*/ 0
   /* If expensive assert() statements are available, do a linear search
   ** of the wal-index file content. Make sure the results agree with the
-  ** result obtained using the hash indexes above.  */
-  {
+  ** result obtained using the hash indexes above.
+  **
+  ** TODO: This is broken for wal2.
+  */
+  if( rc==SQLITE_OK ){
     u32 iRead2 = 0;
     u32 iTest;
     assert( pWal->bShmUnreliable || pWal->minFrame>0 );
@@ -68611,31 +70030,74 @@ SQLITE_PRIVATE int sqlite3WalFindFrame(
 */
 SQLITE_PRIVATE int sqlite3WalReadFrame(
   Wal *pWal,                      /* WAL handle */
-  u32 iRead,                      /* Frame to read */
+  u32 iExternal,                  /* Frame to read */
   int nOut,                       /* Size of buffer pOut in bytes */
   u8 *pOut                        /* Buffer to write page data to */
 ){
   int sz;
+  int iWal = 0;
+  u32 iRead;
   i64 iOffset;
+
+  /* Figure out the page size */
   sz = pWal->hdr.szPage;
   sz = (sz&0xfe00) + ((sz&0x0001)<<16);
   testcase( sz<=32768 );
   testcase( sz>=65536 );
+
+  if( isWalMode2(pWal) ){
+    /* Figure out which of the two wal files, and the frame within, that
+    ** iExternal refers to.  */
+    iWal = walExternalDecode(iExternal, &iRead);
+  }else{
+    iRead = iExternal;
+  }
+
+  WALTRACE(("WAL%p: reading frame %d wal %d\n", pWal, iRead, iWal));
   iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE;
   /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */
-  return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
+  return sqlite3OsRead(pWal->apWalFd[iWal], pOut, (nOut>sz?sz:nOut), iOffset);
 }
 
 /*
 ** Return the size of the database in pages (or zero, if unknown).
 */
 SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){
-  if( pWal && ALWAYS(pWal->readLock>=0) ){
+  if( pWal && ALWAYS(pWal->readLock!=WAL_LOCK_NONE) ){
     return pWal->hdr.nPage;
   }
   return 0;
 }
 
+/*
+** Take the WRITER lock on the WAL file. Return SQLITE_OK if successful,
+** or an SQLite error code otherwise. This routine does not invoke any
+** busy-handler callbacks, that is done at a higher level.
+*/
+static int walWriteLock(Wal *pWal){
+  int rc;
+
+  /* Cannot start a write transaction without first holding a read lock */
+  assert( pWal->readLock>=0 );
+  assert( pWal->writeLock==0 );
+  assert( pWal->iReCksum==0 );
+
+  /* If this is a read-only connection, obtaining a write-lock is not
+  ** possible. In this case return SQLITE_READONLY. Otherwise, attempt
+  ** to grab the WRITER lock. Set Wal.writeLock to true and return
+  ** SQLITE_OK if successful, or leave Wal.writeLock clear and return
+  ** an SQLite error code (possibly SQLITE_BUSY) otherwise. */
+  if( pWal->readOnly ){
+    rc = SQLITE_READONLY;
+  }else{
+    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
+    if( rc==SQLITE_OK ){
+      pWal->writeLock = 1;
+    }
+  }
+
+  return rc;
+}
 
 /*
 ** This function starts a write transaction on the WAL.
@@ -68663,41 +70125,303 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){
   }
 #endif
 
-  /* Cannot start a write transaction without first holding a read
-  ** transaction. */
-  assert( pWal->readLock>=0 );
-  assert( pWal->writeLock==0 && pWal->iReCksum==0 );
-
-  if( pWal->readOnly ){
-    return SQLITE_READONLY;
+  rc = walWriteLock(pWal);
+  if( rc==SQLITE_OK ){
+    /* If another connection has written to the database file since the
+    ** time the read transaction on this connection was started, then
+    ** the write is disallowed. Release the WRITER lock and return
+    ** SQLITE_BUSY_SNAPSHOT in this case.  */
+    SEH_TRY {
+      if( memcmp(&pWal->hdr, (void*)walIndexHdr(pWal),sizeof(WalIndexHdr))!=0 ){
+        rc = SQLITE_BUSY_SNAPSHOT;
+      }
+    }
+    SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+    if( rc!=SQLITE_OK ){
+      walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
+      pWal->writeLock = 0;
+    }
   }
+  return rc;
+}
 
-  /* Only one writer allowed at a time.  Get the write lock.  Return
-  ** SQLITE_BUSY if unable.
-  */
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  if( rc ){
-    return rc;
+/*
+** This function is called by a writer that has a read-lock on aReadmark[0]
+** (pWal->readLock==0). This function relinquishes that lock and takes a
+** lock on a different aReadmark[] slot.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+static int walUpgradeReadlock(Wal *pWal){
+  int cnt;
+  int rc;
+  assert( pWal->writeLock && pWal->readLock==0 );
+  assert( isWalMode2(pWal)==0 );
+  walUnlockShared(pWal, WAL_READ_LOCK(0));
+  pWal->readLock = -1;
+  cnt = 0;
+  do{
+    int notUsed;
+    rc = walTryBeginRead(pWal, &notUsed, 1, &cnt);
+  }while( rc==WAL_RETRY );
+  assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
+  testcase( (rc&0xff)==SQLITE_IOERR );
+  testcase( rc==SQLITE_PROTOCOL );
+  testcase( rc==SQLITE_OK );
+  return rc;
+}
+
+
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** A concurrent transaction has conflicted with external frame iExternal.
+** Transform this value to the one required by SQLITE_COMMIT_CONFLICT_FRAME -
+** the frame offset within its wal file, with the 0x80000000 bit set for
+** wal2, clear for the default wal file.
+*/
+static u32 walConflictFrame(Wal *pWal, u32 iExternal){
+  u32 iRet = iExternal;
+  if( isWalMode2(pWal) ){
+    int bFile = walExternalDecode(iExternal, &iRet);
+    iRet = (iRet | (bFile ? 0x80000000 : 0));
   }
-  pWal->writeLock = 1;
+  return iRet;
+}
+
+/*
+** This function does the work of sqlite3WalLockForCommit(). The difference
+** between this function and sqlite3WalLockForCommit() is that the latter
+** encloses everything in a SEH_TRY {} block.
+*/
+static int walLockForCommit(
+  Wal *pWal,
+  PgHdr *pPg1,
+  Bitvec *pAllRead,
+  u32 *aConflict
+){
+  int rc = walWriteLock(pWal);
 
-  /* If another connection has written to the database file since the
-  ** time the read transaction on this connection was started, then
-  ** the write is disallowed.
+  /* If the database has been modified since this transaction was started,
+  ** check if it is still possible to commit. The transaction can be
+  ** committed if:
+  **
+  **   a) None of the pages in pList have been modified since the
+  **      transaction opened, and
+  **
+  **   b) The database schema cookie has not been modified since the
+  **      transaction was started.
   */
-  SEH_TRY {
-    if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
+  if( rc==SQLITE_OK ){
+    WalIndexHdr head;
+
+    if( walIndexLoadHdr(pWal, &head) ){
+      /* This branch is taken if the wal-index header is corrupted. This
+      ** occurs if some other writer has crashed while committing a
+      ** transaction to this database since the current concurrent transaction
+      ** was opened.  */
       rc = SQLITE_BUSY_SNAPSHOT;
+    }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
+      int bWal2 = isWalMode2(pWal);
+      int iHash;
+      int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr));
+      int iLoop;
+
+      if( pPg1==0 ){
+        /* If pPg1==0, then the current transaction modified the database
+        ** schema. This means it conflicts with all other transactions. */
+        u32 bFile = walidxGetFile(&pWal->hdr);
+        u32 iFrame = walidxGetMxFrame(&head, bFile) | (bFile << 31);
+        aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = 1;
+        aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame;
+        rc = SQLITE_BUSY_SNAPSHOT;
+      }
+
+      assert( nLoop==1 || nLoop==2 );
+      for(iLoop=0; rc==SQLITE_OK && iLoop<nLoop; iLoop++){
+        u32 iFirst;               /* First (external) wal frame to check */
+        int iLastHash;            /* Last hash to check this loop */
+        u32 mxFrame;              /* Last (external) wal frame to check */
+
+        if( bWal2==0 ){
+          assert( iLoop==0 );
+          /* Special case for wal mode. If this concurrent transaction was
+          ** opened after the entire wal file had been checkpointed, and
+          ** another connection has since wrapped the wal file, then we wish to
+          ** iterate through every frame in the new wal file - not just those
+          ** that follow the current value of pWal->hdr.mxFrame (which will be
+          ** set to the size of the old, now overwritten, wal file). This
+          ** doesn't come up in wal2 mode, as in wal2 mode the client always
+          ** has a PART lock on one of the wal files, preventing it from being
+          ** checkpointed or overwritten. */
+          iFirst = pWal->hdr.mxFrame+1;
+          if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){
+            assert( pWal->readLock==0 );
+            iFirst = 1;
+          }
+          mxFrame = head.mxFrame;
+        }else{
+          int iA = walidxGetFile(&pWal->hdr);
+          if( iLoop==0 ){
+            iFirst = walExternalEncode(iA, 1+walidxGetMxFrame(&pWal->hdr, iA));
+            mxFrame = walExternalEncode(iA, walidxGetMxFrame(&head, iA));
+          }else{
+            iFirst = walExternalEncode(!iA, 1);
+            mxFrame = walExternalEncode(!iA, walidxGetMxFrame(&head, !iA));
+          }
+        }
+        iLastHash = walFramePage(mxFrame);
+
+        for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash += (1+bWal2)){
+          WalHashLoc sLoc;
+
+          rc = walHashGet(pWal, iHash, &sLoc);
+          if( rc==SQLITE_OK ){
+            u32 i, iMin, iMax;
+            assert( mxFrame>=sLoc.iZero );
+            iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero);
+            iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE;
+            if( iMax>(mxFrame-sLoc.iZero) ) iMax = (mxFrame-sLoc.iZero);
+            for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){
+              PgHdr *pPg;
+              if( sLoc.aPgno[i-1]==1 ){
+                /* Check that the schema cookie has not been modified. If
+                ** it has not, the commit can proceed. */
+                u8 aNew[4];
+                u8 *aOld = &((u8*)pPg1->pData)[40];
+                int sz;
+                i64 iOff;
+                u32 iFrame = sLoc.iZero + i;
+                int iWal = 0;
+                if( bWal2 ){
+                  iWal = walExternalDecode(iFrame, &iFrame);
+                }
+                sz = head.szPage;
+                sz = (sz&0xfe00) + ((sz&0x0001)<<16);
+                iOff = walFrameOffset(iFrame, sz) + WAL_FRAME_HDRSIZE + 40;
+                rc = sqlite3OsRead(pWal->apWalFd[iWal],aNew,sizeof(aNew),iOff);
+                if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){
+                  u32 iFrame = walConflictFrame(pWal, sLoc.iZero+i);
+                  aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = 1;
+                  aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame;
+                  rc = SQLITE_BUSY_SNAPSHOT;
+                }
+              }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i-1]) ){
+                u32 iFrame = walConflictFrame(pWal, sLoc.iZero+i);
+                aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = sLoc.aPgno[i-1];
+                aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame;
+                rc = SQLITE_BUSY_SNAPSHOT;
+              }else
+              if( (pPg = sqlite3PagerLookup(pPg1->pPager, sLoc.aPgno[i-1])) ){
+                /* Page aPgno[i], which is present in the pager cache, has been
+                ** modified since the current CONCURRENT transaction was
+                ** started.  However it was not read by the current
+                ** transaction, so is not a conflict. There are two
+                ** possibilities: (a) the page was allocated at the of the file
+                ** by the current transaction or (b) was present in the cache
+                ** at the start of the transaction.
+                **
+                ** For case (a), do nothing. This page will be moved within the
+                ** database file by the commit code to avoid the conflict. The
+                ** call to PagerUnref() is to release the reference grabbed by
+                ** the sqlite3PagerLookup() above.
+                **
+                ** In case (b), drop the page from the cache - otherwise
+                ** following the snapshot upgrade the cache would be
+                ** inconsistent with the database as stored on disk. */
+                if( sqlite3PagerIswriteable(pPg) ){
+                  sqlite3PagerUnref(pPg);
+                }else{
+                  sqlite3PcacheDrop(pPg);
+                }
+              }
+            }
+          }
+          if( rc!=SQLITE_OK ) break;
+        }
+      }
     }
   }
-  SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
 
-  if( rc!=SQLITE_OK ){
-    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
-    pWal->writeLock = 0;
-  }
+  pWal->nPriorFrame = walGetPriorFrame(&pWal->hdr);
+  return rc;
+}
+
+/*
+** This function is only ever called when committing a "BEGIN CONCURRENT"
+** transaction. It may be assumed that no frames have been written to
+** the wal file. The second parameter is a pointer to the in-memory
+** representation of page 1 of the database (which may or may not be
+** dirty). The third is a bitvec with a bit set for each page in the
+** database file that was read by the current concurrent transaction.
+**
+** This function performs three tasks:
+**
+**   1) It obtains the WRITER lock on the wal file,
+**
+**   2) It checks that there are no conflicts between the current
+**      transaction and any transactions committed to the wal file since
+**      it was opened, and
+**
+**   3) It ejects any non-dirty pages from the page-cache that have been
+**      written by another client since the CONCURRENT transaction was started
+**      (so as to avoid ending up with an inconsistent cache after the
+**      current transaction is committed).
+**
+** If no error occurs and the caller may proceed with committing the
+** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER
+** lock cannot be obtained. Or, if the WRITER lock can be obtained but there
+** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally,
+** if an error (i.e. an OOM condition or IO error), an SQLite error code
+** is returned.
+*/
+SQLITE_PRIVATE int sqlite3WalLockForCommit(
+  Wal *pWal,
+  PgHdr *pPg1,
+  Bitvec *pAllRead,
+  Pgno *piConflict
+){
+  int rc = SQLITE_OK;
+  SEH_TRY {
+    rc = walLockForCommit(pWal, pPg1, pAllRead, piConflict);
+  } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called as part of committing an CONCURRENT transaction.
+** It is assumed that sqlite3WalLockForCommit() has already been successfully
+** called and so (a) the WRITER lock is held and (b) it is known that the
+** wal-index-header stored in shared memory is not corrupt.
+**
+** Before returning, this function upgrades the client so that it is
+** operating on the database snapshot currently at the head of the wal file
+** (even if the CONCURRENT transaction ran against an older snapshot).
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+SQLITE_PRIVATE int sqlite3WalUpgradeSnapshot(Wal *pWal){
+  int rc = SQLITE_OK;
+  assert( pWal->writeLock );
+
+  SEH_TRY {
+    assert( pWal->szPage==pWal->hdr.szPage );
+    memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+    assert( pWal->szPage==pWal->hdr.szPage || pWal->szPage==0 );
+    pWal->szPage = pWal->hdr.szPage;
+
+    /* If this client has its read-lock on slot aReadmark[0] and the entire
+    ** wal has not been checkpointed, switch it to a different slot. Otherwise
+    ** any reads performed between now and committing the transaction will
+    ** read from the old snapshot - not the one just upgraded to.  */
+    if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){
+      assert( isWalMode2(pWal)==0 );
+      rc = walUpgradeReadlock(pWal);
+    }
+  } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
   return rc;
 }
+#endif   /* SQLITE_OMIT_CONCURRENT */
 
 /*
 ** End a write transaction.  The commit has already been done.  This
@@ -68725,22 +70449,54 @@ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){
 ** Otherwise, if the callback function does not return an error, this
 ** function returns SQLITE_OK.
 */
-SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
+SQLITE_PRIVATE int sqlite3WalUndo(
+  Wal *pWal,
+  int (*xUndo)(void *, Pgno),
+  void *pUndoCtx,
+  int bConcurrent                 /* True if this is a CONCURRENT transaction */
+){
   int rc = SQLITE_OK;
-  if( ALWAYS(pWal->writeLock) ){
-    Pgno iMax = pWal->hdr.mxFrame;
+  if( pWal->writeLock ){
+    int iWal = walidxGetFile(&pWal->hdr);
+    Pgno iMax = walidxGetMxFrame(&pWal->hdr, iWal);
+    Pgno iNew;
     Pgno iFrame;
 
+    assert( isWalMode2(pWal) || iWal==0 );
+
     SEH_TRY {
       /* Restore the clients cache of the wal-index header to the state it
       ** was in before the client began writing to the database.
       */
       memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));
+      iNew = walidxGetMxFrame(&pWal->hdr, walidxGetFile(&pWal->hdr));
+
+      /* BEGIN CONCURRENT transactions are different, as the header just
+      ** memcpy()d into pWal->hdr may not be the same as the current header
+      ** when the transaction was started. Instead, pWal->hdr now contains
+      ** the header written by the most recent successful COMMIT. Because
+      ** Wal.writeLock is set, if this is a BEGIN CONCURRENT transaction,
+      ** the rollback must be taking place because an error occurred during
+      ** a COMMIT.
+      **
+      ** The code below is still valid. All frames between (iNew+1) and iMax
+      ** must have been written by this transaction before the error occurred.
+      ** The exception is in wal2 mode - if the current wal file at the time
+      ** of the last COMMIT is not wal file iWal, then the error must have
+      ** occurred in WalLockForCommit(), before any pages were written
+      ** to the database file. In this case return early.  */
+#ifndef SQLITE_OMIT_CONCURRENT
+      if( bConcurrent ){
+        pWal->hdr.aCksum[0]++;
+      }
+      if( walidxGetFile(&pWal->hdr)!=iWal ){
+        assert( bConcurrent && isWalMode2(pWal) );
+        return SQLITE_OK;
+      }
+#endif
+      assert( walidxGetFile(&pWal->hdr)==iWal );
 
-      for(iFrame=pWal->hdr.mxFrame+1;
-          ALWAYS(rc==SQLITE_OK) && iFrame<=iMax;
-          iFrame++
-      ){
+      for(iFrame=iNew+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++){
         /* This call cannot fail. Unless the page for which the page number
         ** is passed as the second argument is (a) in the cache and
         ** (b) has an outstanding reference, then xUndo is either a no-op
@@ -68752,10 +70508,16 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p
         ** page 1 is never written to the log until the transaction is
         ** committed. As a result, the call to xUndo may not fail.
         */
-        assert( walFramePgno(pWal, iFrame)!=1 );
-        rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame));
+        Pgno pgno;
+        if( isWalMode2(pWal) ){
+          pgno = walFramePgno2(pWal, iWal, iFrame);
+        }else{
+          pgno = walFramePgno(pWal, iFrame);
+        }
+        assert( pgno!=1 );
+        rc = xUndo(pUndoCtx, pgno);
       }
-      if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal);
+      if( iMax!=iNew ) walCleanupHash(pWal);
     }
     SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
   }
@@ -68769,11 +70531,12 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p
 ** point in the event of a savepoint rollback (via WalSavepointUndo()).
 */
 SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
-  assert( pWal->writeLock );
-  aWalData[0] = pWal->hdr.mxFrame;
+  int iWal = walidxGetFile(&pWal->hdr);
+  assert( isWalMode2(pWal) || iWal==0 );
+  aWalData[0] = walidxGetMxFrame(&pWal->hdr, iWal);
   aWalData[1] = pWal->hdr.aFrameCksum[0];
   aWalData[2] = pWal->hdr.aFrameCksum[1];
-  aWalData[3] = pWal->nCkpt;
+  aWalData[3] = isWalMode2(pWal) ? (u32)iWal : pWal->nCkpt;
 }
 
 /*
@@ -68784,21 +70547,24 @@ SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
 */
 SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
   int rc = SQLITE_OK;
+  int iWal = walidxGetFile(&pWal->hdr);
+  u32 iCmp = isWalMode2(pWal) ? (u32)iWal : pWal->nCkpt;
 
-  assert( pWal->writeLock );
-  assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame );
+  assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame );
+  assert( isWalMode2(pWal) || iWal==0 );
+  assert( aWalData[3]!=iCmp || aWalData[0]<=walidxGetMxFrame(&pWal->hdr,iWal) );
 
-  if( aWalData[3]!=pWal->nCkpt ){
+  if( aWalData[3]!=iCmp ){
     /* This savepoint was opened immediately after the write-transaction
     ** was started. Right after that, the writer decided to wrap around
     ** to the start of the log. Update the savepoint values to match.
     */
     aWalData[0] = 0;
-    aWalData[3] = pWal->nCkpt;
+    aWalData[3] = iCmp;
   }
 
-  if( aWalData[0]<pWal->hdr.mxFrame ){
-    pWal->hdr.mxFrame = aWalData[0];
+  if( aWalData[0]<walidxGetMxFrame(&pWal->hdr, iWal) ){
+    walidxSetMxFrame(&pWal->hdr, iWal, aWalData[0]);
     pWal->hdr.aFrameCksum[0] = aWalData[1];
     pWal->hdr.aFrameCksum[1] = aWalData[2];
     SEH_TRY {
@@ -68813,25 +70579,61 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
 /*
 ** This function is called just before writing a set of frames to the log
 ** file (see sqlite3WalFrames()). It checks to see if, instead of appending
-** to the current log file, it is possible to overwrite the start of the
-** existing log file with the new frames (i.e. "reset" the log). If so,
-** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left
-** unchanged.
+** to the current log file, it is possible and desirable to switch to the
+** other log file and write the new transaction to the start of it.
+** If so, the wal-index header is updated accordingly - both in heap memory
+** and in the *-shm file.
 **
 ** SQLITE_OK is returned if no error is encountered (regardless of whether
-** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned
+** or not the wal-index header is modified). An SQLite error code is returned
 ** if an error occurs.
 */
 static int walRestartLog(Wal *pWal){
   int rc = SQLITE_OK;
-  int cnt;
 
-  if( pWal->readLock==0 ){
+  if( isWalMode2(pWal) ){
+    int iApp = walidxGetFile(&pWal->hdr);
+    u32 nWalSize = WAL_DEFAULT_WALSIZE;
+    if( pWal->mxWalSize>0 ){
+      nWalSize = (pWal->mxWalSize-WAL_HDRSIZE+pWal->szPage+WAL_FRAME_HDRSIZE-1)
+        / (pWal->szPage+WAL_FRAME_HDRSIZE);
+      nWalSize = MAX(nWalSize, 1);
+    }
+
+    assert( 1==WAL_LOCK_PART1 );
+    assert( 4==WAL_LOCK_PART2 );
+    assert( 1+(iApp*3)==WAL_LOCK_PART1 || 1+(iApp*3)==WAL_LOCK_PART2 );
+    if( pWal->readLock==1+(iApp*3)
+     && walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize
+    ){
+      volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
+      u32 mxFrame = walidxGetMxFrame(&pWal->hdr, !iApp);
+      if( mxFrame==0 || pInfo->nBackfill ){
+        rc = wal2RestartOk(pWal, iApp);
+        if( rc==SQLITE_OK ){
+          int iNew = !iApp;
+          pWal->nCkpt++;
+          walidxSetFile(&pWal->hdr, iNew);
+          walidxSetMxFrame(&pWal->hdr, iNew, 0);
+          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]);
+          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]);
+          walIndexWriteHdr(pWal);
+          pInfo->nBackfill = 0;
+          wal2RestartFinished(pWal, iApp);
+          walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
+          pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
+          rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
+        }else if( rc==SQLITE_BUSY ){
+          rc = SQLITE_OK;
+        }
+      }
+    }
+  }else if( pWal->readLock==0 ){
     volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
     assert( pInfo->nBackfill==pWal->hdr.mxFrame );
     if( pInfo->nBackfill>0 ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
         /* If all readers are using WAL_READ_LOCK(0) (in other words if no
@@ -68845,22 +70647,24 @@ static int walRestartLog(Wal *pWal){
         ** to handle if this transaction is rolled back.  */
         walRestartHdr(pWal, salt1);
         walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+        pWal->nPriorFrame = 0;
       }else if( rc!=SQLITE_BUSY ){
         return rc;
       }
     }
-    walUnlockShared(pWal, WAL_READ_LOCK(0));
-    pWal->readLock = -1;
-    cnt = 0;
-    do{
-      int notUsed;
-      rc = walTryBeginRead(pWal, &notUsed, 1, &cnt);
-    }while( rc==WAL_RETRY );
-    assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
-    testcase( (rc&0xff)==SQLITE_IOERR );
-    testcase( rc==SQLITE_PROTOCOL );
-    testcase( rc==SQLITE_OK );
+
+    /* Regardless of whether or not the wal file was restarted, change the
+    ** read-lock held by this client to a slot other than aReadmark[0].
+    ** Clients with a lock on aReadmark[0] read from the database file
+    ** only - never from the wal file. This means that if a writer holding
+    ** a lock on aReadmark[0] were to commit a transaction but not close the
+    ** read-transaction, subsequent read operations would read directly from
+    ** the database file - ignoring the new pages just appended
+    ** to the wal file. */
+    rc = walUpgradeReadlock(pWal);
   }
+
+  pWal->nPriorFrame = walGetPriorFrame(&pWal->hdr);
   return rc;
 }
 
@@ -68919,6 +70723,18 @@ static int walWriteOneFrame(
   int rc;                         /* Result code from subfunctions */
   void *pData;                    /* Data actually written */
   u8 aFrame[WAL_FRAME_HDRSIZE];   /* Buffer to assemble frame-header in */
+
+#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
+  {
+    int iWal = walidxGetFile(&p->pWal->hdr);
+    int iFrame = 1 + (iOffset / (WAL_FRAME_HDRSIZE + p->pWal->szPage));
+    assert( p->pWal->apWalFd[iWal]==p->pFd );
+    WALTRACE(("WAL%p: page %d written to frame %d of wal %d\n",
+          p->pWal, (int)pPage->pgno, iFrame, iWal
+    ));
+  }
+#endif
+
   pData = pPage->pData;
   walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame);
   rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset);
@@ -68937,12 +70753,13 @@ static int walWriteOneFrame(
 ** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
 */
 static int walRewriteChecksums(Wal *pWal, u32 iLast){
-  const int szPage = pWal->szPage;/* Database page size */
   int rc = SQLITE_OK;             /* Return code */
+  const int szPage = pWal->szPage;/* Database page size */
   u8 *aBuf;                       /* Buffer to load data from wal file into */
   u8 aFrame[WAL_FRAME_HDRSIZE];   /* Buffer to assemble frame-headers in */
   u32 iRead;                      /* Next frame to read from wal file */
   i64 iCksumOff;
+  sqlite3_file *pWalFd = pWal->apWalFd[walidxGetFile(&pWal->hdr)];
 
   aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE);
   if( aBuf==0 ) return SQLITE_NOMEM_BKPT;
@@ -68958,7 +70775,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){
   }else{
     iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16;
   }
-  rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff);
+  rc = sqlite3OsRead(pWalFd, aBuf, sizeof(u32)*2, iCksumOff);
   pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf);
   pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]);
 
@@ -68966,14 +70783,14 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){
   pWal->iReCksum = 0;
   for(; rc==SQLITE_OK && iRead<=iLast; iRead++){
     i64 iOff = walFrameOffset(iRead, szPage);
-    rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff);
+    rc = sqlite3OsRead(pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff);
     if( rc==SQLITE_OK ){
       u32 iPgno, nDbSize;
       iPgno = sqlite3Get4byte(aBuf);
       nDbSize = sqlite3Get4byte(&aBuf[4]);
 
       walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame);
-      rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff);
+      rc = sqlite3OsWrite(pWalFd, aFrame, sizeof(aFrame), iOff);
     }
   }
 
@@ -69003,6 +70820,8 @@ static int walFrames(
   WalWriter w;                    /* The writer */
   u32 iFirst = 0;                 /* First frame that may be overwritten */
   WalIndexHdr *pLive;             /* Pointer to shared header */
+  int iApp;
+  int bWal2 = isWalMode2(pWal);
 
   assert( pList );
   assert( pWal->writeLock );
@@ -69011,22 +70830,16 @@ static int walFrames(
   ** nTruncate==0 then this frame set does not complete the transaction. */
   assert( (isCommit!=0)==(nTruncate!=0) );
 
-#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
-  { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){}
-    WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n",
-              pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill"));
-  }
-#endif
-
   pLive = (WalIndexHdr*)walIndexHdr(pWal);
   if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){
-    iFirst = pLive->mxFrame+1;
+    /* if( isWalMode2(pWal)==0 ) */
+    iFirst = walidxGetMxFrame(pLive, walidxGetFile(pLive))+1;
   }
 
   /* See if it is possible to write these frames into the start of the
   ** log file, instead of appending to it at pWal->hdr.mxFrame.
   */
-  if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){
+  else if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){
     return rc;
   }
 
@@ -69034,16 +70847,38 @@ static int walFrames(
   ** header to the start of the WAL file. See comments at the top of
   ** this source file for a description of the WAL header format.
   */
-  iFrame = pWal->hdr.mxFrame;
+  iApp = walidxGetFile(&pWal->hdr);
+  iFrame = walidxGetMxFrame(&pWal->hdr, iApp);
+  assert( iApp==0 || bWal2 );
+
+#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
+  { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){}
+    WALTRACE(("WAL%p: frame write begin. %d frames. iWal=%d. mxFrame=%d. %s\n",
+              pWal, cnt, iApp, iFrame, isCommit ? "Commit" : "Spill"));
+  }
+#endif
+
   if( iFrame==0 ){
+    u32 iCkpt = 0;
     u8 aWalHdr[WAL_HDRSIZE];      /* Buffer to assemble wal-header in */
     u32 aCksum[2];                /* Checksum for wal-header */
 
     sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN));
-    sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION);
+    sqlite3Put4byte(&aWalHdr[4], pWal->hdr.iVersion);
     sqlite3Put4byte(&aWalHdr[8], szPage);
-    sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
-    if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt);
+    if( bWal2 ){
+      if( walidxGetMxFrame(&pWal->hdr, !iApp)>0 ){
+        u8 aPrev[4];
+        rc = sqlite3OsRead(pWal->apWalFd[!iApp], aPrev, 4, 12);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+        iCkpt = (sqlite3Get4byte(aPrev) + 1) & 0x0F;
+      }
+    }else{
+      iCkpt = pWal->nCkpt;
+    }
+    sqlite3Put4byte(&aWalHdr[12], iCkpt);
     memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
     walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum);
     sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
@@ -69055,7 +70890,7 @@ static int walFrames(
     pWal->hdr.aFrameCksum[1] = aCksum[1];
     pWal->truncateOnCommit = 1;
 
-    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
+    rc = sqlite3OsWrite(pWal->apWalFd[iApp], aWalHdr, sizeof(aWalHdr), 0);
     WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok"));
     if( rc!=SQLITE_OK ){
       return rc;
@@ -69069,7 +70904,7 @@ static int walFrames(
     **     https://sqlite.org/src/info/ff5be73dee
     */
     if( pWal->syncHeader ){
-      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
+      rc = sqlite3OsSync(pWal->apWalFd[iApp], CKPT_SYNC_FLAGS(sync_flags));
       if( rc ) return rc;
     }
   }
@@ -69079,7 +70914,7 @@ static int walFrames(
 
   /* Setup information needed to write frames into the WAL */
   w.pWal = pWal;
-  w.pFd = pWal->pWalFd;
+  w.pFd = pWal->apWalFd[iApp];
   w.iSyncPoint = 0;
   w.syncFlags = sync_flags;
   w.szPage = szPage;
@@ -69096,8 +70931,11 @@ static int walFrames(
     ** checksums must be recomputed when the transaction is committed.  */
     if( iFirst && (p->pDirty || isCommit==0) ){
       u32 iWrite = 0;
-      VVA_ONLY(rc =) walFindFrame(pWal, p->pgno, &iWrite);
+      VVA_ONLY(rc =) walSearchWal(pWal, iApp, p->pgno, &iWrite);
       assert( rc==SQLITE_OK || iWrite==0 );
+      if( iWrite && bWal2 ){
+        walExternalDecode(iWrite, &iWrite);
+      }
       if( iWrite>=iFirst ){
         i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE;
         void *pData;
@@ -69105,7 +70943,7 @@ static int walFrames(
           pWal->iReCksum = iWrite;
         }
         pData = p->pData;
-        rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff);
+        rc = sqlite3OsWrite(pWal->apWalFd[iApp], pData, szPage, iOff);
         if( rc ) return rc;
         p->flags &= ~PGHDR_WAL_APPEND;
         continue;
@@ -69122,6 +70960,7 @@ static int walFrames(
     p->flags |= PGHDR_WAL_APPEND;
   }
 
+
   /* Recalculate checksums within the wal file if required. */
   if( isCommit && pWal->iReCksum ){
     rc = walRewriteChecksums(pWal, iFrame);
@@ -69145,7 +70984,7 @@ static int walFrames(
   if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){
     int bSync = 1;
     if( pWal->padToSectorBoundary ){
-      int sectorSize = sqlite3SectorSize(pWal->pWalFd);
+      int sectorSize = sqlite3SectorSize(w.pFd);
       w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize;
       bSync = (w.iSyncPoint==iOffset);
       testcase( bSync );
@@ -69181,17 +71020,17 @@ static int walFrames(
   ** guarantees that there are no other writers, and no data that may
   ** be in use by existing readers is being overwritten.
   */
-  iFrame = pWal->hdr.mxFrame;
+  iFrame = walidxGetMxFrame(&pWal->hdr, iApp);
   for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){
     if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue;
     iFrame++;
-    rc = walIndexAppend(pWal, iFrame, p->pgno);
+    rc = walIndexAppend(pWal, iApp, iFrame, p->pgno);
   }
   assert( pLast!=0 || nExtra==0 );
   while( rc==SQLITE_OK && nExtra>0 ){
     iFrame++;
     nExtra--;
-    rc = walIndexAppend(pWal, iFrame, pLast->pgno);
+    rc = walIndexAppend(pWal, iApp, iFrame, pLast->pgno);
   }
 
   if( rc==SQLITE_OK ){
@@ -69199,7 +71038,7 @@ static int walFrames(
     pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16));
     testcase( szPage<=32768 );
     testcase( szPage>=65536 );
-    pWal->hdr.mxFrame = iFrame;
+    walidxSetMxFrame(&pWal->hdr, iApp, iFrame);
     if( isCommit ){
       pWal->hdr.iChange++;
       pWal->hdr.nPage = nTruncate;
@@ -69207,7 +71046,17 @@ static int walFrames(
     /* If this is a commit, update the wal-index header too. */
     if( isCommit ){
       walIndexWriteHdr(pWal);
-      pWal->iCallback = iFrame;
+      if( bWal2 ){
+        int iOther = !walidxGetFile(&pWal->hdr);
+        if( walidxGetMxFrame(&pWal->hdr, iOther)
+            && !walCkptInfo(pWal)->nBackfill
+        ){
+          pWal->iCallback = walidxGetMxFrame(&pWal->hdr, 0);
+          pWal->iCallback += walidxGetMxFrame(&pWal->hdr, 1);
+        }
+      }else{
+        pWal->iCallback = iFrame;
+      }
     }
   }
 
@@ -69215,6 +71064,7 @@ static int walFrames(
   return rc;
 }
 
+
 /*
 ** Write a set of frames to the log. The caller must hold the write-lock
 ** on the log file (obtained using sqlite3WalBeginWriteTransaction()).
@@ -69333,7 +71183,9 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint(
 
     /* Copy data from the log to the database file. */
     if( rc==SQLITE_OK ){
-      if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
+      if( (walPagesize(pWal)!=nBuf)
+       && ((pWal->hdr.mxFrame2 & 0x7FFFFFFF) || pWal->hdr.mxFrame)
+      ){
         rc = SQLITE_CORRUPT_BKPT;
       }else{
         rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf);
@@ -69341,21 +71193,38 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint(
 
       /* If no error occurred, set the output variables. */
       if( rc==SQLITE_OK || rc==SQLITE_BUSY ){
-        if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
+        if( pnLog ){
+          WalIndexHdr *pHdr = &pWal->hdr;
+          *pnLog = walidxGetMxFrame(pHdr, 0) + walidxGetMxFrame(pHdr, 1);
+        }
         SEH_INJECT_FAULT;
-        if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill);
+        if( pnCkpt ){
+          if( isWalMode2(pWal) ){
+            if( (int)(walCkptInfo(pWal)->nBackfill) ){
+              *pnCkpt = walidxGetMxFrame(&pWal->hdr,!walidxGetFile(&pWal->hdr));
+            }else{
+              *pnCkpt = 0;
+            }
+          }else{
+            *pnCkpt = walCkptInfo(pWal)->nBackfill;
+          }
+        }
       }
     }
   }
   SEH_EXCEPT( rc = walHandleException(pWal); )
 
-  if( isChanged ){
+  if( isChanged && pWal->bClosing==0 ){
     /* If a new wal-index header was loaded before the checkpoint was
     ** performed, then the pager-cache associated with pWal is now
     ** out of date. So zero the cached wal-index header to ensure that
     ** next time the pager opens a snapshot on this database it knows that
     ** the cache needs to be reset.
-    */
+    **
+    ** Except, do not do this if the wal is being closed. In this case
+    ** the caller needs the wal-index header to check if the database is
+    ** in wal2 mode and the "other" wal file also needs to be checkpointed.
+    ** Besides, the pager cache will not be used again in this case. */
     memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
   }
 
@@ -69415,6 +71284,7 @@ SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){
 */
 SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){
   int rc;
+
   assert( pWal->writeLock==0 );
   assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 );
 
@@ -69425,14 +71295,15 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){
   ** upgrade to exclusive-mode following such an error.
   */
 #ifndef SQLITE_USE_SEH
-  assert( pWal->readLock>=0 || pWal->lockError );
+  assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError );
 #endif
-  assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) );
+  assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) );
 
   if( op==0 ){
-    if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){
+    if( pWal->exclusiveMode ){
       pWal->exclusiveMode = WAL_NORMAL_MODE;
-      if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){
+      rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
+      if( rc!=SQLITE_OK ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
       rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
@@ -69471,9 +71342,12 @@ SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapsho
   WalIndexHdr *pRet;
   static const u32 aZero[4] = { 0, 0, 0, 0 };
 
+  /* Snapshots may not be used with wal2 mode databases. */
+  if( isWalMode2(pWal) ) return SQLITE_ERROR;
+
   assert( pWal->readLock>=0 && pWal->writeLock==0 );
 
-  if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){
+  if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,8)==0 ){
     *ppSnapshot = 0;
     return SQLITE_ERROR;
   }
@@ -69540,6 +71414,10 @@ SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){
 */
 SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){
   int rc;
+
+  /* Snapshots may not be used with wal2 mode databases. */
+  if( isWalMode2(pWal) ) return SQLITE_ERROR;
+
   SEH_TRY {
     rc = walLockShared(pWal, WAL_CKPT_LOCK);
     if( rc==SQLITE_OK ){
@@ -69552,7 +71430,7 @@ SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapsho
       }
     }
   }
-  SEH_EXCEPT( rc = walHandleException(pWal); )
+  SEH_EXCEPT( rc = walHandleException(pWal) );
   return rc;
 }
 
@@ -69583,7 +71461,27 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){
 /* Return the sqlite3_file object for the WAL file
 */
 SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){
-  return pWal->pWalFd;
+  return pWal->apWalFd[0];
+}
+
+/*
+** Return the values required by sqlite3_wal_info().
+*/
+SQLITE_PRIVATE int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){
+  int rc = SQLITE_OK;
+  if( pWal ){
+    *pnPrior = pWal->nPriorFrame;
+    *pnFrame = walGetPriorFrame(&pWal->hdr);
+  }
+  return rc;
+}
+
+/*
+** Return the journal mode used by this Wal object.
+*/
+SQLITE_PRIVATE int sqlite3WalJournalMode(Wal *pWal){
+  assert( pWal );
+  return (isWalMode2(pWal) ? PAGER_JOURNALMODE_WAL2 : PAGER_JOURNALMODE_WAL);
 }
 
 #endif /* #ifndef SQLITE_OMIT_WAL */
@@ -69843,6 +71741,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){
 typedef struct MemPage MemPage;
 typedef struct BtLock BtLock;
 typedef struct CellInfo CellInfo;
+typedef struct BtreePtrmap BtreePtrmap;
 
 /*
 ** This is a magic string that appears at the beginning of every
@@ -69886,6 +71785,9 @@ struct MemPage {
   u8 intKey;           /* True if table b-trees.  False for index b-trees */
   u8 intKeyLeaf;       /* True if the leaf of an intKey table */
   Pgno pgno;           /* Page number for this page */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Pgno pgnoRoot;       /* Root page of b-tree that this page belongs to */
+#endif
   /* Only the first 8 bytes (above) are zeroed by pager.c when a new page
   ** is allocated. All fields that follow must be initialized before use */
   u8 leaf;             /* True if a leaf page */
@@ -69971,6 +71873,12 @@ struct Btree {
 #ifndef SQLITE_OMIT_SHARED_CACHE
   BtLock lock;       /* Object used to lock page 1 */
 #endif
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* Return values for sqlite3_commit_status() requests:
+  ** SQLITE_COMMIT_FIRSTFRAME, COMMIT_NFRAME.
+  */
+  u32 aCommit[2];
+#endif
 };
 
 /*
@@ -70067,6 +71975,9 @@ struct BtShared {
   Btree *pWriter;       /* Btree with currently open write transaction */
 #endif
   u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
+#ifndef SQLITE_OMIT_CONCURRENT
+  BtreePtrmap *pMap;
+#endif
   int nPreformatSize;   /* Size of last cell written by TransferRow() */
 };
 
@@ -70284,12 +72195,19 @@ struct BtCursor {
 ** (sqliteMallocRaw), it is not possible to use conditional compilation.
 ** So, this macro is defined instead.
 */
-#ifndef SQLITE_OMIT_AUTOVACUUM
+#ifdef SQLITE_OMIT_AUTOVACUUM
+#define ISAUTOVACUUM(pBt) 0
+#else
 #define ISAUTOVACUUM(pBt) (pBt->autoVacuum)
+#endif
+
+#ifdef SQLITE_OMIT_CONCURRENT
+# define ISCONCURRENT 0
 #else
-#define ISAUTOVACUUM(pBt) 0
+# define ISCONCURRENT (pBt->pMap!=0)
 #endif
 
+#define REQUIRE_PTRMAP (ISAUTOVACUUM(pBt) || ISCONCURRENT)
 
 /*
 ** This structure is passed around through all the PRAGMA integrity_check
@@ -71170,7 +73088,240 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);         /* Forward reference */
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** The following structure - BtreePtrmap - stores the in-memory pointer map
+** used for newly allocated pages in CONCURRENT transactions. Such pages are
+** always allocated in a contiguous block (from the end of the file) starting
+** with page BtreePtrmap.iFirst.
+*/
+typedef struct RollbackEntry RollbackEntry;
+typedef struct PtrmapEntry PtrmapEntry;
+struct PtrmapEntry {
+  Pgno parent;
+  u8 eType;
+};
+struct RollbackEntry {
+  Pgno pgno;
+  Pgno parent;
+  u8 eType;
+};
+struct BtreePtrmap {
+  Pgno iFirst;                    /* First new page number aPtr[0] */
+
+  int nPtrAlloc;                  /* Allocated size of aPtr[] array */
+  PtrmapEntry *aPtr;              /* Array of parent page numbers */
+
+  int nSvpt;                      /* Used size of aSvpt[] array */
+  int nSvptAlloc;                 /* Allocated size of aSvpt[] */
+  int *aSvpt;                     /* First aRollback[] entry for savepoint i */
+
+  int nRollback;                  /* Used size of aRollback[] array */
+  int nRollbackAlloc;             /* Allocated size of aRollback[] array */
+  RollbackEntry *aRollback;       /* Array of rollback entries */
+};
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If page number pgno is greater than or equal to BtreePtrmap.iFirst,
+** store an entry for it in the pointer-map structure.
+*/
+static int btreePtrmapStore(
+  BtShared *pBt,
+  Pgno pgno,
+  u8 eType,
+  Pgno parent
+){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pgno>=pMap->iFirst ){
+    int iEntry = pgno - pMap->iFirst;
+
+    /* Grow the aPtr[] array as required */
+    while( iEntry>=pMap->nPtrAlloc ){
+      int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16;
+      PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc(
+          pMap->aPtr, nNew*sizeof(PtrmapEntry)
+      );
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry);
+        memset(&aNew[pMap->nPtrAlloc], 0, nByte);
+        pMap->aPtr = aNew;
+        pMap->nPtrAlloc = nNew;
+      }
+    }
+
+    /* Add an entry to the rollback log if required */
+    if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){
+      if( pMap->nRollback>=pMap->nRollbackAlloc ){
+        int nNew = pMap->nRollback ? pMap->nRollback*2 : 16;
+        RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc(
+            pMap->aRollback, nNew*sizeof(RollbackEntry)
+        );
+        if( aNew==0 ){
+          return SQLITE_NOMEM;
+        }else{
+          pMap->aRollback = aNew;
+          pMap->nRollbackAlloc = nNew;
+        }
+      }
+
+      pMap->aRollback[pMap->nRollback].pgno = pgno;
+      pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent;
+      pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType;
+      pMap->nRollback++;
+    }
+
+    /* Update the aPtr[] array */
+    pMap->aPtr[iEntry].parent = parent;
+    pMap->aPtr[iEntry].eType = eType;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Open savepoint iSavepoint, if it is not already open.
+*/
+static int btreePtrmapBegin(BtShared *pBt, int nSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap && nSvpt>pMap->nSvpt ){
+    int i;
+    if( nSvpt>=pMap->nSvptAlloc ){
+      int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16;
+      int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew);
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        pMap->aSvpt = aNew;
+        pMap->nSvptAlloc = nNew;
+      }
+    }
+
+    for(i=pMap->nSvpt; i<nSvpt; i++){
+      pMap->aSvpt[i] = pMap->nRollback;
+    }
+    pMap->nSvpt = nSvpt;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE)
+** savepoint iSvpt.
+*/
+static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE );
+    assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) );
+    if( iSvpt<0 ){
+      pMap->nSvpt = 0;
+      pMap->nRollback = 0;
+      memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc);
+    }else if( iSvpt<pMap->nSvpt ){
+      if( op==SAVEPOINT_ROLLBACK ){
+        int ii;
+        for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){
+          RollbackEntry *p = &pMap->aRollback[ii];
+          PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst];
+          pEntry->parent = p->parent;
+          pEntry->eType = p->eType;
+        }
+      }
+      pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK);
+      pMap->nRollback = pMap->aSvpt[iSvpt];
+    }
+  }
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called after an CONCURRENT transaction is opened on the
+** database. It allocates the BtreePtrmap structure used to track pointers
+** to allocated pages and zeroes the nFree/iTrunk fields in the database
+** header on page 1.
+*/
+static int btreePtrmapAllocate(BtShared *pBt){
+  int rc = SQLITE_OK;
+  if( pBt->pMap==0 ){
+    BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap));
+    if( pMap==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2);
+      memset(pMap, 0, sizeof(BtreePtrmap));
+      pMap->iFirst = pBt->nPage + 1;
+      pBt->pMap = pMap;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Free any BtreePtrmap structure allocated by an earlier call to
+** btreePtrmapAllocate().
+*/
+static void btreePtrmapDelete(BtShared *pBt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    sqlite3_free(pMap->aRollback);
+    sqlite3_free(pMap->aPtr);
+    sqlite3_free(pMap->aSvpt);
+    sqlite3_free(pMap);
+    pBt->pMap = 0;
+  }
+}
+
+/*
+** Check that the pointer-map does not contain any entries with a parent
+** page of 0. Call sqlite3_log() multiple times to output the entire
+** data structure if it does.
+*/
+static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){
+  Pgno i;
+  int bProblem = 0;
+  BtreePtrmap *p = pBt->pMap;
+
+  for(i=p->iFirst; i<=nPage; i++){
+    PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+    if( pEntry->eType==PTRMAP_OVERFLOW1
+     || pEntry->eType==PTRMAP_OVERFLOW2
+     || pEntry->eType==PTRMAP_BTREE
+    ){
+      if( pEntry->parent==0 ){
+        bProblem = 1;
+        break;
+      }
+    }
+  }
+
+  if( bProblem ){
+    for(i=p->iFirst; i<=nPage; i++){
+      PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+      sqlite3_log(SQLITE_CORRUPT,
+          "btreePtrmapCheck: pgno=%d eType=%d parent=%d",
+          (int)i, (int)pEntry->eType, (int)pEntry->parent
+      );
+    }
+    abort();
+  }
+}
+
+#else  /* SQLITE_OMIT_CONCURRENT */
+# define btreePtrmapAllocate(x) SQLITE_OK
+# define btreePtrmapDelete(x)
+# define btreePtrmapBegin(x,y)  SQLITE_OK
+# define btreePtrmapEnd(x,y,z)
+# define btreePtrmapCheck(y,z)
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+static void releasePage(MemPage *pPage);  /* Forward reference */
 static void releasePageOne(MemPage *pPage);      /* Forward reference */
 static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
@@ -71712,6 +73863,13 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){
   /* The super-journal page number must never be used as a pointer map page */
   assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( pBt->pMap ){
+    *pRC = btreePtrmapStore(pBt, key, eType, parent);
+    return;
+  }
+#endif
+
   assert( pBt->autoVacuum );
   if( key==0 ){
     *pRC = SQLITE_CORRUPT_BKPT;
@@ -73044,6 +75202,17 @@ static int getAndInitPage(
   return SQLITE_OK;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** Set the value of the MemPage.pgnoRoot variable, if it exists.
+*/
+static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){
+  pPg->pgnoRoot = pgnoRoot;
+}
+#else
+# define setMempageRoot(x,y)
+#endif
+
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to btreeGetPage.
@@ -73950,10 +76119,10 @@ static int lockBtree(BtShared *pBt){
       goto page1_init_failed;
     }
 #else
-    if( page1[18]>2 ){
+    if( page1[18]>3 ){
       pBt->btsFlags |= BTS_READ_ONLY;
     }
-    if( page1[19]>2 ){
+    if( page1[19]>3 ){
       goto page1_init_failed;
     }
 
@@ -73965,9 +76134,9 @@ static int lockBtree(BtShared *pBt){
     ** may not be the latest version - there may be a newer one in the log
     ** file.
     */
-    if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
+    if( page1[19]>=2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
       int isOpen = 0;
-      rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen);
+      rc = sqlite3PagerOpenWal(pBt->pPager, (page1[19]==3), &isOpen);
       if( rc!=SQLITE_OK ){
         goto page1_init_failed;
       }else{
@@ -74230,6 +76399,7 @@ static SQLITE_NOINLINE int btreeBeginTrans(
   BtShared *pBt = p->pBt;
   Pager *pPager = pBt->pPager;
   int rc = SQLITE_OK;
+  int bConcurrent = (p->db->eConcurrent && !ISAUTOVACUUM(pBt));
 
   sqlite3BtreeEnter(p);
   btreeIntegrity(p);
@@ -74317,7 +76487,8 @@ static SQLITE_NOINLINE int btreeBeginTrans(
       if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){
         rc = SQLITE_READONLY;
       }else{
-        rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db));
+        int exFlag = bConcurrent ? -1 : (wrflag>1);
+        rc = sqlite3PagerBegin(pPager, exFlag, sqlite3TempInMemory(p->db));
         if( rc==SQLITE_OK ){
           rc = newDatabase(pBt);
         }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){
@@ -74381,6 +76552,15 @@ static SQLITE_NOINLINE int btreeBeginTrans(
   }
 
 trans_begun:
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){
+    rc = sqlite3PagerBeginConcurrent(pBt->pPager);
+    if( rc==SQLITE_OK && wrflag ){
+      rc = btreePtrmapAllocate(pBt);
+    }
+  }
+#endif
+
   if( rc==SQLITE_OK ){
     if( pSchemaVersion ){
       *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]);
@@ -74390,7 +76570,11 @@ static SQLITE_NOINLINE int btreeBeginTrans(
       ** open savepoints. If the second parameter is greater than 0 and
       ** the sub-journal is not already open, then it will be opened here.
       */
-      rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint);
+      int nSavepoint = p->db->nSavepoint;
+      rc = sqlite3PagerOpenSavepoint(pPager, nSavepoint);
+      if( rc==SQLITE_OK && nSavepoint ){
+        rc = btreePtrmapBegin(pBt, nSavepoint);
+      }
     }
   }
 
@@ -74403,6 +76587,15 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVers
   if( p->sharable
    || p->inTrans==TRANS_NONE
    || (p->inTrans==TRANS_READ && wrflag!=0)
+#ifndef SQLITE_OMIT_CONCURRENT
+   /* Always use the full version for "BEGIN CONCURRENT" transactions. This
+   ** is to ensure that any required calls to btreePtrmapBegin() are made.
+   ** These calls are not present on trunk (they're part of the
+   ** begin-concurrent patch), and so they are not present in the fast path
+   ** below. And it's easier just to call the full version every time than
+   ** to complicate the code below by adding btreePtrmapBegin() calls. */
+   || p->db->eConcurrent!=CONCURRENT_NONE
+#endif
   ){
     return btreeBeginTrans(p,wrflag,pSchemaVersion);
   }
@@ -74880,6 +77073,175 @@ static int autoVacuumCommit(Btree *p){
 # define setChildPtrmaps(x) SQLITE_OK
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of merging an CONCURRENT transaction with
+** the snapshot at the head of the wal file. It relocates all pages in the
+** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap
+** structure at BtShared.pMap contains the location of the pointers to each
+** page in the range.
+**
+** If pnCurrent is NULL, then all pages in the range are moved to currently
+** free locations (i.e. free-list entries) within the database file before page
+** iFirst.
+**
+** Or, if pnCurrent is not NULL, then it points to a value containing the
+** current size of the database file in pages. In this case, all pages are
+** relocated to the end of the database file - page iFirst is relocated to
+** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on.
+** Value *pnCurrent is set to the new size of the database before this
+** function returns.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int btreeRelocateRange(
+  BtShared *pBt,                  /* B-tree handle */
+  Pgno iFirst,                    /* First page to relocate */
+  Pgno iLast,                     /* Last page to relocate */
+  Pgno *pnCurrent                 /* If not NULL, IN/OUT: Database size */
+){
+  int rc = SQLITE_OK;
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iPg;
+
+  for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
+    MemPage *pFree = 0;     /* Page allocated from free-list */
+    MemPage *pPg = 0;
+    Pgno iNew;              /* New page number for pPg */
+    PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */
+
+    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
+    pEntry = &pMap->aPtr[iPg - pMap->iFirst];
+
+    if( pEntry->eType==PTRMAP_FREEPAGE ){
+      Pgno dummy;
+      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
+      if( pFree ){
+        assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 );
+        sqlite3PcacheDrop(pFree->pDbPage);
+      }
+      assert( rc!=SQLITE_OK || dummy==iPg );
+    }else if( pnCurrent ){
+      btreeGetPage(pBt, iPg, &pPg, 0);
+      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
+      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
+      iNew = ++(*pnCurrent);
+      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
+      rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
+      releasePageNotNull(pPg);
+    }else{
+      rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE);
+      assert( rc!=SQLITE_OK || iNew<iFirst );
+      if( rc==SQLITE_OK ){
+        releasePage(pFree);
+        btreeGetPage(pBt, iPg, &pPg, 0);
+        rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1);
+        releasePage(pPg);
+      }
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** The b-tree handle passed as the only argument is about to commit an
+** CONCURRENT transaction. At this point it is guaranteed that this is
+** possible - the wal WRITER lock is held and it is known that there are
+** no conflicts with committed transactions.
+*/
+static int btreeFixUnlocked(Btree *p){
+  BtShared *pBt = p->pBt;
+  MemPage *pPage1 = pBt->pPage1;
+  u8 *p1 = pPage1->aData;
+  Pager *pPager = pBt->pPager;
+  int rc = SQLITE_OK;
+
+  /* If page 1 of the database is not writable, then no pages were allocated
+  ** or freed by this transaction. In this case no special handling is
+  ** required. Otherwise, if page 1 is dirty, proceed.  */
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iTrunk = get4byte(&p1[32]);
+  Pgno nPage = btreePagecount(pBt);
+  u32 nFree = get4byte(&p1[36]);
+
+  assert( pBt->pMap );
+  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
+  assert( p1==pPage1->aData );
+
+  if( rc==SQLITE_OK ){
+    Pgno nHPage = get4byte(&p1[28]);
+    Pgno nFin = nHPage;         /* Size of db after transaction merge */
+
+    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
+      Pgno iHTrunk = get4byte(&p1[32]);
+      u32 nHFree = get4byte(&p1[36]);
+
+      btreePtrmapCheck(pBt, nPage);
+
+      /* Attach the head database free list to the end of the current
+      ** transactions free-list (if any).  */
+      if( iTrunk!=0 ){
+        put4byte(&p1[36], nHFree + nFree);
+        put4byte(&p1[32], iTrunk);
+        while( iTrunk ){
+          DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk);
+          iTrunk = get4byte((u8*)pTrunk->pData);
+          if( iTrunk==0 ){
+            put4byte((u8*)pTrunk->pData, iHTrunk);
+          }
+          sqlite3PagerUnref(pTrunk);
+        };
+      }
+
+      if( nHPage<(pMap->iFirst-1) ){
+        /* The database consisted of (pMap->iFirst-1) pages when the current
+        ** concurrent transaction was opened. And an concurrent transaction may
+        ** not be executed on an auto-vacuum database - so the db should
+        ** not have shrunk since the transaction was opened. Therefore nHPage
+        ** should be set to (pMap->iFirst-1) or greater. */
+        rc = SQLITE_CORRUPT_BKPT;
+      }else{
+        /* The current transaction allocated pages pMap->iFirst through
+        ** nPage (inclusive) at the end of the database file. Meanwhile,
+        ** other transactions have allocated (iFirst..nHPage). So move
+        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */
+        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
+        Pgno nCurrent;                      /* Current size of db */
+
+        nCurrent = MAX(nPage, nHPage);
+        pBt->nPage = nCurrent;
+        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);
+
+        /* There are now no collisions with the snapshot at the head of the
+        ** database file. So at this point it would be possible to write
+        ** the transaction out to disk. Before doing so though, attempt to
+        ** relocate some of the new pages to free locations within the body
+        ** of the database file (i.e. free-list entries). */
+        if( rc==SQLITE_OK ){
+          assert( nCurrent!=PENDING_BYTE_PAGE(pBt) );
+          sqlite3PagerSetDbsize(pBt->pPager, nCurrent);
+          nFree = get4byte(&p1[36]);
+          nFin = nCurrent-nFree;
+          if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+            nFin--;
+          }
+          nFin = MAX(nFin, nHPage);
+          rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0);
+        }
+
+        put4byte(&p1[28], nFin);
+      }
+    }
+    sqlite3PagerSetDbsize(pPager, nFin);
+  }
+
+  return rc;
+}
+#else
+# define btreeFixUnlocked(X)  SQLITE_OK
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** This routine does the first phase of a two-phase commit.  This routine
 ** causes a rollback journal to be created (if it does not already exist)
@@ -74911,8 +77273,13 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){
   if( p->inTrans==TRANS_WRITE ){
     BtShared *pBt = p->pBt;
     sqlite3BtreeEnter(p);
+
+#ifndef SQLITE_OMIT_CONCURRENT
+    memset(p->aCommit, 0, sizeof(p->aCommit));
+#endif
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
+      assert( ISCONCURRENT==0 );
       rc = autoVacuumCommit(p);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
@@ -74923,7 +77290,25 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){
       sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage);
     }
 #endif
-    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0);
+    if( rc==SQLITE_OK && ISCONCURRENT && p->db->eConcurrent==CONCURRENT_OPEN ){
+      rc = btreeFixUnlocked(p);
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0);
+    }
+#ifndef SQLITE_OMIT_CONCURRENT
+    if( rc==SQLITE_OK ){
+      u32 iPrev = 0;
+      u32 iCurrent = 0;
+      sqlite3PagerWalInfo(pBt->pPager, &iPrev, &iCurrent);
+      if( (iPrev&0x80000000)!=(iCurrent&0x80000000) ){
+        iPrev = (iPrev & 0x7FFFFFFF) | (iCurrent & 0x80000000);
+      }
+
+      p->aCommit[SQLITE_COMMIT_FIRSTFRAME] = iPrev+1;
+      p->aCommit[SQLITE_COMMIT_NFRAME] = iCurrent-iPrev;
+    }
+#endif
     sqlite3BtreeLeave(p);
   }
   return rc;
@@ -74966,6 +77351,11 @@ static void btreeEndTransaction(Btree *p){
     unlockBtreeIfUnused(pBt);
   }
 
+  /* If this was an CONCURRENT transaction, delete the pBt->pMap object.
+  ** Also call PagerEndConcurrent() to ensure that the pager has discarded
+  ** the record of all pages read within the transaction.  */
+  btreePtrmapDelete(pBt);
+  sqlite3PagerEndConcurrent(pBt->pPager);
   btreeIntegrity(p);
 }
 
@@ -75195,6 +77585,9 @@ SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){
   ** such savepoints while the statement transaction savepoint is active.
   */
   rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement);
+  if( rc==SQLITE_OK ){
+    rc = btreePtrmapBegin(pBt, iStatement);
+  }
   sqlite3BtreeLeave(p);
   return rc;
 }
@@ -75218,6 +77611,7 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){
     assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
     assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
     sqlite3BtreeEnter(p);
+    btreePtrmapEnd(pBt, op, iSavepoint);
     if( op==SAVEPOINT_ROLLBACK ){
       rc = saveAllCursors(pBt, 0, 0);
     }
@@ -75878,6 +78272,8 @@ static int accessPayload(
           u8 *aWrite = &pBuf[-4];
           assert( aWrite>=pBufStart );                         /* due to (6) */
           memcpy(aSave, aWrite, 4);
+          rc = sqlite3PagerUsePage(pBt->pPager, nextPage);
+          if( rc!=SQLITE_OK ) break;
           rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
           nextPage = get4byte(aWrite);
           memcpy(aWrite, aSave, 4);
@@ -75890,6 +78286,9 @@ static int accessPayload(
               (eOp==0 ? PAGER_GET_READONLY : 0)
           );
           if( rc==SQLITE_OK ){
+            setMempageRoot(
+                (MemPage*)sqlite3PagerGetExtra(pDbPage), pCur->pgnoRoot
+            );
             aPayload = sqlite3PagerGetData(pDbPage);
             nextPage = get4byte(aPayload);
             rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage);
@@ -76040,6 +78439,7 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
 ** vice-versa).
 */
 static int moveToChild(BtCursor *pCur, u32 newPgno){
+  BtShared *pBt = pCur->pBt;
   int rc;
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
@@ -76054,13 +78454,14 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
   pCur->iPage++;
-  rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags);
-  assert( pCur->pPage!=0 || rc!=SQLITE_OK );
-  if( rc==SQLITE_OK
-   && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey)
-  ){
-    releasePage(pCur->pPage);
-    rc = SQLITE_CORRUPT_PGNO(newPgno);
+  rc = getAndInitPage(pBt, newPgno, &pCur->pPage, pCur->curPagerFlags);
+  if( rc==SQLITE_OK ){
+    assert( pCur->pPage!=0 );
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
+    if( pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey ){
+      releasePage(pCur->pPage);
+      rc = SQLITE_CORRUPT_PGNO(newPgno);
+    }
   }
   if( rc ){
     pCur->pPage = pCur->apPage[--pCur->iPage];
@@ -76177,6 +78578,7 @@ static int moveToRoot(BtCursor *pCur){
       pCur->eState = CURSOR_INVALID;
       return rc;
     }
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
     pCur->iPage = 0;
     pCur->curIntKey = pCur->pPage->intKey;
   }
@@ -76819,6 +79221,7 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto(
       pCur->pPage = pCur->apPage[--pCur->iPage];
       break;
     }
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
     /*
     ***** End of in-lined moveToChild() call */
  }
@@ -77089,7 +79492,7 @@ static int allocateBtreePage(
   Pgno mxPage;     /* Total size of the database file */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
+  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
   pPage1 = pBt->pPage1;
   mxPage = btreePagecount(pBt);
   /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36
@@ -77099,6 +79502,15 @@ static int allocateBtreePage(
   if( n>=mxPage ){
     return SQLITE_CORRUPT_BKPT;
   }
+
+  /* Ensure page 1 is writable. This function will either change the number
+  ** of pages in the free-list or the size of the database file. Since both
+  ** of these operations involve modifying page 1 header fields, page 1
+  ** will definitely be written by this transaction. If this is an CONCURRENT
+  ** transaction, ensure the BtreePtrmap structure has been allocated.  */
+  rc = sqlite3PagerWrite(pPage1->pDbPage);
+  if( rc ) return rc;
+
   if( n>0 ){
     /* There are pages on the freelist.  Reuse one of those pages. */
     Pgno iTrunk;
@@ -77109,28 +79521,29 @@ static int allocateBtreePage(
     ** shows that the page 'nearby' is somewhere on the free-list, then
     ** the entire-list will be searched for that page.
     */
-#ifndef SQLITE_OMIT_AUTOVACUUM
     if( eMode==BTALLOC_EXACT ){
-      if( nearby<=mxPage ){
-        u8 eType;
-        assert( nearby>0 );
-        assert( pBt->autoVacuum );
-        rc = ptrmapGet(pBt, nearby, &eType, 0);
-        if( rc ) return rc;
-        if( eType==PTRMAP_FREEPAGE ){
-          searchList = 1;
+      assert( ISAUTOVACUUM(pBt)!=ISCONCURRENT );
+      if( ISAUTOVACUUM(pBt) ){
+        if( nearby<=mxPage ){
+          u8 eType;
+          assert( nearby>0 );
+          assert( pBt->autoVacuum );
+          rc = ptrmapGet(pBt, nearby, &eType, 0);
+          if( rc ) return rc;
+          if( eType==PTRMAP_FREEPAGE ){
+            searchList = 1;
+          }
         }
+      }else{
+        searchList = 1;
       }
     }else if( eMode==BTALLOC_LE ){
       searchList = 1;
     }
-#endif
 
     /* Decrement the free-list count by 1. Set iTrunk to the index of the
     ** first free-list trunk page. iPrevTrunk is initially 1.
     */
-    rc = sqlite3PagerWrite(pPage1->pDbPage);
-    if( rc ) return rc;
     put4byte(&pPage1->aData[36], n-1);
 
     /* The code within this loop is run only once if the 'searchList' variable
@@ -77438,7 +79851,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   /* If the database supports auto-vacuum, write an entry in the pointer-map
   ** to indicate that the page is free.
   */
-  if( ISAUTOVACUUM(pBt) ){
+  if( REQUIRE_PTRMAP ){
     ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc);
     if( rc ) goto freepage_out;
   }
@@ -77772,7 +80185,7 @@ static int fillInCell(
       }
 #endif
       rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0);
-#ifndef SQLITE_OMIT_AUTOVACUUM
+
       /* If the database supports auto-vacuum, and the second or subsequent
       ** overflow page is being allocated, add an entry to the pointer-map
       ** for that page now.
@@ -77783,14 +80196,13 @@ static int fillInCell(
       ** may misinterpret the uninitialized values and delete the
       ** wrong pages from the database.
       */
-      if( pBt->autoVacuum && rc==SQLITE_OK ){
+      if( REQUIRE_PTRMAP && rc==SQLITE_OK ){
         u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1);
         ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc);
         if( rc ){
           releasePage(pOvfl);
         }
       }
-#endif
       if( rc ){
         releasePage(pToRelease);
         return rc;
@@ -77934,6 +80346,7 @@ static int insertCell(
     assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
     assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
   }else{
+    BtShared *pBt = pPage->pBt;
     int rc = sqlite3PagerWrite(pPage->pDbPage);
     if( NEVER(rc!=SQLITE_OK) ){
       return rc;
@@ -77964,7 +80377,7 @@ static int insertCell(
     if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
     assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB );
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPage->pBt->autoVacuum ){
+    if( REQUIRE_PTRMAP ){
       int rc2 = SQLITE_OK;
       /* The cell may contain a pointer to an overflow page. If so, write
       ** the entry for the overflow page into the pointer map.
@@ -78024,6 +80437,7 @@ static int insertCellFast(
     assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
     assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
   }else{
+    BtShared *pBt = pPage->pBt;
     int rc = sqlite3PagerWrite(pPage->pDbPage);
     if( rc!=SQLITE_OK ){
       return rc;
@@ -78037,7 +80451,7 @@ static int insertCellFast(
     ** if it returns successfully */
     assert( idx >= 0 );
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
-    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    assert( idx+sz <= (int)pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
     memcpy(&data[idx], pCell, sz);
     pIns = pPage->aCellIdx + i*2;
@@ -78047,8 +80461,7 @@ static int insertCellFast(
     /* increment the cell count */
     if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
     assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB );
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPage->pBt->autoVacuum ){
+    if( REQUIRE_PTRMAP ){
       int rc2 = SQLITE_OK;
       /* The cell may contain a pointer to an overflow page. If so, write
       ** the entry for the overflow page into the pointer map.
@@ -78056,7 +80469,6 @@ static int insertCellFast(
       ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2);
       if( rc2 ) return rc2;
     }
-#endif
   }
   return SQLITE_OK;
 }
@@ -78615,7 +81027,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
     ** be marked as dirty. Returning an error code will cause a
     ** rollback, undoing any changes made to the parent page.
     */
-    if( ISAUTOVACUUM(pBt) ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc);
       if( szCell>pNew->minLocal ){
         ptrmapPutOvflPtr(pNew, pNew, pCell, &rc);
@@ -78753,7 +81165,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){
     /* If this is an auto-vacuum database, update the pointer-map entries
     ** for any b-tree or overflow pages that pTo now contains the pointers to.
     */
-    if( ISAUTOVACUUM(pBt) ){
+    if( REQUIRE_PTRMAP ){
       *pRC = setChildPtrmaps(pTo);
     }
   }
@@ -78804,7 +81216,8 @@ static int balance_nonroot(
   int iParentIdx,                 /* Index of "the page" in pParent */
   u8 *aOvflSpace,                 /* page-size bytes of space for parent ovfl */
   int isRoot,                     /* True if pParent is a root-page */
-  int bBulk                       /* True if this call is part of a bulk load */
+  int bBulk,                      /* True if this call is part of a bulk load */
+  Pgno pgnoRoot                   /* Root page of b-tree being balanced */
 ){
   BtShared *pBt;               /* The whole database */
   int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
@@ -78901,6 +81314,7 @@ static int balance_nonroot(
         goto balance_cleanup;
       }
     }
+    setMempageRoot(apOld[i], pgnoRoot);
     nMaxCells += apOld[i]->nCell + ArraySize(pParent->apOvfl);
     if( (i--)==0 ) break;
 
@@ -79243,7 +81657,7 @@ static int balance_nonroot(
       cntOld[i] = b.nCell;
 
       /* Set the pointer-map entry for the new sibling page. */
-      if( ISAUTOVACUUM(pBt) ){
+      if( REQUIRE_PTRMAP ){
         ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
         if( rc!=SQLITE_OK ){
           goto balance_cleanup;
@@ -79336,7 +81750,7 @@ static int balance_nonroot(
   ** updated. This happens below, after the sibling pages have been
   ** populated, not here.
   */
-  if( ISAUTOVACUUM(pBt) ){
+  if( REQUIRE_PTRMAP ){
     MemPage *pOld;
     MemPage *pNew = pOld = apNew[0];
     int cntOldNext = pNew->nCell + pNew->nOverflow;
@@ -79532,7 +81946,7 @@ static int balance_nonroot(
     );
     copyNodeContent(apNew[0], pParent, &rc);
     freePage(apNew[0], &rc);
-  }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){
+  }else if( REQUIRE_PTRMAP && !leafCorrection ){
     /* Fix the pointer map entries associated with the right-child of each
     ** sibling page. All other pointer map entries have already been taken
     ** care of.  */
@@ -79615,7 +82029,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){
   if( rc==SQLITE_OK ){
     rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0);
     copyNodeContent(pRoot, pChild, &rc);
-    if( ISAUTOVACUUM(pBt) ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc);
     }
   }
@@ -79778,7 +82192,7 @@ static int balance(BtCursor *pCur){
           */
           u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize);
           rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1,
-                               pCur->hints&BTREE_BULKLOAD);
+                               pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot);
           if( pFree ){
             /* If pFree is not NULL, it points to the pSpace buffer used
             ** by a previous call to balance_nonroot(). Its contents are
@@ -79886,6 +82300,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell(
   do{
     rc = btreeGetPage(pBt, ovflPgno, &pPage, 0);
     if( rc ) return rc;
+    setMempageRoot(pPage, pCur->pgnoRoot);
     if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){
       rc = SQLITE_CORRUPT_PAGE(pPage);
     }else{
@@ -80159,6 +82574,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
   pCur->info.nSize = 0;
   if( loc==0 ){
     CellInfo info;
+    BtShared *pBt = p->pBt;
     assert( idx>=0 );
     if( idx>=pPage->nCell ){
       return SQLITE_CORRUPT_PAGE(pPage);
@@ -80175,7 +82591,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
     testcase( pCur->curFlags & BTCF_ValidOvfl );
     invalidateOverflowCache(pCur);
     if( info.nSize==szNew && info.nLocal==info.nPayload
-     && (!ISAUTOVACUUM(p->pBt) || szNew<pPage->minLocal)
+     && (!REQUIRE_PTRMAP || szNew<pPage->minLocal)
     ){
       /* Overwrite the old cell with the new if they are the same size.
       ** We could also try to do this if the old cell is smaller, then add
@@ -80340,6 +82756,9 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64
           pPageIn = 0;
           rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY);
           if( rc==SQLITE_OK ){
+            setMempageRoot(
+                (MemPage*)sqlite3PagerGetExtra(pPageIn), pSrc->pgnoRoot
+            );
             aIn = (const u8*)sqlite3PagerGetData(pPageIn);
             ovflIn = get4byte(aIn);
             aIn += 4;
@@ -80764,7 +83183,8 @@ static int clearDatabasePage(
   BtShared *pBt,           /* The BTree that contains the table */
   Pgno pgno,               /* Page number to clear */
   int freePageFlag,        /* Deallocate page if true */
-  i64 *pnChange            /* Add number of Cells freed to this counter */
+  i64 *pnChange,           /* Add number of Cells freed to this counter */
+  Pgno pgnoRoot
 ){
   MemPage *pPage;
   int rc;
@@ -80779,6 +83199,7 @@ static int clearDatabasePage(
   }
   rc = getAndInitPage(pBt, pgno, &pPage, 0);
   if( rc ) return rc;
+  setMempageRoot(pPage, pgnoRoot);
   if( (pBt->openFlags & BTREE_SINGLE)==0
    && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1))
   ){
@@ -80789,14 +83210,16 @@ static int clearDatabasePage(
   for(i=0; i<pPage->nCell; i++){
     pCell = findCell(pPage, i);
     if( !pPage->leaf ){
-      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange);
+      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot);
       if( rc ) goto cleardatabasepage_out;
     }
     BTREE_CLEAR_CELL(rc, pPage, pCell, info);
     if( rc ) goto cleardatabasepage_out;
   }
   if( !pPage->leaf ){
-    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
+    rc = clearDatabasePage(
+        pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot
+    );
     if( rc ) goto cleardatabasepage_out;
     if( pPage->intKey ) pnChange = 0;
   }
@@ -80842,7 +83265,7 @@ SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, i64 *pnChange){
     if( p->hasIncrblobCur ){
       invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1);
     }
-    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
+    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable);
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -81798,10 +84221,12 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck(
   }
   pBt->db->flags = savedDbFlags;
 
-  /* Make sure every page in the file is referenced
-  */
+  /* Make sure every page in the file is referenced. Skip this if the
+  ** database is currently being written by a CONCURRENT transaction (it
+  ** may fail as pages that were part of the free-list when the transaction
+  ** was opened cannot be counted).  */
   if( !bPartial ){
-    for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){
+    for(i=1; ISCONCURRENT==0 && i<=sCheck.nCkPage && sCheck.mxErr; i++){
 #ifdef SQLITE_OMIT_AUTOVACUUM
       if( getPageReferenced(&sCheck, i)==0 ){
         checkAppendMsg(&sCheck, "Page %u: never used", i);
@@ -82055,7 +84480,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
   BtShared *pBt = pBtree->pBt;
   int rc;                         /* Return code */
 
-  assert( iVersion==1 || iVersion==2 );
+  assert( iVersion==1 || iVersion==2 || iVersion==3 );
 
   /* If setting the version fields to 1, do not automatically open the
   ** WAL connection, even if the version fields are currently set to 2.
@@ -82102,6 +84527,109 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){
 */
 SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); }
 
+/*
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
+**
+** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER
+** lock on the wal file. In this case this function also checks that the
+** CONCURRENT transaction can be safely committed (does not commit with any
+** other transaction committed since it was opened).
+**
+** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks
+** cannot be obtained due to a conflicting lock. If the locks cannot be
+** obtained for an CONCURRENT transaction due to a conflict with an already
+** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if
+** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code
+** is returned.
+*/
+SQLITE_PRIVATE int sqlite3BtreeExclusiveLock(Btree *p){
+  sqlite3 *db = p->db;
+  int rc;
+  Pgno pgno = 0;
+  BtShared *pBt = p->pBt;
+  assert( p->inTrans==TRANS_WRITE && pBt->pPage1 );
+  memset(db->aCommit, 0, sizeof(db->aCommit));
+  sqlite3BtreeEnter(p);
+  rc = sqlite3PagerExclusiveLock(pBt->pPager,
+    (db->eConcurrent==CONCURRENT_SCHEMA) ? 0 : pBt->pPage1->pDbPage,
+    db->aCommit
+  );
+#ifdef SQLITE_OMIT_CONCURRENT
+  assert( db->aCommit[SQLITE_COMMIT_CONFLICT_PGNO]==0 );
+#else
+  if( (rc==SQLITE_BUSY_SNAPSHOT)
+   && (pgno = db->aCommit[SQLITE_COMMIT_CONFLICT_PGNO])
+  ){
+    int iDb;
+    PgHdr *pPg = 0;
+    for(iDb=0; db->aDb[iDb].pBt!=p; iDb++);
+    db->aCommit[SQLITE_COMMIT_CONFLICT_DB] = (u32)iDb;
+    (void)sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0);
+    if( pPg ){
+      int bWrite = -1;
+      const char *zObj = 0;
+      const char *zTab = 0;
+      char zContent[17];
+
+      if( pPg ){
+        Pgno pgnoRoot = 0;
+        HashElem *pE;
+        Schema *pSchema;
+        u8 *aData = (u8*)sqlite3PagerGetData(pPg);
+        int i;
+        for(i=0; i<8; i++){
+          static const char hexdigits[] = {
+            '0', '1', '2', '3', '4', '5', '6', '7',
+            '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'
+          };
+          zContent[i*2] = hexdigits[(aData[i] >> 4)];
+          zContent[i*2+1] = hexdigits[(aData[i] & 0xF)];
+        }
+        zContent[16] = '\0';
+
+        pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot;
+        bWrite = sqlite3PagerIswriteable(pPg);
+        sqlite3PagerUnref(pPg);
+
+        pSchema = sqlite3SchemaGet(p->db, p);
+        if( pSchema ){
+          for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){
+            Table *pTab = (Table *)sqliteHashData(pE);
+            if( pTab->tnum==pgnoRoot ){
+              zObj = pTab->zName;
+              zTab = 0;
+            }else{
+              Index *pIdx;
+              for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+                if( pIdx->tnum==pgnoRoot ){
+                  zObj = pIdx->zName;
+                  zTab = pTab->zName;
+                }
+              }
+            }
+          }
+        }
+      }
+
+      sqlite3_log(SQLITE_OK,
+          "cannot commit CONCURRENT transaction "
+          "- conflict at page %d "
+          "(%s page; part of db %s %s%s%s; content=%s...)",
+          (int)pgno,
+          (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")),
+          (zTab ? "index" : "table"),
+          (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"),
+          zContent
+      );
+    }
+  }
+#endif
+  sqlite3BtreeLeave(p);
+  return rc;
+}
+
 /*
 ** If no transaction is active and the database is not a temp-db, clear
 ** the in-memory pager cache.
@@ -82132,6 +84660,40 @@ SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){
 }
 #endif
 
+/*
+** Access details of recent COMMIT commands. This function allows various
+** details related to the most recent COMMIT command to be accessed.
+** The requested value is always returned via output parameter (*piVal).
+** The specific value requested is identified by parameter op (see
+** below).
+**
+** SQLITE_OK is returned if successful, or SQLITE_ERROR if the "op" or
+** "zDb" paramters are unrecognized.
+*/
+SQLITE_API int sqlite3_commit_status(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Name of database - "main" etc. */
+  int op,                         /* SQLITE_COMMIT_XXX constant */
+  unsigned int *piVal             /* OUT: Write requested value here */
+){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( op<0 || op>SQLITE_COMMIT_CONFLICT_PGNO ){
+    rc = SQLITE_ERROR;
+  }else if( op==SQLITE_COMMIT_FIRSTFRAME || op==SQLITE_COMMIT_NFRAME ){
+    int iDb = sqlite3FindDbName(db, zDb);
+    if( iDb<0 ){
+      rc = SQLITE_ERROR;
+    }else{
+      *piVal = db->aDb[iDb].pBt->aCommit[op];
+    }
+  }else{
+    *piVal = db->aCommit[op];
+  }
+#endif
+  return rc;
+}
+
 /************** End of btree.c ***********************************************/
 /************** Begin file backup.c ******************************************/
 /*
@@ -87903,7 +90465,8 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
         /* OFF       */ 0,
         /* TRUNCATE  */ 1,
         /* MEMORY    */ 0,
-        /* WAL       */ 0
+        /* WAL       */ 0,
+        /* WAL2      */ 0
       };
       Pager *pPager;   /* Pager associated with pBt */
       needXcommit = 1;
@@ -87916,10 +90479,27 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
         assert( i!=1 );
         nTrans++;
       }
-      rc = sqlite3PagerExclusiveLock(pPager);
+      rc = sqlite3BtreeExclusiveLock(pBt);
       sqlite3BtreeLeave(pBt);
     }
   }
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->eConcurrent && (rc & 0xFF)==SQLITE_BUSY ){
+    /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while
+    ** attempting to take the WRITER lock on a wal file. Release the
+    ** WRITER locks on all wal files and return early.  */
+    for(i=0; i<db->nDb; i++){
+      Btree *pBt = db->aDb[i].pBt;
+      if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
+        sqlite3BtreeEnter(pBt);
+        sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt));
+        sqlite3BtreeLeave(pBt);
+      }
+    }
+  }
+#endif
+
   if( rc!=SQLITE_OK ){
     return rc;
   }
@@ -88321,6 +90901,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){
           sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
           sqlite3CloseSavepoints(db);
           db->autoCommit = 1;
+          db->eConcurrent = CONCURRENT_NONE;
           p->nChange = 0;
         }
       }
@@ -88359,9 +90940,9 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){
           ** is required. */
           rc = vdbeCommit(db, p);
         }
-        if( rc==SQLITE_BUSY && p->readOnly ){
+        if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){
           sqlite3VdbeLeave(p);
-          return SQLITE_BUSY;
+          return rc;
         }else if( rc!=SQLITE_OK ){
           sqlite3SystemError(db, rc);
           p->rc = rc;
@@ -88389,6 +90970,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->eConcurrent = CONCURRENT_NONE;
         p->nChange = 0;
       }
     }
@@ -88410,6 +90992,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->eConcurrent = CONCURRENT_NONE;
         p->nChange = 0;
       }
     }
@@ -97226,6 +99809,7 @@ case OP_Savepoint: {
       ** is committed.
       */
       int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
+      assert( db->eConcurrent==0 || db->isTransactionSavepoint==0 );
       if( isTransaction && p1==SAVEPOINT_RELEASE ){
         if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
           goto vdbe_return;
@@ -97312,23 +99896,31 @@ case OP_Savepoint: {
   break;
 }
 
-/* Opcode: AutoCommit P1 P2 * * *
+/* Opcode: AutoCommit P1 P2 P3 * *
 **
 ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll
 ** back any currently active btree transactions. If there are any active
 ** VMs (apart from this one), then a ROLLBACK fails.  A COMMIT fails if
 ** there are active writing VMs or active VMs that use shared cache.
 **
+** If P3 is non-zero, then this instruction is being executed as part of
+** a "BEGIN CONCURRENT" command.
+**
 ** This instruction causes the VM to halt.
 */
 case OP_AutoCommit: {
   int desiredAutoCommit;
   int iRollback;
+  int bConcurrent;
+  int hrc;
 
   desiredAutoCommit = pOp->p1;
   iRollback = pOp->p2;
+  bConcurrent = pOp->p3;
   assert( desiredAutoCommit==1 || desiredAutoCommit==0 );
   assert( desiredAutoCommit==1 || iRollback==0 );
+  assert( desiredAutoCommit==0 || bConcurrent==0 );
+  assert( db->autoCommit==0 || db->eConcurrent==CONCURRENT_NONE );
   assert( db->nVdbeActive>0 );  /* At least this one VM is active */
   assert( p->bIsReader );
 
@@ -97337,10 +99929,17 @@ case OP_AutoCommit: {
       assert( desiredAutoCommit==1 );
       sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
       db->autoCommit = 1;
-    }else if( desiredAutoCommit && db->nVdbeWrite>0 ){
-      /* If this instruction implements a COMMIT and other VMs are writing
-      ** return an error indicating that the other VMs must complete first.
-      */
+      db->eConcurrent = CONCURRENT_NONE;
+    }else if( desiredAutoCommit
+            && (db->nVdbeWrite>0 || (db->eConcurrent && db->nVdbeActive>1)) ){
+      /* A transaction may only be committed if there are no other active
+      ** writer VMs. If the transaction is CONCURRENT, then it may only be
+      ** committed if there are no active VMs at all (readers or writers).
+      **
+      ** If this instruction is a COMMIT and the transaction may not be
+      ** committed due to one of the conditions above, return an error
+      ** indicating that other VMs must complete before the COMMIT can
+      ** be processed.  */
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
@@ -97350,12 +99949,16 @@ case OP_AutoCommit: {
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
     }
-    if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
+    hrc = sqlite3VdbeHalt(p);
+    if( (hrc & 0xFF)==SQLITE_BUSY ){
       p->pc = (int)(pOp - aOp);
       db->autoCommit = (u8)(1-desiredAutoCommit);
-      p->rc = rc = SQLITE_BUSY;
+      p->rc = hrc;
+      rc = SQLITE_BUSY;
       goto vdbe_return;
     }
+    assert( bConcurrent==CONCURRENT_NONE || bConcurrent==CONCURRENT_OPEN );
+    db->eConcurrent = (u8)bConcurrent;
     sqlite3CloseSavepoints(db);
     if( p->rc==SQLITE_OK ){
       rc = SQLITE_DONE;
@@ -97568,6 +100171,17 @@ case OP_SetCookie: {
   pDb = &db->aDb[pOp->p1];
   assert( pDb->pBt!=0 );
   assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->eConcurrent
+   && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID)
+  ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction",
+        pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id"
+    );
+    goto abort_due_to_error;
+  }
+#endif
   /* See note about index shifting on OP_ReadCookie */
   rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3);
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
@@ -97717,6 +100331,11 @@ case OP_OpenWrite:
   pX = pDb->pBt;
   assert( pX!=0 );
   if( pOp->opcode==OP_OpenWrite ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    if( db->eConcurrent==CONCURRENT_OPEN && p2==1 && iDb!=1 ){
+      db->eConcurrent = CONCURRENT_SCHEMA;
+    }
+#endif
     assert( OPFLAG_FORDELETE==BTREE_FORDELETE );
     wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE);
     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
@@ -101290,6 +103909,7 @@ case OP_JournalMode: {    /* out2 */
        || eNew==PAGER_JOURNALMODE_OFF
        || eNew==PAGER_JOURNALMODE_MEMORY
        || eNew==PAGER_JOURNALMODE_WAL
+       || eNew==PAGER_JOURNALMODE_WAL2
        || eNew==PAGER_JOURNALMODE_QUERY
   );
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
@@ -101308,16 +103928,25 @@ case OP_JournalMode: {    /* out2 */
   /* Do not allow a transition to journal_mode=WAL for a database
   ** in temporary storage or if the VFS does not support shared memory
   */
-  if( eNew==PAGER_JOURNALMODE_WAL
+  if( isWalMode(eNew)
    && (sqlite3Strlen30(zFilename)==0           /* Temp file */
        || !sqlite3PagerWalSupported(pPager))   /* No shared-memory support */
   ){
     eNew = eOld;
   }
 
-  if( (eNew!=eOld)
-   && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL)
-  ){
+  if( eNew!=eOld && (isWalMode(eNew) || isWalMode(eOld)) ){
+
+    /* Prevent changing directly to wal2 from wal mode. And vice versa. */
+    if( isWalMode(eNew) && isWalMode(eOld) ){
+      rc = SQLITE_ERROR;
+      sqlite3VdbeError(p, "cannot change from %s to %s mode",
+          sqlite3JournalModename(eOld), sqlite3JournalModename(eNew)
+      );
+      goto abort_due_to_error;
+    }
+
+    /* Prevent switching into or out of wal/wal2 mode mid-transaction */
     if( !db->autoCommit || db->nVdbeRead>1 ){
       rc = SQLITE_ERROR;
       sqlite3VdbeError(p,
@@ -101325,31 +103954,34 @@ case OP_JournalMode: {    /* out2 */
           (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of")
       );
       goto abort_due_to_error;
-    }else{
-
-      if( eOld==PAGER_JOURNALMODE_WAL ){
-        /* If leaving WAL mode, close the log file. If successful, the call
-        ** to PagerCloseWal() checkpoints and deletes the write-ahead-log
-        ** file. An EXCLUSIVE lock may still be held on the database file
-        ** after a successful return.
-        */
-        rc = sqlite3PagerCloseWal(pPager, db);
-        if( rc==SQLITE_OK ){
-          sqlite3PagerSetJournalMode(pPager, eNew);
-        }
-      }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
-        /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
-        ** as an intermediate */
-        sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
-      }
+    }
 
-      /* Open a transaction on the database file. Regardless of the journal
-      ** mode, this transaction always uses a rollback journal.
+    if( isWalMode(eOld) ){
+      /* If leaving WAL mode, close the log file. If successful, the call
+      ** to PagerCloseWal() checkpoints and deletes the write-ahead-log
+      ** file. An EXCLUSIVE lock may still be held on the database file
+      ** after a successful return.
       */
-      assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+      rc = sqlite3PagerCloseWal(pPager, db);
       if( rc==SQLITE_OK ){
-        rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1));
+        sqlite3PagerSetJournalMode(pPager, eNew);
       }
+    }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
+      /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
+      ** as an intermediate */
+      sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
+    }
+
+    /* Open a transaction on the database file. Regardless of the journal
+    ** mode, this transaction always uses a rollback journal.
+    */
+    assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+    if( rc==SQLITE_OK ){
+      /* 1==rollback, 2==wal, 3==wal2 */
+      rc = sqlite3BtreeSetVersion(pBt,
+          1 + isWalMode(eNew) + (eNew==PAGER_JOURNALMODE_WAL2)
+      );
+
     }
   }
 #endif /* ifndef SQLITE_OMIT_WAL */
@@ -101485,6 +104117,11 @@ case OP_CursorUnlock: {
 */
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( isWriteLock && db->eConcurrent && pOp->p2==1 && pOp->p1!=1 ){
+    db->eConcurrent = CONCURRENT_SCHEMA;
+  }
+#endif
   if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){
     int p1 = pOp->p1;
     assert( p1>=0 && p1<db->nDb );
@@ -127169,7 +129806,7 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){
   }
   v = sqlite3GetVdbe(pParse);
   if( !v ) return;
-  if( type!=TK_DEFERRED ){
+  if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){
     for(i=0; i<db->nDb; i++){
       int eTxnType;
       Btree *pBt = db->aDb[i].pBt;
@@ -127184,7 +129821,7 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){
       sqlite3VdbeUsesBtree(v, i);
     }
   }
-  sqlite3VdbeAddOp0(v, OP_AutoCommit);
+  sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT));
 }
 
 /*
@@ -129869,8 +132506,9 @@ static void randomFunc(
   sqlite3_value **NotUsed2
 ){
   sqlite_int64 r;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   UNUSED_PARAMETER2(NotUsed, NotUsed2);
-  sqlite3_randomness(sizeof(r), &r);
+  sqlite3FastRandomness(&db->sPrng, sizeof(r), &r);
   if( r<0 ){
     /* We need to prevent a random number of 0x8000000000000000
     ** (or -9223372036854775808) since when you do abs() of that
@@ -129896,6 +132534,7 @@ static void randomBlob(
 ){
   sqlite3_int64 n;
   unsigned char *p;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   assert( argc==1 );
   UNUSED_PARAMETER(argc);
   n = sqlite3_value_int64(argv[0]);
@@ -129904,7 +132543,7 @@ static void randomBlob(
   }
   p = contextMalloc(context, n);
   if( p ){
-    sqlite3_randomness(n, p);
+    sqlite3FastRandomness(&db->sPrng, n, p);
     sqlite3_result_blob(context, (char*)p, n, sqlite3_free);
   }
 }
@@ -139305,6 +141944,15 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ 0 },
 #endif
 #endif
+#endif
+#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+#if defined(SQLITE_ENABLE_NOOP_UPDATE)
+ {/* zName:     */ "noop_update",
+  /* ePragTyp:  */ PragTyp_FLAG,
+  /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
+  /* ColNames:  */ 0, 0,
+  /* iArg:      */ SQLITE_NoopUpdate },
+#endif
 #endif
  {/* zName:     */ "optimize",
   /* ePragTyp:  */ PragTyp_OPTIMIZE,
@@ -139521,7 +142169,7 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ SQLITE_WriteSchema|SQLITE_NoSchemaError },
 #endif
 };
-/* Number of pragmas: 68 on by default, 78 total. */
+/* Number of pragmas: 68 on by default, 79 total. */
 
 /************** End of pragma.h **********************************************/
 /************** Continuing where we left off in pragma.c *********************/
@@ -139786,7 +142434,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){
   static char * const azModeName[] = {
     "delete", "persist", "off", "truncate", "memory"
 #ifndef SQLITE_OMIT_WAL
-     , "wal"
+     , "wal", "wal2"
 #endif
   };
   assert( PAGER_JOURNALMODE_DELETE==0 );
@@ -139795,6 +142443,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){
   assert( PAGER_JOURNALMODE_TRUNCATE==3 );
   assert( PAGER_JOURNALMODE_MEMORY==4 );
   assert( PAGER_JOURNALMODE_WAL==5 );
+  assert( PAGER_JOURNALMODE_WAL2==6 );
   assert( eMode>=0 && eMode<=ArraySize(azModeName) );
 
   if( eMode==ArraySize(azModeName) ) return 0;
@@ -145960,7 +148609,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList(
       zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
       sqlite3ProgressCheck(pParse);
       if( cnt>3 ){
-        sqlite3_randomness(sizeof(cnt), &cnt);
+        sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt);
       }
     }
     pCol->zCnName = zName;
@@ -154680,6 +157329,17 @@ SQLITE_PRIVATE void sqlite3Update(
   */
   chngRowid = chngPk = 0;
   for(i=0; i<pChanges->nExpr; i++){
+#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+    if( db->flags & SQLITE_NoopUpdate ){
+      Token x;
+      sqlite3ExprDelete(db, pChanges->a[i].pExpr);
+      x.z = pChanges->a[i].zEName;
+      x.n = sqlite3Strlen30(x.z);
+      pChanges->a[i].pExpr =
+         sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0);
+      if( db->mallocFailed ) goto update_cleanup;
+    }
+#endif
     u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName);
     /* If this is an UPDATE with a FROM clause, do not resolve expressions
     ** here. The call to sqlite3Select() below will do that. */
@@ -156319,6 +158979,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
   ** is closed by the DETACH.
   */
   db->autoCommit = 1;
+  assert( db->eConcurrent==0 );
 
   if( pDb ){
     sqlite3BtreeClose(pDb->pBt);
@@ -173841,6 +176502,13 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep(
 */
 struct TrigEvent { int a; IdList * b; };
 
+/*
+** Generate a syntax error
+*/
+static void parserSyntaxError(Parse *pParse, Token *p){
+  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p);
+}
+
 struct FrameBound     { int eType; Expr *pExpr; };
 
 /*
@@ -174014,7 +176682,7 @@ static void updateDeleteLimitError(
 #define TK_TRANSACTION                     6
 #define TK_DEFERRED                        7
 #define TK_IMMEDIATE                       8
-#define TK_EXCLUSIVE                       9
+#define TK_ID                              9
 #define TK_COMMIT                         10
 #define TK_END                            11
 #define TK_ROLLBACK                       12
@@ -174047,25 +176715,25 @@ static void updateDeleteLimitError(
 #define TK_DESC                           39
 #define TK_DETACH                         40
 #define TK_EACH                           41
-#define TK_FAIL                           42
-#define TK_OR                             43
-#define TK_AND                            44
-#define TK_IS                             45
-#define TK_ISNOT                          46
-#define TK_MATCH                          47
-#define TK_LIKE_KW                        48
-#define TK_BETWEEN                        49
-#define TK_IN                             50
-#define TK_ISNULL                         51
-#define TK_NOTNULL                        52
-#define TK_NE                             53
-#define TK_EQ                             54
-#define TK_GT                             55
-#define TK_LE                             56
-#define TK_LT                             57
-#define TK_GE                             58
-#define TK_ESCAPE                         59
-#define TK_ID                             60
+#define TK_EXCLUSIVE                      42
+#define TK_FAIL                           43
+#define TK_OR                             44
+#define TK_AND                            45
+#define TK_IS                             46
+#define TK_ISNOT                          47
+#define TK_MATCH                          48
+#define TK_LIKE_KW                        49
+#define TK_BETWEEN                        50
+#define TK_IN                             51
+#define TK_ISNULL                         52
+#define TK_NOTNULL                        53
+#define TK_NE                             54
+#define TK_EQ                             55
+#define TK_GT                             56
+#define TK_LE                             57
+#define TK_LT                             58
+#define TK_GE                             59
+#define TK_ESCAPE                         60
 #define TK_COLUMNKW                       61
 #define TK_DO                             62
 #define TK_FOR                            63
@@ -174182,15 +176850,16 @@ static void updateDeleteLimitError(
 #define TK_UMINUS                         174
 #define TK_TRUTH                          175
 #define TK_REGISTER                       176
-#define TK_VECTOR                         177
-#define TK_SELECT_COLUMN                  178
-#define TK_IF_NULL_ROW                    179
-#define TK_ASTERISK                       180
-#define TK_SPAN                           181
-#define TK_ERROR                          182
-#define TK_QNUMBER                        183
-#define TK_SPACE                          184
-#define TK_ILLEGAL                        185
+#define TK_CONCURRENT                     177
+#define TK_VECTOR                         178
+#define TK_SELECT_COLUMN                  179
+#define TK_IF_NULL_ROW                    180
+#define TK_ASTERISK                       181
+#define TK_SPAN                           182
+#define TK_ERROR                          183
+#define TK_QNUMBER                        184
+#define TK_SPACE                          185
+#define TK_ILLEGAL                        186
 #endif
 /**************** End token definitions ***************************************/
 
@@ -174255,31 +176924,31 @@ static void updateDeleteLimitError(
 #endif
 /************* Begin control #defines *****************************************/
 #define YYCODETYPE unsigned short int
-#define YYNOCODE 322
+#define YYNOCODE 323
 #define YYACTIONTYPE unsigned short int
 #define YYWILDCARD 102
 #define sqlite3ParserTOKENTYPE Token
 typedef union {
   int yyinit;
   sqlite3ParserTOKENTYPE yy0;
-  ExprList* yy14;
-  With* yy59;
-  Cte* yy67;
-  Upsert* yy122;
-  IdList* yy132;
-  int yy144;
-  const char* yy168;
-  SrcList* yy203;
-  Window* yy211;
-  OnOrUsing yy269;
-  struct TrigEvent yy286;
-  struct {int value; int mask;} yy383;
-  u32 yy391;
-  TriggerStep* yy427;
-  Expr* yy454;
-  u8 yy462;
-  struct FrameBound yy509;
-  Select* yy555;
+  u32 yy9;
+  struct TrigEvent yy28;
+  With* yy125;
+  IdList* yy204;
+  struct FrameBound yy205;
+  TriggerStep* yy319;
+  const char* yy342;
+  Cte* yy361;
+  ExprList* yy402;
+  Upsert* yy403;
+  OnOrUsing yy421;
+  u8 yy444;
+  struct {int value; int mask;} yy481;
+  Window* yy483;
+  int yy502;
+  SrcList* yy563;
+  Expr* yy590;
+  Select* yy637;
 } YYMINORTYPE;
 #ifndef YYSTACKDEPTH
 #define YYSTACKDEPTH 100
@@ -174301,7 +176970,7 @@ typedef union {
 #define YYNSTATE             587
 #define YYNRULE              409
 #define YYNRULE_WITH_ACTION  344
-#define YYNTOKEN             186
+#define YYNTOKEN             187
 #define YY_MAX_SHIFT         586
 #define YY_MIN_SHIFTREDUCE   849
 #define YY_MAX_SHIFTREDUCE   1257
@@ -174310,8 +176979,8 @@ typedef union {
 #define YY_NO_ACTION         1260
 #define YY_MIN_REDUCE        1261
 #define YY_MAX_REDUCE        1669
-#define YY_MIN_DSTRCTR       205
-#define YY_MAX_DSTRCTR       319
+#define YY_MIN_DSTRCTR       206
+#define YY_MAX_DSTRCTR       320
 /************* End control #defines *******************************************/
 #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])))
 
@@ -174394,582 +177063,576 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (2207)
+#define YY_ACTTAB_COUNT (2176)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */   130,  127,  234,  282,  282, 1332,  580, 1311,  464,  289,
- /*    10 */   289,  580, 1626,  385,  580, 1332,  577,  580,  566,  417,
- /*    20 */  1304, 1546,  577,  485,  566,  528,  464,  463,  562,   82,
- /*    30 */    82,  987,  294,  379,   51,   51,  502,   61,   61,  988,
- /*    40 */    82,   82, 1581,  137,  138,   91,    7, 1232, 1232, 1067,
- /*    50 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  417,
- /*    60 */   288,  288,  182,  288,  288,  485,  540,  288,  288,  130,
- /*    70 */   127,  234,  436,  577,  529,  566,  577,  561,  566, 1294,
- /*    80 */   577,  425,  566,  137,  138,   91,  563, 1232, 1232, 1067,
- /*    90 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  296,
- /*   100 */   464,  402, 1253,  134,  134,  134,  134,  133,  133,  132,
- /*   110 */   132,  132,  131,  128,  455,   44, 1054, 1054, 1068, 1071,
- /*   120 */  1259,    1,    1,  586,    2, 1263,  585, 1178, 1263, 1178,
- /*   130 */   321,  417,  155,  321, 1588,  155,  383,  112,  502, 1345,
- /*   140 */   460,  299, 1345,  134,  134,  134,  134,  133,  133,  132,
- /*   150 */   132,  132,  131,  128,  455,  137,  138,   91, 1109, 1232,
- /*   160 */  1232, 1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,
- /*   170 */   136, 1208,  320,  571,  288,  288,  283,  288,  288,  527,
- /*   180 */   527, 1254,  139, 1545,    7,  214,  507,  577, 1173,  566,
- /*   190 */   577, 1058,  566,  136,  136,  136,  136,  129,  405,  551,
- /*   200 */   491, 1173,  245, 1572, 1173,  245,  133,  133,  132,  132,
- /*   210 */   132,  131,  128,  455,  261,  134,  134,  134,  134,  133,
- /*   220 */   133,  132,  132,  132,  131,  128,  455,  455, 1208, 1209,
- /*   230 */  1208,  130,  127,  234,  459,  417,  182,  459,  130,  127,
- /*   240 */   234,  134,  134,  134,  134,  133,  133,  132,  132,  132,
- /*   250 */   131,  128,  455,  136,  136,  136,  136,  542,  580,  137,
- /*   260 */   138,   91,  261, 1232, 1232, 1067, 1070, 1057, 1057,  135,
- /*   270 */   135,  136,  136,  136,  136,   44,  476,  350, 1208,  476,
- /*   280 */   350,   51,   51,  422,   93,  157,  134,  134,  134,  134,
- /*   290 */   133,  133,  132,  132,  132,  131,  128,  455,  166,  367,
- /*   300 */   298,  134,  134,  134,  134,  133,  133,  132,  132,  132,
- /*   310 */   131,  128,  455, 1297,  465, 1574,  427,  381,  275,  134,
- /*   320 */   134,  134,  134,  133,  133,  132,  132,  132,  131,  128,
- /*   330 */   455,  422,  320,  571, 1296, 1208, 1209, 1208,  257,  417,
- /*   340 */   487,  515,  512,  511,   94,  132,  132,  132,  131,  128,
- /*   350 */   455,  510, 1208,  552,  552,  392,  580,  388,    7,  417,
- /*   360 */   554,  229,  526,  137,  138,   91,  534, 1232, 1232, 1067,
- /*   370 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,   51,
- /*   380 */    51, 1586,  384,  137,  138,   91,  331, 1232, 1232, 1067,
- /*   390 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  320,
- /*   400 */   571,  288,  288,  320,  571, 1606,  586,    2, 1263, 1208,
- /*   410 */  1209, 1208, 1632,  321,  577,  155,  566,  580, 1515,  264,
- /*   420 */   231,  524, 1345,  134,  134,  134,  134,  133,  133,  132,
- /*   430 */   132,  132,  131,  128,  455,  523, 1515, 1517, 1337, 1337,
- /*   440 */    82,   82,  502,  134,  134,  134,  134,  133,  133,  132,
- /*   450 */   132,  132,  131,  128,  455, 1439,  257,  288,  288,  515,
- /*   460 */   512,  511,  948, 1572,  417, 1023, 1208,  947,  364,  510,
- /*   470 */   577, 1602,  566,   44,  579,  555,  555,  561, 1111, 1586,
- /*   480 */   548,  580, 1111,   40,  421,  245,  535, 1509,  137,  138,
- /*   490 */    91,  219, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,
- /*   500 */   136,  136,  136,  136,   81,   81, 1285, 1208,  417,  557,
- /*   510 */  1515,   48,  516,  452,  451,  497,  582,  459,  582,  348,
- /*   520 */    45, 1208, 1237, 1208, 1209, 1208,  432, 1239,  158,  886,
- /*   530 */   320,  571,  137,  138,   91, 1238, 1232, 1232, 1067, 1070,
- /*   540 */  1057, 1057,  135,  135,  136,  136,  136,  136,  134,  134,
- /*   550 */   134,  134,  133,  133,  132,  132,  132,  131,  128,  455,
- /*   560 */  1240,  580, 1240,  329, 1208, 1209, 1208,  391,  496,  407,
- /*   570 */  1044,  386,  493,  123,  572, 1573,    4,  381, 1208, 1209,
- /*   580 */  1208,  574,  574,  574,   82,   82,  886, 1033, 1335, 1335,
- /*   590 */   575, 1032,  134,  134,  134,  134,  133,  133,  132,  132,
- /*   600 */   132,  131,  128,  455,  288,  288, 1285, 1208,  580,  427,
- /*   610 */   580, 1572,  417,  427,  456,  382,  890,  577, 1283,  566,
- /*   620 */    46,  561,  536, 1032, 1032, 1034,  569,  130,  127,  234,
- /*   630 */   560,   82,   82,   82,   82,  483,  137,  138,   91,  466,
- /*   640 */  1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,  136,
- /*   650 */   136,  136, 1192,  491, 1510, 1044,  417,    6, 1208,   50,
- /*   660 */   883,  121,  121,  952, 1208, 1209, 1208,  362,  561,  122,
- /*   670 */   316,  456,  581,  456,  539, 1208, 1032,  443,  303,  212,
- /*   680 */   137,  138,   91,  213, 1232, 1232, 1067, 1070, 1057, 1057,
- /*   690 */   135,  135,  136,  136,  136,  136,  134,  134,  134,  134,
- /*   700 */   133,  133,  132,  132,  132,  131,  128,  455, 1032, 1032,
- /*   710 */  1034, 1035,   35,  288,  288, 1208, 1209, 1208, 1044, 1343,
- /*   720 */   537,  123,  572, 1573,    4,  381,  577, 1023,  566,  357,
- /*   730 */  1281,  360, 1208, 1209, 1208, 1033,  492, 1192,  575, 1032,
- /*   740 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
- /*   750 */   128,  455,  580,  347,  288,  288,  453,  453,  453,  975,
- /*   760 */   417, 1631,  456,  915, 1191,  288,  288,  577,  468,  566,
- /*   770 */   238, 1032, 1032, 1034,  569,   82,   82,  502,  577,  415,
- /*   780 */   566,  348,  471,  334,  137,  138,   91,  197, 1232, 1232,
- /*   790 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
- /*   800 */  1192,  532, 1173, 1044,  417, 1114, 1114,  499, 1045,  121,
- /*   810 */   121, 1208,  317,  544,  866, 1173, 1248,  122, 1173,  456,
- /*   820 */   581,  456, 1344,  198, 1032, 1208,  485,  530,  137,  138,
- /*   830 */    91,  564, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,
- /*   840 */   136,  136,  136,  136,  134,  134,  134,  134,  133,  133,
- /*   850 */   132,  132,  132,  131,  128,  455, 1032, 1032, 1034, 1035,
- /*   860 */    35, 1208,  288,  288, 1208,  481,  288,  288, 1208, 1209,
- /*   870 */  1208,  543,  485,  441,  474,  577, 1455,  566,  368,  577,
- /*   880 */  1157,  566, 1208, 1209, 1208, 1192,    5,  580,  134,  134,
- /*   890 */   134,  134,  133,  133,  132,  132,  132,  131,  128,  455,
- /*   900 */   221,  214,  302,   96, 1153, 1661,  232, 1661,  417,  396,
- /*   910 */    19,   19, 1028,  953,  410,  377, 1599, 1089, 1208, 1209,
- /*   920 */  1208, 1208, 1209, 1208, 1208,  430, 1153, 1662,  417, 1662,
- /*   930 */  1663,  403,  137,  138,   91,    3, 1232, 1232, 1067, 1070,
- /*   940 */  1057, 1057,  135,  135,  136,  136,  136,  136,  304, 1315,
- /*   950 */   518, 1208,  137,  138,   91, 1502, 1232, 1232, 1067, 1070,
- /*   960 */  1057, 1057,  135,  135,  136,  136,  136,  136,  438,  131,
- /*   970 */   128,  455,  379, 1208,  274,  291,  376,  521,  371,  520,
- /*   980 */   262, 1208, 1209, 1208, 1151,  227,  367,  452,  451, 1439,
- /*   990 */  1572, 1314,  134,  134,  134,  134,  133,  133,  132,  132,
- /*  1000 */   132,  131,  128,  455, 1572,  580, 1151,  491, 1208, 1209,
- /*  1010 */  1208,  446,  134,  134,  134,  134,  133,  133,  132,  132,
- /*  1020 */   132,  131,  128,  455,  390,  580,  489,  580,   19,   19,
- /*  1030 */  1208, 1209, 1208, 1349, 1240,  974, 1240,  578,   47,  940,
- /*  1040 */   940,  477,  417,  435, 1556,  577, 1129,  566,   19,   19,
- /*  1050 */    19,   19,   49,  340,  854,  855,  856,  111, 1372,  315,
- /*  1060 */   433,  580,  417,  437,  345,  306,  137,  138,   91,  115,
- /*  1070 */  1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,  136,
- /*  1080 */   136,  136,  580, 1313,   82,   82,  137,  138,   91,  533,
- /*  1090 */  1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,  136,
- /*  1100 */   136,  136, 1573,  222,  381,   19,   19,  305, 1130, 1173,
- /*  1110 */   402, 1152,   22,   22,  502,  337, 1573,  339,  381,  580,
- /*  1120 */   442,  449, 1173, 1131,  490, 1173,  134,  134,  134,  134,
- /*  1130 */   133,  133,  132,  132,  132,  131,  128,  455, 1132,  580,
- /*  1140 */   906,  580,  145,  145,    6,  580,  134,  134,  134,  134,
- /*  1150 */   133,  133,  132,  132,  132,  131,  128,  455,  214, 1340,
- /*  1160 */   926,  580,   19,   19,   19,   19, 1286,  423,   19,   19,
- /*  1170 */   927,  416,  519,  141,  580, 1173,  417,  206,  469,  207,
- /*  1180 */   907,  215, 1579,  556,  147,  147,    7,  227, 1173,  415,
- /*  1190 */  1254, 1173,  120,  307,  117,  307,  417,   66,   66,  338,
- /*  1200 */   137,  138,   91,  119, 1232, 1232, 1067, 1070, 1057, 1057,
- /*  1210 */   135,  135,  136,  136,  136,  136,  417,  285,  209,  973,
- /*  1220 */   137,  138,   91,  475, 1232, 1232, 1067, 1070, 1057, 1057,
- /*  1230 */   135,  135,  136,  136,  136,  136,  439,   10, 1454,  267,
- /*  1240 */   137,  126,   91, 1439, 1232, 1232, 1067, 1070, 1057, 1057,
- /*  1250 */   135,  135,  136,  136,  136,  136, 1439, 1439,  414,  413,
- /*  1260 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
- /*  1270 */   128,  455,  580,  973,  580, 1228,  502,  377, 1599, 1558,
- /*  1280 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
- /*  1290 */   128,  455,  536,  461,  580,   82,   82,   82,   82,  111,
- /*  1300 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
- /*  1310 */   128,  455,  109,  233,  434, 1580,  550,   67,   67,    7,
- /*  1320 */   417,  355,  554, 1554,  260,  259,  258,  498,  447,  573,
- /*  1330 */   423,  987,  450, 1228,  454,  549, 1211,  580,  973,  988,
- /*  1340 */   417,  479, 1453, 1578, 1184,  138,   91,    7, 1232, 1232,
- /*  1350 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
- /*  1360 */    21,   21,  267,  580,  300, 1130,   91,  233, 1232, 1232,
- /*  1370 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
- /*  1380 */  1131,  377, 1599,  161, 1577,   16,   53,   53,    7,  108,
- /*  1390 */   537,   38,  973,  125, 1211, 1132, 1184,  580, 1228,  123,
- /*  1400 */   572,  897,    4,  324,  134,  134,  134,  134,  133,  133,
- /*  1410 */   132,  132,  132,  131,  128,  455,  575,  568,  538,  580,
- /*  1420 */    68,   68,  580,   39,  134,  134,  134,  134,  133,  133,
- /*  1430 */   132,  132,  132,  131,  128,  455,  580,  160, 1575, 1227,
- /*  1440 */   456,  580,   54,   54,  580,   69,   69,  580, 1370,  580,
- /*  1450 */   424,  184,  569,  467,  297,  580, 1228,  467,  297,   70,
- /*  1460 */    70,  580,   44,  478,   71,   71,  580,   72,   72,  580,
- /*  1470 */    73,   73,   55,   55,  415,  878,  242,  580,   56,   56,
- /*  1480 */   580, 1044,  580,  482,   57,   57,  580,  121,  121,   59,
- /*  1490 */    59,   23,   60,   60,  415,  122,  319,  456,  581,  456,
- /*  1500 */    74,   74, 1032,   75,   75,   76,   76,  415,  290,   20,
- /*  1510 */    20,  108,  287,  231,  557,  123,  572,  325,    4,  320,
- /*  1520 */   571,   97,  218,  948, 1148,  328,  404,  580,  947,  580,
- /*  1530 */  1384,  428,  575,  878, 1032, 1032, 1034, 1035,   35,  293,
- /*  1540 */   538,  580, 1108,  580, 1108,    9,  580,  346,  580,  111,
- /*  1550 */    77,   77,  143,  143,  580,  205,  456,  222, 1383,  893,
- /*  1560 */   580,  905,  904, 1192,  144,  144,   78,   78,  569,   62,
- /*  1570 */    62,   79,   79,  323, 1025,  580,  266,   63,   63,  912,
- /*  1580 */   913, 1593,  546,   80,   80,  580,  375,  545,  123,  572,
- /*  1590 */   484,    4,  266,  486,  244,  266,  374, 1044,   64,   64,
- /*  1600 */   580,  470,  580,  121,  121,  575, 1561,  580,  170,  170,
- /*  1610 */   580,  122,  580,  456,  581,  456,  580,  893, 1032,  580,
- /*  1620 */   165,  580,  111,  171,  171,   87,   87,  341, 1620,  456,
- /*  1630 */    65,   65, 1534,   83,   83,  146,  146,  990,  991,   84,
- /*  1640 */    84,  569,  168,  168,  148,  148, 1096,  351, 1036,  111,
- /*  1650 */  1032, 1032, 1034, 1035,   35,  546, 1107,  580, 1107,  580,
- /*  1660 */   547,  123,  572,  508,    4,  263,  580,  365, 1533,  111,
- /*  1670 */  1044, 1092,  580,  263,  580,  494,  121,  121,  575, 1192,
- /*  1680 */   142,  142,  169,  169,  122,  580,  456,  581,  456,  162,
- /*  1690 */   162, 1032,  580,  567,  580,  152,  152,  151,  151,  352,
- /*  1700 */  1380,  978,  456,  266, 1096,  946, 1036,  125,  149,  149,
- /*  1710 */   943,  580,  125,  580,  569,  150,  150,   86,   86,  876,
- /*  1720 */   356,  159,  580, 1032, 1032, 1034, 1035,   35,  546,  945,
- /*  1730 */   580,  125,  359,  545,   88,   88,   85,   85,  361,  363,
- /*  1740 */  1328, 1312,  370, 1044,  380,   52,   52,  503, 1393,  121,
- /*  1750 */   121, 1438, 1192,   58,   58, 1366, 1378,  122, 1443,  456,
- /*  1760 */   581,  456, 1293,  167, 1032, 1284,  280, 1272, 1271, 1273,
- /*  1770 */  1613, 1363,  312,  313,   12,  314,  401, 1425,  224,  333,
- /*  1780 */   295,  237,  336,  343,  344, 1430,  301,  349,  488,  228,
- /*  1790 */  1375, 1311, 1376, 1374, 1429,  408, 1032, 1032, 1034, 1035,
- /*  1800 */    35, 1605, 1196,  458,  513,  373,  292, 1506,  210, 1505,
- /*  1810 */  1373,  400,  400,  399,  277,  397,  211,  570,  863, 1616,
- /*  1820 */  1248,  123,  572,  395,    4, 1192,  223,  270, 1553, 1551,
- /*  1830 */  1245,  239,  186,  327,  426,   96,  195,  220,  575,  235,
- /*  1840 */   140,  326, 1420,  557,  332, 1511,  180,  188,   92, 1413,
- /*  1850 */   335,   95, 1426,   13,  472,  190,  191,  192,  193,  473,
- /*  1860 */   506,  247,  456,  109, 1432,   14, 1431, 1434,  406,  199,
- /*  1870 */   480,  495,  241,  409,  569,  251, 1500,  102,  501,  253,
- /*  1880 */   175,  281,  203,   43,  504,  358,  411, 1522,  254,  255,
- /*  1890 */   354, 1274,  522, 1331,  440, 1330, 1329, 1322,  104,  897,
- /*  1900 */   240, 1321,  229, 1044,  444,  445, 1630,  448,  531,  121,
- /*  1910 */   121, 1598,  268, 1301,  412,  310, 1300,  122, 1629,  456,
- /*  1920 */   581,  456,  372, 1299, 1032,  311, 1628,  378, 1584,   90,
- /*  1930 */   572,  269,    4, 1583, 1398,  418,   11,  116, 1397, 1487,
- /*  1940 */   320,  571,  389,  559,  318,  110,  575,  541,  216, 1354,
- /*  1950 */   387,   42, 1353,  583, 1202,  394, 1032, 1032, 1034, 1035,
- /*  1960 */    35,  276,  393,  278,  462,  279,  584, 1269, 1264,  172,
- /*  1970 */   456,  419, 1538,  420,  185,  173, 1539, 1537, 1536,  156,
- /*  1980 */   308,  850,  569,  457,  225, 1192,  226,  174,  217,  322,
- /*  1990 */   236,   89,  154, 1106, 1104,  330,  187,  176, 1227,  189,
- /*  2000 */   342,  929,  243,  246, 1120,  194,  177,  429,  178,  431,
- /*  2010 */    98, 1044,   99,  196,  100,  101,  179,  121,  121, 1123,
- /*  2020 */   248,  249, 1119,  163,   24,  122,  353,  456,  581,  456,
- /*  2030 */  1196,  458, 1032,  250,  292,  266, 1112,  200, 1242,  400,
- /*  2040 */   400,  399,  277,  397,  201,  500,  863,  252,  292,   15,
- /*  2050 */   865,  558,  505,  400,  400,  399,  277,  397,  256,  239,
- /*  2060 */   863,  327,  374,  202, 1032, 1032, 1034, 1035,   35,  326,
- /*  2070 */   103,   25,   26,  239,  509,  327,  366,  514,  369,  895,
- /*  2080 */   517,  105,  309,  326,  908,  164,   27,  106,  181,  525,
- /*  2090 */   107, 1189, 1073, 1192, 1159,   17, 1158,  230,  284,  286,
- /*  2100 */   241,  204,  982,  125,  265, 1175,   28,  976,  175, 1177,
- /*  2110 */  1179,   43, 1183, 1164,  241,   29,   30,   31,    8,   32,
- /*  2120 */  1182,   41,  175,  208,  553,   43,  111,   33,  240, 1087,
- /*  2130 */   113,  114, 1074, 1072, 1076,   34, 1077,  565, 1128,  118,
- /*  2140 */   271,   36,  240,   18,  939, 1037,  877,  272,  124,   37,
- /*  2150 */   398, 1198,  153,  576,  273, 1621,  183, 1197, 1260, 1260,
- /*  2160 */  1260, 1260, 1260,  418, 1260, 1260, 1260, 1260,  320,  571,
- /*  2170 */  1260, 1260, 1260, 1260, 1260, 1260, 1260,  418, 1260, 1260,
- /*  2180 */  1260, 1260,  320,  571, 1260, 1260, 1260, 1260, 1260, 1260,
- /*  2190 */  1260, 1260,  462, 1260, 1260, 1260, 1260, 1260, 1260, 1260,
- /*  2200 */  1260, 1260, 1260, 1260, 1260, 1260,  462,
+ /*     0 */  1332,  580, 1311,  580,  379,  580, 1285,  282,  282, 1626,
+ /*    10 */  1332, 1259,    1,    1,  586,    2, 1263, 1304, 1283,  417,
+ /*    20 */   577,  321,  566,  155,   81,   81,   51,   51,   51,   51,
+ /*    30 */  1345,  987,  130,  127,  234, 1153, 1661, 1294, 1661,  988,
+ /*    40 */   130,  127,  234,  436,  137,  138,   91,  534, 1232, 1232,
+ /*    50 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
+ /*    60 */  1208, 1546,  580,   38, 1285,  288,  288, 1606,  586,    2,
+ /*    70 */  1263,  285, 1208,  973,  582,  321,  582,  155,  577,  502,
+ /*    80 */   566,  214,  288,  288, 1345,   82,   82,  391,  136,  136,
+ /*    90 */   136,  136,  129,  245,  416,  577,   39,  566, 1337, 1337,
+ /*   100 */   264,  231,  283,  134,  134,  134,  134,  133,  133,  132,
+ /*   110 */   132,  132,  131,  128,  455, 1151,  307, 1581,  307,  288,
+ /*   120 */   288,    7,  561,  417, 1545,  459, 1586,  384, 1586,  548,
+ /*   130 */  1208,  535,  577, 1572,  566,  134,  134,  134,  134,  133,
+ /*   140 */   133,  132,  132,  132,  131,  128,  455,  245,  137,  138,
+ /*   150 */    91,  455, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,
+ /*   160 */   136,  136,  136,  136,  130,  127,  234,  233, 1208, 1209,
+ /*   170 */  1208,  257,  953, 1297,  515,  512,  511,  182,  441,  459,
+ /*   180 */  1208, 1209, 1208,  368,  510,  132,  132,  132,  131,  128,
+ /*   190 */   455, 1178,  973, 1178,  134,  134,  134,  134,  133,  133,
+ /*   200 */   132,  132,  132,  131,  128,  455,  362,  134,  134,  134,
+ /*   210 */   134,  133,  133,  132,  132,  132,  131,  128,  455,  133,
+ /*   220 */   133,  132,  132,  132,  131,  128,  455,  417,  452,  451,
+ /*   230 */    44,  289,  289,  112,  485, 1023,  261, 1237, 1208, 1209,
+ /*   240 */  1208,  111, 1239,   44,  577, 1574,  566,  381,  580,  329,
+ /*   250 */  1238,  502,  137,  138,   91,  518, 1232, 1232, 1067, 1070,
+ /*   260 */  1057, 1057,  135,  135,  136,  136,  136,  136,  357,  465,
+ /*   270 */   360,   19,   19,  438,  392, 1240,  388, 1240,  139,  274,
+ /*   280 */   291,  376,  521,  371,  520,  262,  430,  320,  571,  348,
+ /*   290 */  1296,  367, 1173, 1173,  527,  527, 1509, 1023,  417,    7,
+ /*   300 */   320,  571,  487,  544,  422, 1173, 1173,  294, 1173, 1173,
+ /*   310 */   296,  134,  134,  134,  134,  133,  133,  132,  132,  132,
+ /*   320 */   131,  128,  455,  137,  138,   91, 1632, 1232, 1232, 1067,
+ /*   330 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  417,
+ /*   340 */  1510, 1455,  288,  288,   94,  257,  214,   93,  515,  512,
+ /*   350 */   511,  348,  471,  334,  396,  577,  385,  566,  510,  410,
+ /*   360 */   182,  543,  386,  502,  137,  138,   91,  417, 1232, 1232,
+ /*   370 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
+ /*   380 */   377, 1599,  134,  134,  134,  134,  133,  133,  132,  132,
+ /*   390 */   132,  131,  128,  455,   91,  421, 1232, 1232, 1067, 1070,
+ /*   400 */  1057, 1057,  135,  135,  136,  136,  136,  136,  425, 1602,
+ /*   410 */   417, 1208,  130,  127,  234,   44,  579, 1208,  130,  127,
+ /*   420 */   234,  476,  350,  134,  134,  134,  134,  133,  133,  132,
+ /*   430 */   132,  132,  131,  128,  455,  137,  138,   91,  427, 1232,
+ /*   440 */  1232, 1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,
+ /*   450 */   136,  134,  134,  134,  134,  133,  133,  132,  132,  132,
+ /*   460 */   131,  128,  455,  580,  452,  451, 1439,  460, 1208,  580,
+ /*   470 */   157, 1208,  320,  571,  562,   45,  554,  552,  552,  496,
+ /*   480 */   528,   46,    7,  493,  197,  275,   82,   82, 1054, 1054,
+ /*   490 */  1068, 1071,   61,   61,  134,  134,  134,  134,  133,  133,
+ /*   500 */   132,  132,  132,  131,  128,  455,  468, 1208,  331,  288,
+ /*   510 */   288, 1240,  580, 1240,  417,  288,  288,  415,  364, 1208,
+ /*   520 */  1209, 1208,  577,  561,  566, 1208, 1209, 1208,  577,  529,
+ /*   530 */   566,  382,  563,  580,  417,   51,   51,  432,  516,  137,
+ /*   540 */   138,   91,  219, 1232, 1232, 1067, 1070, 1057, 1057,  135,
+ /*   550 */   135,  136,  136,  136,  136,  379,   82,   82,  539,  137,
+ /*   560 */   138,   91, 1058, 1232, 1232, 1067, 1070, 1057, 1057,  135,
+ /*   570 */   135,  136,  136,  136,  136, 1173, 1208, 1209, 1208, 1208,
+ /*   580 */  1209, 1208,  536,  108,  320,  571,  551,  580, 1173,  987,
+ /*   590 */   580, 1173, 1575,  540,  446,  407, 1208,  988,  134,  134,
+ /*   600 */   134,  134,  133,  133,  132,  132,  132,  131,  128,  455,
+ /*   610 */    82,   82,  538,   82,   82, 1208, 1209, 1208,  134,  134,
+ /*   620 */   134,  134,  133,  133,  132,  132,  132,  131,  128,  455,
+ /*   630 */   288,  288,  550, 1153, 1662, 1588, 1662,  383,  417,  574,
+ /*   640 */   574,  574,  890,  577,  542,  566,  578,  561,  940,  940,
+ /*   650 */   561,  549,  131,  128,  455, 1208,  560,  238,  417,  443,
+ /*   660 */  1184,  483,  883,  137,  138,   91,  303, 1232, 1232, 1067,
+ /*   670 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  108,
+ /*   680 */   537,  464,  111,  137,  138,   91,  533, 1232, 1232, 1067,
+ /*   690 */  1070, 1057, 1057,  135,  135,  136,  136,  136,  136,  464,
+ /*   700 */   463,  288,  288, 1248, 1208, 1209, 1208, 1439,  538,   22,
+ /*   710 */    22,  427, 1184, 1151,  577, 1208,  566,  580,  232, 1343,
+ /*   720 */  1572,  554,  134,  134,  134,  134,  133,  133,  132,  132,
+ /*   730 */   132,  131,  128,  455,  580, 1281,  580,  229,  526,   96,
+ /*   740 */    82,   82,  134,  134,  134,  134,  133,  133,  132,  132,
+ /*   750 */   132,  131,  128,  455,  288,  288,  580,   19,   19,   19,
+ /*   760 */    19,    6,  417, 1208, 1209, 1208, 1191,  577,   48,  566,
+ /*   770 */   288,  288,  435,  464,  437,  320,  571,  316,  433,  145,
+ /*   780 */   145,  212,  417,  577,  897,  566, 1045,  137,  138,   91,
+ /*   790 */   975, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,
+ /*   800 */   136,  136,  136,  580,  390,  580,  523,  137,  138,   91,
+ /*   810 */     6, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,
+ /*   820 */   136,  136,  136, 1208, 1209, 1208,   19,   19,   19,   19,
+ /*   830 */   427,  469, 1573,  948,  381,  209,  555,  555,  947,  580,
+ /*   840 */   475,  306,  415,  442,  530, 1556,  134,  134,  134,  134,
+ /*   850 */   133,  133,  132,  132,  132,  131,  128,  455,  580, 1558,
+ /*   860 */   580, 1579,   82,   82,  580,    7,  134,  134,  134,  134,
+ /*   870 */   133,  133,  132,  132,  132,  131,  128,  455, 1208,  288,
+ /*   880 */   288,   19,   19,   19,   19,  491,  417,   19,   19,  492,
+ /*   890 */  1028,  198,  577, 1111,  566,  461,  206, 1111,  207,  317,
+ /*   900 */   213, 1208,  556, 1631,  580,  915,  417,  136,  136,  136,
+ /*   910 */   136,  137,  138,   91,   40, 1232, 1232, 1067, 1070, 1057,
+ /*   920 */  1057,  135,  135,  136,  136,  136,  136,  147,  147, 1515,
+ /*   930 */   497,  137,  138,   91, 1228, 1232, 1232, 1067, 1070, 1057,
+ /*   940 */  1057,  135,  135,  136,  136,  136,  136, 1515, 1517,  267,
+ /*   950 */   340, 1130,  141, 1439,  134,  134,  134,  134,  133,  133,
+ /*   960 */   132,  132,  132,  131,  128,  455, 1131, 1109, 1572,  536,
+ /*   970 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
+ /*   980 */   128,  455, 1132,  477, 1208,  298, 1208, 1209, 1208, 1208,
+ /*   990 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
+ /*  1000 */   128,  455,    5,  926,  580,  485,  345, 1208, 1044, 1208,
+ /*  1010 */  1209, 1208,  337,  927,  339,  478,   50,  580,  125,  417,
+ /*  1020 */     3, 1515, 1349, 1129,  434, 1033,  415,   66,   66, 1032,
+ /*  1030 */   453,  453,  453,  290,  577,  866,  566,  414,  413,  417,
+ /*  1040 */    67,   67, 1572, 1228,  137,  138,   91,  115, 1232, 1232,
+ /*  1050 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
+ /*  1060 */   580, 1032, 1032, 1034,  137,  138,   91,  537, 1232, 1232,
+ /*  1070 */  1067, 1070, 1057, 1057,  135,  135,  136,  136,  136,  136,
+ /*  1080 */  1573,  299,  381,   82,   82,  476,  350, 1228, 1554,  485,
+ /*  1090 */    47, 1192, 1208, 1209, 1208,  474,  338, 1208, 1209, 1208,
+ /*  1100 */    10,  564,  267,  134,  134,  134,  134,  133,  133,  132,
+ /*  1110 */   132,  132,  131,  128,  455, 1208, 1209, 1208, 1572,  974,
+ /*  1120 */   449,  507,  580,  134,  134,  134,  134,  133,  133,  132,
+ /*  1130 */   132,  132,  131,  128,  455,  580,  288,  288, 1089,  557,
+ /*  1140 */   580, 1114, 1114,  499,  580,   21,   21,  580,  485,  577,
+ /*  1150 */   261,  566,  417, 1620, 1573, 1439,  381,  215,   82,   82,
+ /*  1160 */   260,  259,  258,   82,   82,  302,   49,   53,   53, 1211,
+ /*  1170 */    68,   68,  417, 1335, 1335, 1454, 1502,  137,  138,   91,
+ /*  1180 */   119, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,
+ /*  1190 */   136,  136,  136,  906,  324,  450, 1228,  137,  138,   91,
+ /*  1200 */   454, 1232, 1232, 1067, 1070, 1057, 1057,  135,  135,  136,
+ /*  1210 */   136,  136,  136, 1453,  377, 1599, 1663,  403,  422,  854,
+ /*  1220 */   855,  856, 1286,  423,  304,  519,  498,  973, 1372,  315,
+ /*  1230 */  1573,  580,  381,  907,  367,  502,  134,  134,  134,  134,
+ /*  1240 */   133,  133,  132,  132,  132,  131,  128,  455,  580,  227,
+ /*  1250 */   580,  491,  377, 1599,   54,   54,  134,  134,  134,  134,
+ /*  1260 */   133,  133,  132,  132,  132,  131,  128,  455,  580, 1173,
+ /*  1270 */   580,   69,   69,   70,   70,  580,  417,  214, 1211,  227,
+ /*  1280 */  1344,  557, 1173,  325,  878, 1173,  573,  423, 1439,  439,
+ /*  1290 */   405,   71,   71,   72,   72,  580,  417,  160,   73,   73,
+ /*  1300 */   158,  137,  126,   91,  502, 1232, 1232, 1067, 1070, 1057,
+ /*  1310 */  1057,  135,  135,  136,  136,  136,  136,  580,   55,   55,
+ /*  1320 */   886,  233,  138,   91,  580, 1232, 1232, 1067, 1070, 1057,
+ /*  1330 */  1057,  135,  135,  136,  136,  136,  136,  402, 1253,  491,
+ /*  1340 */    56,   56, 1130,  424,  184,  456,  973,   57,   57, 1340,
+ /*  1350 */   502,  347,  580,  466,  580,  123,  572, 1131,    4,  447,
+ /*  1360 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
+ /*  1370 */   128,  455,  575, 1132,  109,   59,   59,   60,   60,  580,
+ /*  1380 */   134,  134,  134,  134,  133,  133,  132,  132,  132,  131,
+ /*  1390 */   128,  455,  580,  878,  568,  355,  580,  222,  585,  580,
+ /*  1400 */  1263,  580,   74,   74,  120,  321,  117,  155,  569,  402,
+ /*  1410 */  1152,  161, 1044,   16, 1345,   75,   75, 1254,   44,   76,
+ /*  1420 */    76,  482,   20,   20,   77,   77, 1580, 1578,  886, 1033,
+ /*  1430 */     7,    7,  415, 1032, 1577,  489,  580, 1044,    7,  305,
+ /*  1440 */  1227,  467,  948,  121,  121,  467,  580,  947,  297,  288,
+ /*  1450 */   288,  122,  297,  456,  581,  456,  580,  319, 1032,  143,
+ /*  1460 */   143,  456,  577,  166,  566, 1032, 1032, 1034,  415,  144,
+ /*  1470 */   144,  123,  572,  242,    4,  320,  571,  245,  328,   78,
+ /*  1480 */    78,  287,  231,  293,  481,  479,  456,  580,  575, 1254,
+ /*  1490 */  1032, 1032, 1034, 1035,   35, 1192,  123,  572,  580,    4,
+ /*  1500 */    97,  218,    9,  580, 1148,  580,  404,  580,  300,  459,
+ /*  1510 */    62,   62,  580,  575,   23,  375,  580,  323,  580,  221,
+ /*  1520 */  1192,   79,   79,  580,  569,  374,   63,   63,   80,   80,
+ /*  1530 */    64,   64,  580,  490, 1384,  170,  170,  532,  546,  171,
+ /*  1540 */   171,   87,   87,  545,  912,  913,   65,   65,  346,  569,
+ /*  1550 */   111, 1383,  580, 1044,  428,   83,   83,  580,  222,  121,
+ /*  1560 */   121,  470, 1025,  546,  266,  905,  904,  122,  547,  456,
+ /*  1570 */   581,  456, 1561,  893, 1032,  146,  146,  456, 1044, 1096,
+ /*  1580 */    84,   84,  990,  991,  121,  121,  524,  123,  572,  341,
+ /*  1590 */     4,  244,  122, 1534,  456,  581,  456,  580,  484, 1032,
+ /*  1600 */   266,  580,  456,  580,  575, 1036, 1032, 1032, 1034, 1035,
+ /*  1610 */    35,  205,  123,  572,  580,    4, 1157, 1108,  580, 1108,
+ /*  1620 */   168,  168, 1533,  494,  148,  148,  142,  142,  486,  575,
+ /*  1630 */   266, 1032, 1032, 1034, 1035,   35, 1192,  169,  169,  292,
+ /*  1640 */   569,  162,  162,  352,  400,  400,  399,  277,  397,  580,
+ /*  1650 */   351,  863,  111, 1380,  546,  580,  508,  580,  263,  545,
+ /*  1660 */   365, 1192,  111,  356,  239,  569,  327,  359,  580, 1044,
+ /*  1670 */   361,  363,  152,  152,  326,  121,  121, 1328,  151,  151,
+ /*  1680 */   149,  149,  893,  122, 1315,  456,  581,  456, 1096, 1314,
+ /*  1690 */  1032,  150,  150, 1313, 1044,  580,  165, 1092,  111,  263,
+ /*  1700 */   121,  121,  952, 1312,  978,  241,  266,  580,  122,  370,
+ /*  1710 */   456,  581,  456,  175, 1036, 1032,   43,  380,   86,   86,
+ /*  1720 */  1370, 1393, 1032, 1032, 1034, 1035,   35, 1605, 1196,  458,
+ /*  1730 */    88,   88,  292,  240,  946, 1438,  125,  400,  400,  399,
+ /*  1740 */   277,  397,  580,  943,  863,  125, 1366, 1032, 1032, 1034,
+ /*  1750 */  1035,   35, 1192, 1593, 1107,  580, 1107,  239,  876,  327,
+ /*  1760 */   159,  945, 1378,  125,  580,   85,   85,  326,  418,  567,
+ /*  1770 */   503, 1443, 1293,  320,  571, 1284, 1272, 1192,   52,   52,
+ /*  1780 */  1271, 1273, 1613,  280,  401,  167, 1363,   58,   58,   12,
+ /*  1790 */   312,  313,  314, 1425,  224,  237,  295,  462,  241,  333,
+ /*  1800 */   336,  343,  344,  301,  349,  488,  175, 1375,  456,   43,
+ /*  1810 */  1430,  513, 1429,  228, 1376, 1311,  408,  210,  123,  572,
+ /*  1820 */   373,    4, 1506, 1505, 1374, 1373,  240, 1616,  456,  570,
+ /*  1830 */   211,  395, 1248,  270, 1553,  575, 1245,  223,   90,  572,
+ /*  1840 */  1551,    4,  426,  186,   96, 1511,  220,   92,  235, 1426,
+ /*  1850 */    95,  195,  140,  557,  332,  575,   13,  180, 1420,  188,
+ /*  1860 */  1413,  418,  335,  472,  190,  191,  320,  571,  473,  192,
+ /*  1870 */   193,  569,  506,  247,  109, 1434,  406,  199,  495, 1432,
+ /*  1880 */   251,  102, 1431,  480,  409,   14,  501, 1522,  281,  354,
+ /*  1890 */   462,  569, 1500,  203,  253,  522,  358,  254,  504, 1274,
+ /*  1900 */  1044,  255, 1331,  411, 1330, 1329,  121,  121,  440,  104,
+ /*  1910 */  1301,  412, 1322, 1630,  122, 1629,  456,  581,  456,  897,
+ /*  1920 */  1044, 1032,  372, 1300, 1299, 1628,  121,  121, 1321,  229,
+ /*  1930 */  1598,  444,  531,  310,  122,  445,  456,  581,  456,  311,
+ /*  1940 */   559, 1032,  378,  268,  269,  448, 1398,   11, 1487,  389,
+ /*  1950 */   116,  318,  110, 1032, 1032, 1034, 1035,   35, 1354,  387,
+ /*  1960 */   558, 1353,  216, 1584, 1583,  541, 1397,  393,   42,  394,
+ /*  1970 */   583, 1202,  276, 1032, 1032, 1034, 1035,   35, 1196,  458,
+ /*  1980 */   278,  279,  292, 1192,  584, 1538,  172,  400,  400,  399,
+ /*  1990 */   277,  397,  156, 1539,  863, 1269, 1264, 1537, 1536,  308,
+ /*  2000 */   456,  225,  173, 1192,  226,  174,  850,  239,  457,  327,
+ /*  2010 */   123,  572,   89,    4,  217,  322,  419,  326,  185,  420,
+ /*  2020 */   154,  236, 1106, 1104,  330,  187,  176,  575, 1227,  189,
+ /*  2030 */   929,  243,  342,  246, 1120,  194,  177,  178,  429,   98,
+ /*  2040 */    99,  196,  100,  101,  179,  431, 1123,  248,  241, 1119,
+ /*  2050 */   249,  163,   24,  250,  266,  353,  175, 1242, 1112,   43,
+ /*  2060 */   500,  252,  200,  569,  201,   15,  374,  865,  505,  509,
+ /*  2070 */   256,  895,  202,  103,   25,   26,  240,  366,  164,  514,
+ /*  2080 */   369,  105,  309,  517, 1189,  908,  106,  525,  107, 1073,
+ /*  2090 */  1159,   17, 1044, 1158,   27,  181,  230,  284,  121,  121,
+ /*  2100 */   286,  204,  265,  976,   28,  125,  122,  982,  456,  581,
+ /*  2110 */   456,  418,   29, 1032, 1175,   30,  320,  571,   31, 1179,
+ /*  2120 */     8, 1177, 1182,   32, 1164,   41,  553,   33,   34,  208,
+ /*  2130 */   111, 1087, 1074, 1072, 1076, 1128,  271,  113,  565,  114,
+ /*  2140 */   462,  118, 1077,   36,   18, 1032, 1032, 1034, 1035,   35,
+ /*  2150 */  1037,  877, 1183,  939,  124,   37,  398,  272,  153,  576,
+ /*  2160 */   273,  183, 1621, 1198, 1197, 1260, 1260, 1260, 1260, 1260,
+ /*  2170 */  1260, 1260, 1260, 1260, 1260, 1192,
 };
 static const YYCODETYPE yy_lookahead[] = {
- /*     0 */   276,  277,  278,  240,  241,  224,  194,  226,  194,  240,
- /*    10 */   241,  194,  216,  220,  194,  234,  253,  194,  255,   19,
- /*    20 */   224,  297,  253,  194,  255,  205,  212,  213,  205,  217,
- /*    30 */   218,   31,  205,  194,  217,  218,  194,  217,  218,   39,
- /*    40 */   217,  218,  312,   43,   44,   45,  316,   47,   48,   49,
- /*    50 */    50,   51,   52,   53,   54,   55,   56,   57,   58,   19,
- /*    60 */   240,  241,  194,  240,  241,  194,  254,  240,  241,  276,
- /*    70 */   277,  278,  233,  253,  254,  255,  253,  254,  255,  217,
- /*    80 */   253,  239,  255,   43,   44,   45,  263,   47,   48,   49,
- /*    90 */    50,   51,   52,   53,   54,   55,   56,   57,   58,  270,
- /*   100 */   286,   22,   23,  103,  104,  105,  106,  107,  108,  109,
- /*   110 */   110,  111,  112,  113,  114,   82,   47,   48,   49,   50,
- /*   120 */   186,  187,  188,  189,  190,  191,  189,   87,  191,   89,
- /*   130 */   196,   19,  198,  196,  317,  198,  319,   25,  194,  205,
- /*   140 */   298,  270,  205,  103,  104,  105,  106,  107,  108,  109,
- /*   150 */   110,  111,  112,  113,  114,   43,   44,   45,   11,   47,
- /*   160 */    48,   49,   50,   51,   52,   53,   54,   55,   56,   57,
- /*   170 */    58,   60,  139,  140,  240,  241,  214,  240,  241,  311,
- /*   180 */   312,  102,   70,  239,  316,  194,   19,  253,   77,  255,
- /*   190 */   253,  122,  255,   55,   56,   57,   58,   59,  207,   88,
- /*   200 */   194,   90,  268,  194,   93,  268,  107,  108,  109,  110,
- /*   210 */   111,  112,  113,  114,   47,  103,  104,  105,  106,  107,
- /*   220 */   108,  109,  110,  111,  112,  113,  114,  114,  117,  118,
- /*   230 */   119,  276,  277,  278,  300,   19,  194,  300,  276,  277,
- /*   240 */   278,  103,  104,  105,  106,  107,  108,  109,  110,  111,
- /*   250 */   112,  113,  114,   55,   56,   57,   58,  146,  194,   43,
- /*   260 */    44,   45,   47,   47,   48,   49,   50,   51,   52,   53,
- /*   270 */    54,   55,   56,   57,   58,   82,  129,  130,   60,  129,
- /*   280 */   130,  217,  218,  116,   68,   25,  103,  104,  105,  106,
- /*   290 */   107,  108,  109,  110,  111,  112,  113,  114,   23,  132,
- /*   300 */   294,  103,  104,  105,  106,  107,  108,  109,  110,  111,
- /*   310 */   112,  113,  114,  217,  121,  306,  194,  308,   26,  103,
- /*   320 */   104,  105,  106,  107,  108,  109,  110,  111,  112,  113,
- /*   330 */   114,  116,  139,  140,  217,  117,  118,  119,  120,   19,
- /*   340 */   194,  123,  124,  125,   24,  109,  110,  111,  112,  113,
- /*   350 */   114,  133,   60,  311,  312,  250,  194,  252,  316,   19,
- /*   360 */   194,  166,  167,   43,   44,   45,  205,   47,   48,   49,
- /*   370 */    50,   51,   52,   53,   54,   55,   56,   57,   58,  217,
- /*   380 */   218,  317,  318,   43,   44,   45,  264,   47,   48,   49,
- /*   390 */    50,   51,   52,   53,   54,   55,   56,   57,   58,  139,
- /*   400 */   140,  240,  241,  139,  140,  188,  189,  190,  191,  117,
- /*   410 */   118,  119,  231,  196,  253,  198,  255,  194,  194,  258,
- /*   420 */   259,  146,  205,  103,  104,  105,  106,  107,  108,  109,
- /*   430 */   110,  111,  112,  113,  114,  109,  212,  213,  236,  237,
- /*   440 */   217,  218,  194,  103,  104,  105,  106,  107,  108,  109,
- /*   450 */   110,  111,  112,  113,  114,  194,  120,  240,  241,  123,
- /*   460 */   124,  125,  136,  194,   19,   74,   60,  141,   23,  133,
- /*   470 */   253,  194,  255,   82,  194,  309,  310,  254,   29,  317,
- /*   480 */   318,  194,   33,   22,  199,  268,  263,  239,   43,   44,
- /*   490 */    45,  151,   47,   48,   49,   50,   51,   52,   53,   54,
- /*   500 */    55,   56,   57,   58,  217,  218,  194,   60,   19,  146,
- /*   510 */   286,  242,   23,  107,  108,   66,  204,  300,  206,  128,
- /*   520 */    73,   60,  116,  117,  118,  119,  265,  121,  165,   60,
- /*   530 */   139,  140,   43,   44,   45,  129,   47,   48,   49,   50,
- /*   540 */    51,   52,   53,   54,   55,   56,   57,   58,  103,  104,
- /*   550 */   105,  106,  107,  108,  109,  110,  111,  112,  113,  114,
- /*   560 */   154,  194,  156,  194,  117,  118,  119,  280,  283,  205,
- /*   570 */   101,  220,  287,   19,   20,  306,   22,  308,  117,  118,
- /*   580 */   119,  211,  212,  213,  217,  218,  117,  118,  236,  237,
- /*   590 */    36,  122,  103,  104,  105,  106,  107,  108,  109,  110,
- /*   600 */   111,  112,  113,  114,  240,  241,  194,   60,  194,  194,
- /*   610 */   194,  194,   19,  194,   60,  194,   23,  253,  206,  255,
- /*   620 */    73,  254,   19,  154,  155,  156,   72,  276,  277,  278,
- /*   630 */   263,  217,  218,  217,  218,  271,   43,   44,   45,  271,
- /*   640 */    47,   48,   49,   50,   51,   52,   53,   54,   55,   56,
- /*   650 */    57,   58,  183,  194,  285,  101,   19,  214,   60,  242,
- /*   660 */    23,  107,  108,  109,  117,  118,  119,   16,  254,  115,
- /*   670 */   254,  117,  118,  119,  194,   60,  122,  263,  205,  264,
- /*   680 */    43,   44,   45,  264,   47,   48,   49,   50,   51,   52,
- /*   690 */    53,   54,   55,   56,   57,   58,  103,  104,  105,  106,
- /*   700 */   107,  108,  109,  110,  111,  112,  113,  114,  154,  155,
- /*   710 */   156,  157,  158,  240,  241,  117,  118,  119,  101,  205,
- /*   720 */   117,   19,   20,  306,   22,  308,  253,   74,  255,   78,
- /*   730 */   205,   80,  117,  118,  119,  118,  293,  183,   36,  122,
- /*   740 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
- /*   750 */   113,  114,  194,  294,  240,  241,  211,  212,  213,  144,
- /*   760 */    19,   23,   60,   25,   23,  240,  241,  253,  245,  255,
- /*   770 */    15,  154,  155,  156,   72,  217,  218,  194,  253,  256,
- /*   780 */   255,  128,  129,  130,   43,   44,   45,   22,   47,   48,
- /*   790 */    49,   50,   51,   52,   53,   54,   55,   56,   57,   58,
- /*   800 */   183,   19,   77,  101,   19,  128,  129,  130,   23,  107,
- /*   810 */   108,   60,  254,   88,   21,   90,   61,  115,   93,  117,
- /*   820 */   118,  119,  239,   22,  122,   60,  194,  205,   43,   44,
- /*   830 */    45,  205,   47,   48,   49,   50,   51,   52,   53,   54,
- /*   840 */    55,   56,   57,   58,  103,  104,  105,  106,  107,  108,
- /*   850 */   109,  110,  111,  112,  113,  114,  154,  155,  156,  157,
- /*   860 */   158,   60,  240,  241,   60,  116,  240,  241,  117,  118,
- /*   870 */   119,  146,  194,   19,   81,  253,  275,  255,   24,  253,
- /*   880 */    98,  255,  117,  118,  119,  183,   22,  194,  103,  104,
- /*   890 */   105,  106,  107,  108,  109,  110,  111,  112,  113,  114,
- /*   900 */   151,  194,  270,  152,   22,   23,  194,   25,   19,  202,
- /*   910 */   217,  218,   23,  109,  207,  314,  315,  124,  117,  118,
- /*   920 */   119,  117,  118,  119,   60,  232,   22,   23,   19,   25,
- /*   930 */   303,  304,   43,   44,   45,   22,   47,   48,   49,   50,
- /*   940 */    51,   52,   53,   54,   55,   56,   57,   58,  270,  227,
- /*   950 */    96,   60,   43,   44,   45,  162,   47,   48,   49,   50,
- /*   960 */    51,   52,   53,   54,   55,   56,   57,   58,  114,  112,
- /*   970 */   113,  114,  194,   60,  120,  121,  122,  123,  124,  125,
- /*   980 */   126,  117,  118,  119,  102,   25,  132,  107,  108,  194,
- /*   990 */   194,  227,  103,  104,  105,  106,  107,  108,  109,  110,
- /*  1000 */   111,  112,  113,  114,  194,  194,  102,  194,  117,  118,
- /*  1010 */   119,  233,  103,  104,  105,  106,  107,  108,  109,  110,
- /*  1020 */   111,  112,  113,  114,  194,  194,   19,  194,  217,  218,
- /*  1030 */   117,  118,  119,  241,  154,  144,  156,  135,  242,  137,
- /*  1040 */   138,  130,   19,  232,  194,  253,   23,  255,  217,  218,
- /*  1050 */   217,  218,  242,   16,    7,    8,    9,   25,  261,  262,
- /*  1060 */   265,  194,   19,  232,  153,  232,   43,   44,   45,  160,
- /*  1070 */    47,   48,   49,   50,   51,   52,   53,   54,   55,   56,
- /*  1080 */    57,   58,  194,  227,  217,  218,   43,   44,   45,  194,
- /*  1090 */    47,   48,   49,   50,   51,   52,   53,   54,   55,   56,
- /*  1100 */    57,   58,  306,  143,  308,  217,  218,  294,   12,   77,
- /*  1110 */    22,   23,  217,  218,  194,   78,  306,   80,  308,  194,
- /*  1120 */   232,  254,   90,   27,  117,   93,  103,  104,  105,  106,
- /*  1130 */   107,  108,  109,  110,  111,  112,  113,  114,   42,  194,
- /*  1140 */    35,  194,  217,  218,  214,  194,  103,  104,  105,  106,
- /*  1150 */   107,  108,  109,  110,  111,  112,  113,  114,  194,  239,
- /*  1160 */    64,  194,  217,  218,  217,  218,  209,  210,  217,  218,
- /*  1170 */    74,  207,   67,   22,  194,   77,   19,  232,  245,  232,
- /*  1180 */    75,   24,  312,  232,  217,  218,  316,   25,   90,  256,
- /*  1190 */   102,   93,  159,  229,  161,  231,   19,  217,  218,  162,
- /*  1200 */    43,   44,   45,  160,   47,   48,   49,   50,   51,   52,
- /*  1210 */    53,   54,   55,   56,   57,   58,   19,   23,  288,   25,
- /*  1220 */    43,   44,   45,  293,   47,   48,   49,   50,   51,   52,
- /*  1230 */    53,   54,   55,   56,   57,   58,  131,   22,  275,   24,
- /*  1240 */    43,   44,   45,  194,   47,   48,   49,   50,   51,   52,
- /*  1250 */    53,   54,   55,   56,   57,   58,  194,  194,  107,  108,
- /*  1260 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
- /*  1270 */   113,  114,  194,   25,  194,   60,  194,  314,  315,  194,
- /*  1280 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
- /*  1290 */   113,  114,   19,  194,  194,  217,  218,  217,  218,   25,
- /*  1300 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
- /*  1310 */   113,  114,  150,  119,  265,  312,   67,  217,  218,  316,
- /*  1320 */    19,  239,  194,  194,  128,  129,  130,  265,  265,  209,
- /*  1330 */   210,   31,  254,  118,  254,   86,   60,  194,  144,   39,
- /*  1340 */    19,  130,  275,  312,   95,   44,   45,  316,   47,   48,
- /*  1350 */    49,   50,   51,   52,   53,   54,   55,   56,   57,   58,
- /*  1360 */   217,  218,   24,  194,  153,   12,   45,  119,   47,   48,
- /*  1370 */    49,   50,   51,   52,   53,   54,   55,   56,   57,   58,
- /*  1380 */    27,  314,  315,   22,  312,   24,  217,  218,  316,  116,
- /*  1390 */   117,   22,  144,   25,  118,   42,  147,  194,   60,   19,
- /*  1400 */    20,  127,   22,  194,  103,  104,  105,  106,  107,  108,
- /*  1410 */   109,  110,  111,  112,  113,  114,   36,   64,  145,  194,
- /*  1420 */   217,  218,  194,   54,  103,  104,  105,  106,  107,  108,
- /*  1430 */   109,  110,  111,  112,  113,  114,  194,   22,  310,   25,
- /*  1440 */    60,  194,  217,  218,  194,  217,  218,  194,  260,  194,
- /*  1450 */   301,  302,   72,  262,  262,  194,  118,  266,  266,  217,
- /*  1460 */   218,  194,   82,  245,  217,  218,  194,  217,  218,  194,
- /*  1470 */   217,  218,  217,  218,  256,   60,   24,  194,  217,  218,
- /*  1480 */   194,  101,  194,  245,  217,  218,  194,  107,  108,  217,
- /*  1490 */   218,   22,  217,  218,  256,  115,  245,  117,  118,  119,
- /*  1500 */   217,  218,  122,  217,  218,  217,  218,  256,   22,  217,
- /*  1510 */   218,  116,  258,  259,  146,   19,   20,  194,   22,  139,
- /*  1520 */   140,  150,  151,  136,   23,  194,   25,  194,  141,  194,
- /*  1530 */   194,   62,   36,  118,  154,  155,  156,  157,  158,  100,
- /*  1540 */   145,  194,  154,  194,  156,   49,  194,   23,  194,   25,
- /*  1550 */   217,  218,  217,  218,  194,  257,   60,  143,  194,   60,
- /*  1560 */   194,  121,  122,  183,  217,  218,  217,  218,   72,  217,
- /*  1570 */   218,  217,  218,  134,   23,  194,   25,  217,  218,    7,
- /*  1580 */     8,  321,   86,  217,  218,  194,  122,   91,   19,   20,
- /*  1590 */    23,   22,   25,   23,  142,   25,  132,  101,  217,  218,
- /*  1600 */   194,  194,  194,  107,  108,   36,  194,  194,  217,  218,
- /*  1610 */   194,  115,  194,  117,  118,  119,  194,  118,  122,  194,
- /*  1620 */    23,  194,   25,  217,  218,  217,  218,  194,  142,   60,
- /*  1630 */   217,  218,  194,  217,  218,  217,  218,   84,   85,  217,
- /*  1640 */   218,   72,  217,  218,  217,  218,   60,   23,   60,   25,
- /*  1650 */   154,  155,  156,  157,  158,   86,  154,  194,  156,  194,
- /*  1660 */    91,   19,   20,   23,   22,   25,  194,   23,  194,   25,
- /*  1670 */   101,   23,  194,   25,  194,  194,  107,  108,   36,  183,
- /*  1680 */   217,  218,  217,  218,  115,  194,  117,  118,  119,  217,
- /*  1690 */   218,  122,  194,  237,  194,  217,  218,  217,  218,  194,
- /*  1700 */   194,   23,   60,   25,  118,   23,  118,   25,  217,  218,
- /*  1710 */    23,  194,   25,  194,   72,  217,  218,  217,  218,   23,
- /*  1720 */   194,   25,  194,  154,  155,  156,  157,  158,   86,   23,
- /*  1730 */   194,   25,  194,   91,  217,  218,  217,  218,  194,  194,
- /*  1740 */   194,  194,  194,  101,  194,  217,  218,  290,  194,  107,
- /*  1750 */   108,  194,  183,  217,  218,  194,  194,  115,  194,  117,
- /*  1760 */   118,  119,  194,  243,  122,  194,  289,  194,  194,  194,
- /*  1770 */   194,  257,  257,  257,  244,  257,  192,  273,  215,  269,
- /*  1780 */   246,  299,  269,  295,  247,  273,  247,  246,  295,  230,
- /*  1790 */   261,  226,  261,  261,  273,  273,  154,  155,  156,  157,
- /*  1800 */   158,    0,    1,    2,  221,  220,    5,  220,  250,  220,
- /*  1810 */   261,   10,   11,   12,   13,   14,  250,  282,   17,  197,
- /*  1820 */    61,   19,   20,  246,   22,  183,  244,  142,  201,  201,
- /*  1830 */    38,   30,  299,   32,  201,  152,   22,  151,   36,  299,
- /*  1840 */   149,   40,  251,  146,  250,  285,   43,  235,  296,  251,
- /*  1850 */   250,  296,  274,  272,   18,  238,  238,  238,  238,  201,
- /*  1860 */    18,  200,   60,  150,  274,  272,  274,  235,  247,  235,
- /*  1870 */   247,  201,   71,  247,   72,  200,  247,  159,   63,  200,
- /*  1880 */    79,  201,   22,   82,  222,  201,  222,  292,  200,  200,
- /*  1890 */   291,  201,  116,  219,   65,  219,  219,  228,   22,  127,
- /*  1900 */    99,  228,  166,  101,   24,  114,  225,   83,  307,  107,
- /*  1910 */   108,  315,  201,  219,  222,  284,  221,  115,  225,  117,
- /*  1920 */   118,  119,  219,  219,  122,  284,  219,  222,  320,   19,
- /*  1930 */    20,   92,   22,  320,  267,  134,   22,  159,  267,  279,
- /*  1940 */   139,  140,  201,  141,  281,  148,   36,  147,  249,  251,
- /*  1950 */   250,   25,  251,  203,   13,  247,  154,  155,  156,  157,
- /*  1960 */   158,  195,  248,  195,  163,    6,  193,  193,  193,  208,
- /*  1970 */    60,  305,  214,  305,  302,  208,  214,  214,  214,  223,
- /*  1980 */   223,    4,   72,    3,  215,  183,  215,  208,   22,  164,
- /*  1990 */    15,  214,   16,   23,   23,  140,  152,  131,   25,  143,
- /*  2000 */    16,   20,   24,  145,    1,  143,  131,   62,  131,   37,
- /*  2010 */    54,  101,   54,  152,   54,   54,  131,  107,  108,  117,
- /*  2020 */    34,  142,    1,    5,   22,  115,  162,  117,  118,  119,
- /*  2030 */     1,    2,  122,  116,    5,   25,   69,   69,   76,   10,
- /*  2040 */    11,   12,   13,   14,  116,   41,   17,  142,    5,   24,
- /*  2050 */    20,  141,   19,   10,   11,   12,   13,   14,  126,   30,
- /*  2060 */    17,   32,  132,   22,  154,  155,  156,  157,  158,   40,
- /*  2070 */    22,   22,   22,   30,   68,   32,   23,   68,   24,   60,
- /*  2080 */    97,   22,   68,   40,   28,   23,   34,  150,   37,   22,
- /*  2090 */    25,   23,   23,  183,   23,   22,   98,  142,   23,   23,
- /*  2100 */    71,   22,  117,   25,   34,   89,   34,  144,   79,   87,
- /*  2110 */    76,   82,   76,   23,   71,   34,   34,   34,   44,   34,
- /*  2120 */    94,   22,   79,   25,   24,   82,   25,   34,   99,   23,
- /*  2130 */   143,  143,   23,   23,   23,   22,   11,   25,   23,   25,
- /*  2140 */    22,   22,   99,   22,  136,   23,   23,  142,   22,   22,
- /*  2150 */    15,    1,   23,   25,  142,  142,   25,    1,  322,  322,
- /*  2160 */   322,  322,  322,  134,  322,  322,  322,  322,  139,  140,
- /*  2170 */   322,  322,  322,  322,  322,  322,  322,  134,  322,  322,
- /*  2180 */   322,  322,  139,  140,  322,  322,  322,  322,  322,  322,
- /*  2190 */   322,  322,  163,  322,  322,  322,  322,  322,  322,  322,
- /*  2200 */   322,  322,  322,  322,  322,  322,  163,  322,  322,  322,
- /*  2210 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2220 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2230 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2240 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2250 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2260 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2270 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2280 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2290 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2300 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2310 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2320 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2330 */   322,  322,  322,  322,  322,  322,  322,  322,  322,  322,
- /*  2340 */   322,  322,  322,  322,  186,  186,  186,  186,  186,  186,
- /*  2350 */   186,  186,  186,  186,  186,  186,  186,  186,  186,  186,
- /*  2360 */   186,  186,  186,  186,  186,  186,  186,  186,  186,  186,
- /*  2370 */   186,  186,  186,  186,  186,  186,  186,  186,  186,  186,
- /*  2380 */   186,  186,  186,  186,  186,  186,  186,  186,  186,  186,
- /*  2390 */   186,  186,  186,
+ /*     0 */   225,  195,  227,  195,  195,  195,  195,  241,  242,  217,
+ /*    10 */   235,  187,  188,  189,  190,  191,  192,  225,  207,   19,
+ /*    20 */   254,  197,  256,  199,  218,  219,  218,  219,  218,  219,
+ /*    30 */   206,   31,  277,  278,  279,   22,   23,  218,   25,   39,
+ /*    40 */   277,  278,  279,  234,   44,   45,   46,  206,   48,   49,
+ /*    50 */    50,   51,   52,   53,   54,   55,   56,   57,   58,   59,
+ /*    60 */     9,  298,  195,   22,  195,  241,  242,  189,  190,  191,
+ /*    70 */   192,   23,    9,   25,  205,  197,  207,  199,  254,  195,
+ /*    80 */   256,  195,  241,  242,  206,  218,  219,  281,   56,   57,
+ /*    90 */    58,   59,   60,  269,  208,  254,   55,  256,  237,  238,
+ /*   100 */   259,  260,  215,  103,  104,  105,  106,  107,  108,  109,
+ /*   110 */   110,  111,  112,  113,  114,  102,  230,  313,  232,  241,
+ /*   120 */   242,  317,  255,   19,  240,  301,  318,  319,  318,  319,
+ /*   130 */     9,  264,  254,  195,  256,  103,  104,  105,  106,  107,
+ /*   140 */   108,  109,  110,  111,  112,  113,  114,  269,   44,   45,
+ /*   150 */    46,  114,   48,   49,   50,   51,   52,   53,   54,   55,
+ /*   160 */    56,   57,   58,   59,  277,  278,  279,  119,  117,  118,
+ /*   170 */   119,  120,  109,  218,  123,  124,  125,  195,   19,  301,
+ /*   180 */   117,  118,  119,   24,  133,  109,  110,  111,  112,  113,
+ /*   190 */   114,   87,  144,   89,  103,  104,  105,  106,  107,  108,
+ /*   200 */   109,  110,  111,  112,  113,  114,   16,  103,  104,  105,
+ /*   210 */   106,  107,  108,  109,  110,  111,  112,  113,  114,  107,
+ /*   220 */   108,  109,  110,  111,  112,  113,  114,   19,  107,  108,
+ /*   230 */    82,  241,  242,   25,  195,   74,   48,  116,  117,  118,
+ /*   240 */   119,   25,  121,   82,  254,  307,  256,  309,  195,  195,
+ /*   250 */   129,  195,   44,   45,   46,   96,   48,   49,   50,   51,
+ /*   260 */    52,   53,   54,   55,   56,   57,   58,   59,   78,  121,
+ /*   270 */    80,  218,  219,  114,  251,  154,  253,  156,   70,  120,
+ /*   280 */   121,  122,  123,  124,  125,  126,  233,  139,  140,  128,
+ /*   290 */   218,  132,   77,   77,  312,  313,  240,   74,   19,  317,
+ /*   300 */   139,  140,  195,   88,  116,   90,   90,  206,   93,   93,
+ /*   310 */   271,  103,  104,  105,  106,  107,  108,  109,  110,  111,
+ /*   320 */   112,  113,  114,   44,   45,   46,  232,   48,   49,   50,
+ /*   330 */    51,   52,   53,   54,   55,   56,   57,   58,   59,   19,
+ /*   340 */   286,  276,  241,  242,   24,  120,  195,   68,  123,  124,
+ /*   350 */   125,  128,  129,  130,  203,  254,  221,  256,  133,  208,
+ /*   360 */   195,  146,  221,  195,   44,   45,   46,   19,   48,   49,
+ /*   370 */    50,   51,   52,   53,   54,   55,   56,   57,   58,   59,
+ /*   380 */   315,  316,  103,  104,  105,  106,  107,  108,  109,  110,
+ /*   390 */   111,  112,  113,  114,   46,  200,   48,   49,   50,   51,
+ /*   400 */    52,   53,   54,   55,   56,   57,   58,   59,  240,  195,
+ /*   410 */    19,    9,  277,  278,  279,   82,  195,    9,  277,  278,
+ /*   420 */   279,  129,  130,  103,  104,  105,  106,  107,  108,  109,
+ /*   430 */   110,  111,  112,  113,  114,   44,   45,   46,  195,   48,
+ /*   440 */    49,   50,   51,   52,   53,   54,   55,   56,   57,   58,
+ /*   450 */    59,  103,  104,  105,  106,  107,  108,  109,  110,  111,
+ /*   460 */   112,  113,  114,  195,  107,  108,  195,  299,    9,  195,
+ /*   470 */    25,    9,  139,  140,  206,   73,  195,  312,  313,  284,
+ /*   480 */   206,   73,  317,  288,   22,   26,  218,  219,   48,   49,
+ /*   490 */    50,   51,  218,  219,  103,  104,  105,  106,  107,  108,
+ /*   500 */   109,  110,  111,  112,  113,  114,  246,    9,  265,  241,
+ /*   510 */   242,  154,  195,  156,   19,  241,  242,  257,   23,  117,
+ /*   520 */   118,  119,  254,  255,  256,  117,  118,  119,  254,  255,
+ /*   530 */   256,  195,  264,  195,   19,  218,  219,  266,   23,   44,
+ /*   540 */    45,   46,  151,   48,   49,   50,   51,   52,   53,   54,
+ /*   550 */    55,   56,   57,   58,   59,  195,  218,  219,  195,   44,
+ /*   560 */    45,   46,  122,   48,   49,   50,   51,   52,   53,   54,
+ /*   570 */    55,   56,   57,   58,   59,   77,  117,  118,  119,  117,
+ /*   580 */   118,  119,   19,  116,  139,  140,   88,  195,   90,   31,
+ /*   590 */   195,   93,  311,  255,  234,  206,    9,   39,  103,  104,
+ /*   600 */   105,  106,  107,  108,  109,  110,  111,  112,  113,  114,
+ /*   610 */   218,  219,  145,  218,  219,  117,  118,  119,  103,  104,
+ /*   620 */   105,  106,  107,  108,  109,  110,  111,  112,  113,  114,
+ /*   630 */   241,  242,   67,   22,   23,  318,   25,  320,   19,  212,
+ /*   640 */   213,  214,   23,  254,  146,  256,  135,  255,  137,  138,
+ /*   650 */   255,   86,  112,  113,  114,    9,  264,   15,   19,  264,
+ /*   660 */    95,  272,   23,   44,   45,   46,  206,   48,   49,   50,
+ /*   670 */    51,   52,   53,   54,   55,   56,   57,   58,   59,  116,
+ /*   680 */   117,  195,   25,   44,   45,   46,  195,   48,   49,   50,
+ /*   690 */    51,   52,   53,   54,   55,   56,   57,   58,   59,  213,
+ /*   700 */   214,  241,  242,   61,  117,  118,  119,  195,  145,  218,
+ /*   710 */   219,  195,  147,  102,  254,    9,  256,  195,  195,  206,
+ /*   720 */   195,  195,  103,  104,  105,  106,  107,  108,  109,  110,
+ /*   730 */   111,  112,  113,  114,  195,  206,  195,  166,  167,  152,
+ /*   740 */   218,  219,  103,  104,  105,  106,  107,  108,  109,  110,
+ /*   750 */   111,  112,  113,  114,  241,  242,  195,  218,  219,  218,
+ /*   760 */   219,  215,   19,  117,  118,  119,   23,  254,  243,  256,
+ /*   770 */   241,  242,  233,  287,  233,  139,  140,  255,  266,  218,
+ /*   780 */   219,  265,   19,  254,  127,  256,   23,   44,   45,   46,
+ /*   790 */   144,   48,   49,   50,   51,   52,   53,   54,   55,   56,
+ /*   800 */    57,   58,   59,  195,  195,  195,  109,   44,   45,   46,
+ /*   810 */   215,   48,   49,   50,   51,   52,   53,   54,   55,   56,
+ /*   820 */    57,   58,   59,  117,  118,  119,  218,  219,  218,  219,
+ /*   830 */   195,  246,  307,  136,  309,  289,  310,  311,  141,  195,
+ /*   840 */   294,  233,  257,  233,  206,  195,  103,  104,  105,  106,
+ /*   850 */   107,  108,  109,  110,  111,  112,  113,  114,  195,  195,
+ /*   860 */   195,  313,  218,  219,  195,  317,  103,  104,  105,  106,
+ /*   870 */   107,  108,  109,  110,  111,  112,  113,  114,    9,  241,
+ /*   880 */   242,  218,  219,  218,  219,  195,   19,  218,  219,  294,
+ /*   890 */    23,   22,  254,   29,  256,  195,  233,   33,  233,  255,
+ /*   900 */   265,    9,  233,   23,  195,   25,   19,   56,   57,   58,
+ /*   910 */    59,   44,   45,   46,   22,   48,   49,   50,   51,   52,
+ /*   920 */    53,   54,   55,   56,   57,   58,   59,  218,  219,  195,
+ /*   930 */    66,   44,   45,   46,    9,   48,   49,   50,   51,   52,
+ /*   940 */    53,   54,   55,   56,   57,   58,   59,  213,  214,   24,
+ /*   950 */    16,   12,   22,  195,  103,  104,  105,  106,  107,  108,
+ /*   960 */   109,  110,  111,  112,  113,  114,   27,   11,  195,   19,
+ /*   970 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
+ /*   980 */   113,  114,   43,  130,    9,  295,  117,  118,  119,    9,
+ /*   990 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
+ /*  1000 */   113,  114,   22,   64,  195,  195,  153,    9,  101,  117,
+ /*  1010 */   118,  119,   78,   74,   80,  246,  243,  195,   25,   19,
+ /*  1020 */    22,  287,  242,   23,  266,  118,  257,  218,  219,  122,
+ /*  1030 */   212,  213,  214,   22,  254,   21,  256,  107,  108,   19,
+ /*  1040 */   218,  219,  195,  118,   44,   45,   46,  160,   48,   49,
+ /*  1050 */    50,   51,   52,   53,   54,   55,   56,   57,   58,   59,
+ /*  1060 */   195,  154,  155,  156,   44,   45,   46,  117,   48,   49,
+ /*  1070 */    50,   51,   52,   53,   54,   55,   56,   57,   58,   59,
+ /*  1080 */   307,  271,  309,  218,  219,  129,  130,    9,  195,  195,
+ /*  1090 */   243,  184,  117,  118,  119,   81,  162,  117,  118,  119,
+ /*  1100 */    22,  206,   24,  103,  104,  105,  106,  107,  108,  109,
+ /*  1110 */   110,  111,  112,  113,  114,  117,  118,  119,  195,  144,
+ /*  1120 */   255,   19,  195,  103,  104,  105,  106,  107,  108,  109,
+ /*  1130 */   110,  111,  112,  113,  114,  195,  241,  242,  124,  146,
+ /*  1140 */   195,  128,  129,  130,  195,  218,  219,  195,  195,  254,
+ /*  1150 */    48,  256,   19,  142,  307,  195,  309,   24,  218,  219,
+ /*  1160 */   128,  129,  130,  218,  219,  271,  243,  218,  219,    9,
+ /*  1170 */   218,  219,   19,  237,  238,  276,  162,   44,   45,   46,
+ /*  1180 */   160,   48,   49,   50,   51,   52,   53,   54,   55,   56,
+ /*  1190 */    57,   58,   59,   35,  195,  255,  118,   44,   45,   46,
+ /*  1200 */   255,   48,   49,   50,   51,   52,   53,   54,   55,   56,
+ /*  1210 */    57,   58,   59,  276,  315,  316,  304,  305,  116,    7,
+ /*  1220 */     8,    9,  210,  211,  271,   67,  266,   25,  262,  263,
+ /*  1230 */   307,  195,  309,   75,  132,  195,  103,  104,  105,  106,
+ /*  1240 */   107,  108,  109,  110,  111,  112,  113,  114,  195,   25,
+ /*  1250 */   195,  195,  315,  316,  218,  219,  103,  104,  105,  106,
+ /*  1260 */   107,  108,  109,  110,  111,  112,  113,  114,  195,   77,
+ /*  1270 */   195,  218,  219,  218,  219,  195,   19,  195,  118,   25,
+ /*  1280 */   240,  146,   90,  195,    9,   93,  210,  211,  195,  131,
+ /*  1290 */   208,  218,  219,  218,  219,  195,   19,   22,  218,  219,
+ /*  1300 */   165,   44,   45,   46,  195,   48,   49,   50,   51,   52,
+ /*  1310 */    53,   54,   55,   56,   57,   58,   59,  195,  218,  219,
+ /*  1320 */     9,  119,   45,   46,  195,   48,   49,   50,   51,   52,
+ /*  1330 */    53,   54,   55,   56,   57,   58,   59,   22,   23,  195,
+ /*  1340 */   218,  219,   12,  302,  303,    9,  144,  218,  219,  240,
+ /*  1350 */   195,  295,  195,  272,  195,   19,   20,   27,   22,  266,
+ /*  1360 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
+ /*  1370 */   113,  114,   36,   43,  150,  218,  219,  218,  219,  195,
+ /*  1380 */   103,  104,  105,  106,  107,  108,  109,  110,  111,  112,
+ /*  1390 */   113,  114,  195,  118,   64,  240,  195,  143,  190,  195,
+ /*  1400 */   192,  195,  218,  219,  159,  197,  161,  199,   72,   22,
+ /*  1410 */    23,   22,  101,   24,  206,  218,  219,  102,   82,  218,
+ /*  1420 */   219,  246,  218,  219,  218,  219,  313,  313,  117,  118,
+ /*  1430 */   317,  317,  257,  122,  313,   19,  195,  101,  317,  295,
+ /*  1440 */    25,  263,  136,  107,  108,  267,  195,  141,  263,  241,
+ /*  1450 */   242,  115,  267,  117,  118,  119,  195,  246,  122,  218,
+ /*  1460 */   219,    9,  254,   23,  256,  154,  155,  156,  257,  218,
+ /*  1470 */   219,   19,   20,   24,   22,  139,  140,  269,  195,  218,
+ /*  1480 */   219,  259,  260,  100,  116,  130,    9,  195,   36,  102,
+ /*  1490 */   154,  155,  156,  157,  158,  184,   19,   20,  195,   22,
+ /*  1500 */   150,  151,   50,  195,   23,  195,   25,  195,  153,  301,
+ /*  1510 */   218,  219,  195,   36,   22,  122,  195,  134,  195,  151,
+ /*  1520 */   184,  218,  219,  195,   72,  132,  218,  219,  218,  219,
+ /*  1530 */   218,  219,  195,  117,  195,  218,  219,   19,   86,  218,
+ /*  1540 */   219,  218,  219,   91,    7,    8,  218,  219,   23,   72,
+ /*  1550 */    25,  195,  195,  101,   62,  218,  219,  195,  143,  107,
+ /*  1560 */   108,  195,   23,   86,   25,  121,  122,  115,   91,  117,
+ /*  1570 */   118,  119,  195,    9,  122,  218,  219,    9,  101,    9,
+ /*  1580 */   218,  219,   84,   85,  107,  108,  146,   19,   20,  195,
+ /*  1590 */    22,  142,  115,  195,  117,  118,  119,  195,   23,  122,
+ /*  1600 */    25,  195,    9,  195,   36,    9,  154,  155,  156,  157,
+ /*  1610 */   158,  258,   19,   20,  195,   22,   98,  154,  195,  156,
+ /*  1620 */   218,  219,  195,  195,  218,  219,  218,  219,   23,   36,
+ /*  1630 */    25,  154,  155,  156,  157,  158,  184,  218,  219,    5,
+ /*  1640 */    72,  218,  219,  195,   10,   11,   12,   13,   14,  195,
+ /*  1650 */    23,   17,   25,  195,   86,  195,   23,  195,   25,   91,
+ /*  1660 */    23,  184,   25,  195,   30,   72,   32,  195,  195,  101,
+ /*  1670 */   195,  195,  218,  219,   40,  107,  108,  195,  218,  219,
+ /*  1680 */   218,  219,  118,  115,  228,  117,  118,  119,  118,  228,
+ /*  1690 */   122,  218,  219,  228,  101,  195,   23,   23,   25,   25,
+ /*  1700 */   107,  108,  109,  195,   23,   71,   25,  195,  115,  195,
+ /*  1710 */   117,  118,  119,   79,  118,  122,   82,  195,  218,  219,
+ /*  1720 */   261,  195,  154,  155,  156,  157,  158,    0,    1,    2,
+ /*  1730 */   218,  219,    5,   99,   23,  195,   25,   10,   11,   12,
+ /*  1740 */    13,   14,  195,   23,   17,   25,  195,  154,  155,  156,
+ /*  1750 */   157,  158,  184,  322,  154,  195,  156,   30,   23,   32,
+ /*  1760 */    25,   23,  195,   25,  195,  218,  219,   40,  134,  238,
+ /*  1770 */   291,  195,  195,  139,  140,  195,  195,  184,  218,  219,
+ /*  1780 */   195,  195,  195,  290,  193,  244,  258,  218,  219,  245,
+ /*  1790 */   258,  258,  258,  274,  216,  300,  247,  163,   71,  270,
+ /*  1800 */   270,  296,  248,  248,  247,  296,   79,  262,    9,   82,
+ /*  1810 */   274,  222,  274,  231,  262,  227,  274,  251,   19,   20,
+ /*  1820 */   221,   22,  221,  221,  262,  262,   99,  198,    9,  283,
+ /*  1830 */   251,  247,   61,  142,  202,   36,   38,  245,   19,   20,
+ /*  1840 */   202,   22,  202,  300,  152,  286,  151,  297,  300,  275,
+ /*  1850 */   297,   22,  149,  146,  251,   36,  273,   44,  252,  236,
+ /*  1860 */   252,  134,  251,   18,  239,  239,  139,  140,  202,  239,
+ /*  1870 */   239,   72,   18,  201,  150,  236,  248,  236,  202,  275,
+ /*  1880 */   201,  159,  275,  248,  248,  273,   63,  293,  202,  292,
+ /*  1890 */   163,   72,  248,   22,  201,  116,  202,  201,  223,  202,
+ /*  1900 */   101,  201,  220,  223,  220,  220,  107,  108,   65,   22,
+ /*  1910 */   220,  223,  229,  226,  115,  226,  117,  118,  119,  127,
+ /*  1920 */   101,  122,  220,  222,  220,  220,  107,  108,  229,  166,
+ /*  1930 */   316,   24,  308,  285,  115,  114,  117,  118,  119,  285,
+ /*  1940 */   141,  122,  223,  202,   92,   83,  268,   22,  280,  202,
+ /*  1950 */   159,  282,  148,  154,  155,  156,  157,  158,  252,  251,
+ /*  1960 */   141,  252,  250,  321,  321,  147,  268,  249,   25,  248,
+ /*  1970 */   204,   13,  196,  154,  155,  156,  157,  158,    1,    2,
+ /*  1980 */   196,    6,    5,  184,  194,  215,  209,   10,   11,   12,
+ /*  1990 */    13,   14,  224,  215,   17,  194,  194,  215,  215,  224,
+ /*  2000 */     9,  216,  209,  184,  216,  209,    4,   30,    3,   32,
+ /*  2010 */    19,   20,  215,   22,   22,  164,  306,   40,  303,  306,
+ /*  2020 */    16,   15,   23,   23,  140,  152,  131,   36,   25,  143,
+ /*  2030 */    20,   24,   16,  145,    1,  143,  131,  131,   62,   55,
+ /*  2040 */    55,  152,   55,   55,  131,   37,  117,   34,   71,    1,
+ /*  2050 */   142,    5,   22,  116,   25,  162,   79,   76,   69,   82,
+ /*  2060 */    41,  142,   69,   72,  116,   24,  132,   20,   19,   68,
+ /*  2070 */   126,    9,   22,   22,   22,   22,   99,   23,   23,   68,
+ /*  2080 */    24,   22,   68,   97,   23,   28,  150,   22,   25,   23,
+ /*  2090 */    23,   22,  101,   98,   34,   37,  142,   23,  107,  108,
+ /*  2100 */    23,   22,   34,  144,   34,   25,  115,  117,  117,  118,
+ /*  2110 */   119,  134,   34,  122,   89,   34,  139,  140,   34,   76,
+ /*  2120 */    45,   87,   94,   34,   23,   22,   24,   34,   22,   25,
+ /*  2130 */    25,   23,   23,   23,   23,   23,   22,  143,   25,  143,
+ /*  2140 */   163,   25,   11,   22,   22,  154,  155,  156,  157,  158,
+ /*  2150 */    23,   23,   76,  136,   22,   22,   15,  142,   23,   25,
+ /*  2160 */   142,   25,  142,    1,    1,  323,  323,  323,  323,  323,
+ /*  2170 */   323,  323,  323,  323,  323,  184,  323,  323,  323,  323,
+ /*  2180 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2190 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2200 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2210 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2220 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2230 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2240 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2250 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2260 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2270 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2280 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2290 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2300 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2310 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2320 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2330 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2340 */   323,  323,  323,  323,  323,  323,  323,  323,  323,  323,
+ /*  2350 */   323,  323,  187,  187,  187,  187,  187,  187,  187,  187,
+ /*  2360 */   187,  187,  187,
 };
 #define YY_SHIFT_COUNT    (586)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (2156)
+#define YY_SHIFT_MAX      (2163)
 static const unsigned short int yy_shift_ofst[] = {
- /*     0 */  2029, 1801, 2043, 1380, 1380,   33,  391, 1496, 1569, 1642,
- /*    10 */   702,  702,  702,  193,   33,   33,   33,   33,   33,    0,
- /*    20 */     0,  216, 1177,  702,  702,  702,  702,  702,  702,  702,
- /*    30 */   702,  702,  702,  702,  702,  702,  702,  702,  406,  406,
- /*    40 */   111,  111,  218,  447,  547,  598,  598,  260,  260,  260,
- /*    50 */   260,   40,  112,  320,  340,  445,  489,  593,  637,  741,
- /*    60 */   785,  889,  909, 1023, 1043, 1157, 1177, 1177, 1177, 1177,
- /*    70 */  1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177,
- /*    80 */  1177, 1177, 1177, 1177, 1197, 1177, 1301, 1321, 1321,  554,
- /*    90 */  1802, 1910,  702,  702,  702,  702,  702,  702,  702,  702,
- /*   100 */   702,  702,  702,  702,  702,  702,  702,  702,  702,  702,
- /*   110 */   702,  702,  702,  702,  702,  702,  702,  702,  702,  702,
- /*   120 */   702,  702,  702,  702,  702,  702,  702,  702,  702,  702,
- /*   130 */   702,  702,  702,  702,  702,  702,  702,  702,  702,  702,
- /*   140 */   702,  702,  138,  198,  198,  198,  198,  198,  198,  198,
- /*   150 */   183,   99,  236,  292,  598,  793,  167,  598,  598,  880,
- /*   160 */   880,  598,  857,  150,  195,  195,  195,  264,  113,  113,
- /*   170 */  2207, 2207,  854,  854,  854,  751,  765,  765,  765,  765,
- /*   180 */  1096, 1096,  725,  292,  882,  904,  598,  598,  598,  598,
- /*   190 */   598,  598,  598,  598,  598,  598,  598,  598,  598,  598,
- /*   200 */   598,  598,  598,  598,  598, 1273, 1032, 1032,  598,  147,
- /*   210 */  1098, 1098,  603,  603, 1276, 1276,  363, 2207, 2207, 2207,
- /*   220 */  2207, 2207, 2207, 2207,  469,  617,  617,  801,  336,  461,
- /*   230 */   804,  864,  615,  891,  913,  598,  598,  598,  598,  598,
- /*   240 */   598,  598,  598,  598,  598,  653,  598,  598,  598,  598,
- /*   250 */   598,  598,  598,  598,  598,  598,  598,  598, 1105, 1105,
- /*   260 */  1105,  598,  598,  598, 1194,  598,  598,  598, 1215, 1249,
- /*   270 */   598, 1353,  598,  598,  598,  598,  598,  598,  598,  598,
- /*   280 */   677,  449,  902, 1338, 1338, 1338, 1338, 1248,  902,  902,
- /*   290 */   326, 1151, 1047,  755,  749, 1371,  960, 1371, 1007, 1162,
- /*   300 */   749,  749, 1162,  749,  960, 1007, 1274,  738,  215, 1300,
- /*   310 */  1300, 1300, 1395, 1395, 1395, 1395, 1368, 1368, 1033, 1414,
- /*   320 */  1387, 1361, 1759, 1759, 1685, 1685, 1792, 1792, 1685, 1683,
- /*   330 */  1686, 1814, 1691, 1697, 1803, 1691, 1697, 1836, 1836, 1836,
- /*   340 */  1836, 1685, 1842, 1713, 1686, 1686, 1713, 1814, 1803, 1713,
- /*   350 */  1803, 1713, 1685, 1842, 1718, 1815, 1685, 1842, 1860, 1685,
- /*   360 */  1842, 1685, 1842, 1860, 1776, 1776, 1776, 1829, 1876, 1876,
- /*   370 */  1860, 1776, 1772, 1776, 1829, 1776, 1776, 1736, 1880, 1791,
- /*   380 */  1791, 1860, 1685, 1839, 1839, 1824, 1824, 1691, 1697, 1914,
- /*   390 */  1685, 1778, 1691, 1797, 1800, 1713, 1926, 1941, 1941, 1959,
- /*   400 */  1959, 1959, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207,
- /*   410 */  2207, 2207, 2207, 2207, 2207, 2207, 2207,   69, 1037,   79,
- /*   420 */  1088,  651, 1196, 1415, 1501, 1439, 1369, 1452,  911, 1211,
- /*   430 */  1524, 1469, 1551, 1567, 1570, 1624, 1640, 1644, 1499, 1440,
- /*   440 */  1572, 1464, 1597,  275,  782, 1586, 1648, 1678, 1553, 1682,
- /*   450 */  1687, 1388, 1502, 1696, 1706, 1588, 1486, 1977, 1980, 1966,
- /*   460 */  1825, 1975, 1976, 1970, 1971, 1855, 1844, 1866, 1973, 1973,
- /*   470 */  1978, 1856, 1981, 1858, 1984, 2003, 1862, 1875, 1973, 1877,
- /*   480 */  1945, 1972, 1973, 1861, 1956, 1958, 1960, 1961, 1885, 1902,
- /*   490 */  1986, 1879, 2021, 2018, 2002, 1917, 1864, 1967, 2010, 1968,
- /*   500 */  1962, 2004, 1905, 1928, 2025, 2030, 2033, 1930, 1932, 2041,
- /*   510 */  2006, 2048, 2049, 2053, 2050, 2009, 2019, 2054, 1983, 2056,
- /*   520 */  2059, 2014, 2051, 2062, 2052, 1937, 2067, 2068, 2069, 2065,
- /*   530 */  2071, 2073, 1998, 1955, 2075, 2076, 1985, 2070, 2079, 1963,
- /*   540 */  2078, 2072, 2081, 2082, 2083, 2016, 2034, 2022, 2074, 2036,
- /*   550 */  2026, 2085, 2090, 2099, 2100, 2098, 2101, 2093, 1987, 1988,
- /*   560 */  2106, 2078, 2109, 2110, 2111, 2113, 2112, 2114, 2115, 2118,
- /*   570 */  2125, 2119, 2121, 2122, 2123, 2126, 2127, 2128, 2008, 2005,
- /*   580 */  2012, 2013, 2131, 2129, 2135, 2150, 2156,
+ /*     0 */  1977, 1727, 1634, 1336, 1336,  333,  161, 1452, 1477, 1568,
+ /*    10 */  1991, 1991, 1991,  148,  333,  333,  333,  333,  333,    0,
+ /*    20 */     0,  279, 1153, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*    30 */  1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,  121,  121,
+ /*    40 */   498,  498,   51,  402,  408,  706,  706,  445,  445,  445,
+ /*    50 */   445,  104,  208,  320,  391,  495,  515,  619,  639,  743,
+ /*    60 */   763,  867,  887, 1000, 1020, 1133, 1153, 1153, 1153, 1153,
+ /*    70 */  1153, 1153, 1153, 1153, 1153, 1153, 1153, 1153, 1153, 1153,
+ /*    80 */  1153, 1153, 1153, 1153, 1257, 1153, 1277,  348,  348, 1593,
+ /*    90 */  1799, 1819, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*   100 */  1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*   110 */  1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*   120 */  1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*   130 */  1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991, 1991,
+ /*   140 */  1991, 1991,   32,  851,  851,  851,  851,  851,  851,  851,
+ /*   150 */    91,  112,   76,  459,  706, 1014, 1102,  706,  706,  357,
+ /*   160 */   357,  706,  540,  292,  571,  571,  571,  636,   37,   37,
+ /*   170 */  2176, 2176,  159,  159,  159,  587,  462,  462,  462,  462,
+ /*   180 */   939,  939,  215,  459,   13,  611,  706,  706,  706,  706,
+ /*   190 */   706,  706,  706,  706,  706,  706,  706,  706,  706,  706,
+ /*   200 */   706,  706,  706,  706,  706,  563,  216,  216,  706,  956,
+ /*   210 */  1192, 1192,  950,  950, 1160, 1160, 1135, 2176, 2176, 2176,
+ /*   220 */  2176, 2176, 2176, 2176, 1311,  907,  907,  869,  225,  892,
+ /*   230 */    63,  980,  646,  975,  998,  706,  706,  706,  706,  706,
+ /*   240 */   706,  706,  706,  706,  706,  223,  706,  706,  706,  706,
+ /*   250 */   706,  706,  706,  706,  706,  706,  706,  706, 1158, 1158,
+ /*   260 */  1158,  706,  706,  706,   48,  706,  706,  706, 1078,  565,
+ /*   270 */   706, 1330,  706,  706,  706,  706,  706,  706,  706,  706,
+ /*   280 */  1013,  864,  511,  925,  925,  925,  925, 1202,  511,  511,
+ /*   290 */   697,  930, 1212,  642, 1368, 1350, 1254, 1350, 1416, 1224,
+ /*   300 */  1368, 1368, 1224, 1368, 1254, 1416,  657,  880,  188,  558,
+ /*   310 */   558,  558,  467,  467,  467,  467,  993,  993, 1245, 1415,
+ /*   320 */  1306, 1389, 1771, 1771, 1691, 1691, 1798, 1798, 1691, 1692,
+ /*   330 */  1695, 1829, 1703, 1707, 1813, 1703, 1707, 1845, 1845, 1845,
+ /*   340 */  1845, 1691, 1854, 1724, 1695, 1695, 1724, 1829, 1813, 1724,
+ /*   350 */  1813, 1724, 1691, 1854, 1722, 1823, 1691, 1854, 1871, 1691,
+ /*   360 */  1854, 1691, 1854, 1871, 1779, 1779, 1779, 1843, 1887, 1887,
+ /*   370 */  1871, 1779, 1792, 1779, 1843, 1779, 1779, 1763, 1907, 1821,
+ /*   380 */  1821, 1871, 1691, 1852, 1852, 1862, 1862, 1703, 1707, 1925,
+ /*   390 */  1691, 1791, 1703, 1804, 1818, 1724, 1943, 1958, 1958, 1975,
+ /*   400 */  1975, 1975, 2176, 2176, 2176, 2176, 2176, 2176, 2176, 2176,
+ /*   410 */  2176, 2176, 2176, 2176, 2176, 2176, 2176,  440,  934, 1315,
+ /*   420 */  1387,  190, 1032, 1275, 1481, 1383,   41, 1449,  853, 1355,
+ /*   430 */  1525, 1492, 1539, 1575, 1605, 1627, 1633, 1637, 1564, 1444,
+ /*   440 */  1537, 1393, 1673, 1440, 1518, 1570, 1674, 1681, 1498, 1711,
+ /*   450 */  1720, 1463, 1600, 1735, 1738, 1596, 1011, 2002, 2005, 1992,
+ /*   460 */  1851, 2006, 2004, 1999, 2000, 1884, 1873, 1895, 2003, 2003,
+ /*   470 */  2007, 1886, 2010, 1888, 2016, 2033, 1892, 1905, 2003, 1906,
+ /*   480 */  1976, 2008, 2003, 1889, 1984, 1985, 1987, 1988, 1913, 1929,
+ /*   490 */  2013, 1908, 2048, 2046, 2030, 1937, 1893, 1989, 2029, 1993,
+ /*   500 */  1981, 2019, 1919, 1948, 2041, 2047, 2049, 1934, 1944, 2050,
+ /*   510 */  2001, 2051, 2052, 2054, 2053, 2011, 2062, 2056, 1986, 2057,
+ /*   520 */  2059, 2014, 2058, 2055, 2060, 1936, 2065, 2061, 2066, 2063,
+ /*   530 */  2067, 2069, 1995, 1954, 2074, 2077, 1990, 2068, 2079, 1959,
+ /*   540 */  2080, 2070, 2078, 2081, 2084, 2025, 2043, 2034, 2075, 2076,
+ /*   550 */  2028, 2089, 2101, 2103, 2102, 2104, 2105, 2093, 1994, 1996,
+ /*   560 */  2108, 2080, 2109, 2110, 2111, 2106, 2113, 2116, 2112, 2114,
+ /*   570 */  2131, 2121, 2122, 2127, 2128, 2132, 2133, 2134, 2017, 2015,
+ /*   580 */  2018, 2020, 2136, 2135, 2141, 2162, 2163,
 };
 #define YY_REDUCE_COUNT (416)
-#define YY_REDUCE_MIN   (-276)
-#define YY_REDUCE_MAX   (1779)
+#define YY_REDUCE_MIN   (-245)
+#define YY_REDUCE_MAX   (1802)
 static const short yy_reduce_ofst[] = {
- /*     0 */   -66,  217,  -63, -177, -180,  161,  364,   64, -183,  162,
- /*    10 */   223,  367,  414, -173,  473,  514,  525,  622,  626, -207,
- /*    20 */   351, -276,  -38,  693,  811,  831,  833,  888, -188,  945,
- /*    30 */   947,  416,  558,  951,  867,  287, 1078, 1080, -186,  224,
- /*    40 */  -132,   42,  964,  269,  417,  796,  810, -237, -231, -237,
- /*    50 */  -231,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,
- /*    60 */   -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,
- /*    70 */   -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,
- /*    80 */   -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  895,
- /*    90 */   925,  967,  980, 1100, 1143, 1169, 1203, 1225, 1228, 1242,
- /*   100 */  1247, 1250, 1253, 1255, 1261, 1267, 1272, 1275, 1283, 1286,
- /*   110 */  1288, 1292, 1333, 1335, 1347, 1349, 1352, 1354, 1360, 1366,
- /*   120 */  1381, 1391, 1406, 1408, 1413, 1416, 1418, 1422, 1425, 1427,
- /*   130 */  1463, 1465, 1472, 1478, 1480, 1491, 1498, 1500, 1517, 1519,
- /*   140 */  1528, 1536,  -45,  -45,  -45,  -45,  -45,  -45,  -45,  -45,
- /*   150 */   -45,  -45,  -45,  312, -158,  285, -219,    9,  166,  370,
- /*   160 */   545,  707,  -45,  930,  601,  963, 1067,  792,  -45,  -45,
- /*   170 */   -45,  -45, -204, -204, -204,  369, -171, -129,  632,  678,
- /*   180 */   202,  352, -270,  412,  627,  627,   -9,  122,  415,  419,
- /*   190 */   -56,  248,  583,  920,    6,  261,  459,  795, 1049,  813,
- /*   200 */  1062, 1082, -161,  778, 1063,  797,  870, 1003, 1128,  443,
- /*   210 */  1031, 1072, 1191, 1192,  957, 1120,  105, 1149,  523,  933,
- /*   220 */  1218, 1238, 1254, 1251, -138,   96,  117,  146,  181,  277,
- /*   230 */   280,  421,  480,  712,  830,  850, 1085, 1099, 1129, 1209,
- /*   240 */  1323, 1331, 1336, 1364, 1407,  368, 1412, 1433, 1438, 1474,
- /*   250 */  1481, 1505, 1506, 1526, 1538, 1544, 1545, 1546,  722,  764,
- /*   260 */   856, 1547, 1548, 1550, 1188, 1554, 1557, 1561, 1298, 1260,
- /*   270 */  1562, 1456, 1564,  280, 1568, 1571, 1573, 1574, 1575, 1576,
- /*   280 */  1457, 1477, 1520, 1514, 1515, 1516, 1518, 1188, 1520, 1520,
- /*   290 */  1530, 1563, 1584, 1482, 1504, 1510, 1534, 1513, 1488, 1537,
- /*   300 */  1512, 1521, 1539, 1522, 1541, 1493, 1583, 1559, 1565, 1585,
- /*   310 */  1587, 1589, 1529, 1531, 1532, 1549, 1558, 1566, 1535, 1577,
- /*   320 */  1582, 1622, 1533, 1540, 1627, 1628, 1552, 1555, 1633, 1560,
- /*   330 */  1578, 1581, 1591, 1594, 1612, 1598, 1600, 1617, 1618, 1619,
- /*   340 */  1620, 1658, 1661, 1621, 1590, 1592, 1623, 1593, 1632, 1626,
- /*   350 */  1634, 1629, 1670, 1675, 1595, 1599, 1680, 1679, 1662, 1684,
- /*   360 */  1688, 1690, 1689, 1664, 1674, 1676, 1677, 1669, 1681, 1693,
- /*   370 */  1692, 1694, 1695, 1703, 1673, 1704, 1707, 1596, 1601, 1631,
- /*   380 */  1641, 1705, 1711, 1608, 1613, 1667, 1671, 1698, 1700, 1660,
- /*   390 */  1741, 1663, 1701, 1699, 1714, 1708, 1750, 1766, 1768, 1773,
- /*   400 */  1774, 1775, 1666, 1668, 1672, 1761, 1758, 1762, 1763, 1764,
- /*   410 */  1767, 1756, 1757, 1769, 1771, 1777, 1779,
+ /*     0 */  -176, -122, 1208,  268,  274, -159,  389, -192,  317, -190,
+ /*    10 */  -133,  392,  395,  101,  460,  513,  529,  638,  895,  135,
+ /*    20 */   141, -237, -113,   53,  539,  541,  608,  610,  338,  663,
+ /*    30 */   665,  522,  644,  669,  865, -194,  940,  945,  486,  734,
+ /*    40 */   -18,  165, -114,  525,  773,  847,  923, -234,  -10, -234,
+ /*    50 */   -10, -245, -245, -245, -245, -245, -245, -245, -245, -245,
+ /*    60 */  -245, -245, -245, -245, -245, -245, -245, -245, -245, -245,
+ /*    70 */  -245, -245, -245, -245, -245, -245, -245, -245, -245, -245,
+ /*    80 */  -245, -245, -245, -245, -245, -245, -245, -245, -245,  491,
+ /*    90 */   561,  709,  809,  822,  927,  949,  952, 1036, 1053, 1055,
+ /*   100 */  1073, 1075, 1080, 1100, 1122, 1129, 1157, 1159, 1184, 1197,
+ /*   110 */  1201, 1204, 1206, 1241, 1251, 1261, 1292, 1303, 1308, 1310,
+ /*   120 */  1312, 1317, 1321, 1323, 1328, 1337, 1357, 1362, 1402, 1406,
+ /*   130 */  1408, 1419, 1423, 1454, 1460, 1462, 1473, 1500, 1512, 1547,
+ /*   140 */  1560, 1569, -245, -245, -245, -245, -245, -245, -245, -245,
+ /*   150 */  -245, -245, -245, -131,  168,  195, -225,  -62,  526,  427,
+ /*   160 */   818,  151, -245,  546,   65,  899,  937,  780, -245, -245,
+ /*   170 */  -245, -245, -208, -208, -208,   54,   39,  810,  894,  953,
+ /*   180 */  -139,  936, -196, -189,  912,  912, 1082,  243,  516,  635,
+ /*   190 */  -116,   56, 1040, 1109,  690,  271, 1056,  512,  758, 1144,
+ /*   200 */   960, 1155, -191,  360, 1093,  966,  548, 1113,  281,  595,
+ /*   210 */  1114, 1121, 1178, 1185, 1012, 1076,   23, 1041,  260,  585,
+ /*   220 */   769, 1175, 1222, 1211, -181,  -45,   72,  107,   94,  214,
+ /*   230 */   221,  336,  363,  523,  609,  650,  664,  700,  893,  999,
+ /*   240 */  1088, 1283, 1339, 1356, 1366, 1081, 1377, 1394, 1398, 1427,
+ /*   250 */  1428, 1448, 1458, 1468, 1472, 1475, 1476, 1482, 1456, 1461,
+ /*   260 */  1465, 1508, 1514, 1522, 1459, 1526, 1540, 1551, 1353, 1431,
+ /*   270 */  1567, 1531, 1576,  221, 1577, 1580, 1581, 1585, 1586, 1587,
+ /*   280 */  1479, 1493, 1541, 1528, 1532, 1533, 1534, 1459, 1541, 1541,
+ /*   290 */  1544, 1578, 1591, 1495, 1519, 1529, 1549, 1530, 1505, 1554,
+ /*   300 */  1536, 1538, 1555, 1542, 1557, 1509, 1589, 1582, 1588, 1599,
+ /*   310 */  1601, 1602, 1545, 1552, 1562, 1563, 1566, 1579, 1546, 1584,
+ /*   320 */  1592, 1629, 1543, 1548, 1632, 1638, 1550, 1553, 1640, 1559,
+ /*   330 */  1574, 1583, 1606, 1603, 1623, 1608, 1611, 1625, 1626, 1630,
+ /*   340 */  1631, 1666, 1672, 1628, 1604, 1607, 1635, 1612, 1639, 1636,
+ /*   350 */  1641, 1644, 1676, 1679, 1594, 1597, 1686, 1693, 1675, 1694,
+ /*   360 */  1696, 1697, 1700, 1680, 1682, 1684, 1685, 1683, 1687, 1689,
+ /*   370 */  1688, 1690, 1701, 1702, 1699, 1704, 1705, 1614, 1624, 1648,
+ /*   380 */  1654, 1719, 1741, 1642, 1643, 1678, 1698, 1706, 1708, 1668,
+ /*   390 */  1747, 1669, 1709, 1712, 1718, 1721, 1766, 1776, 1784, 1790,
+ /*   400 */  1801, 1802, 1710, 1713, 1715, 1777, 1770, 1778, 1782, 1783,
+ /*   410 */  1793, 1768, 1775, 1785, 1788, 1797, 1796,
 };
 static const YYACTIONTYPE yy_default[] = {
  /*     0 */  1667, 1667, 1667, 1495, 1258, 1371, 1258, 1258, 1258, 1258,
@@ -175052,53 +177715,54 @@ static const YYACTIONTYPE yy_default[] = {
 static const YYCODETYPE yyFallback[] = {
     0,  /*          $ => nothing */
     0,  /*       SEMI => nothing */
-   60,  /*    EXPLAIN => ID */
-   60,  /*      QUERY => ID */
-   60,  /*       PLAN => ID */
-   60,  /*      BEGIN => ID */
+    9,  /*    EXPLAIN => ID */
+    9,  /*      QUERY => ID */
+    9,  /*       PLAN => ID */
+    9,  /*      BEGIN => ID */
     0,  /* TRANSACTION => nothing */
-   60,  /*   DEFERRED => ID */
-   60,  /*  IMMEDIATE => ID */
-   60,  /*  EXCLUSIVE => ID */
+    9,  /*   DEFERRED => ID */
+    9,  /*  IMMEDIATE => ID */
+    0,  /*         ID => nothing */
     0,  /*     COMMIT => nothing */
-   60,  /*        END => ID */
-   60,  /*   ROLLBACK => ID */
-   60,  /*  SAVEPOINT => ID */
-   60,  /*    RELEASE => ID */
+    9,  /*        END => ID */
+    9,  /*   ROLLBACK => ID */
+    9,  /*  SAVEPOINT => ID */
+    9,  /*    RELEASE => ID */
     0,  /*         TO => nothing */
     0,  /*      TABLE => nothing */
     0,  /*     CREATE => nothing */
-   60,  /*         IF => ID */
+    9,  /*         IF => ID */
     0,  /*        NOT => nothing */
     0,  /*     EXISTS => nothing */
-   60,  /*       TEMP => ID */
+    9,  /*       TEMP => ID */
     0,  /*         LP => nothing */
     0,  /*         RP => nothing */
     0,  /*         AS => nothing */
     0,  /*      COMMA => nothing */
-   60,  /*    WITHOUT => ID */
-   60,  /*      ABORT => ID */
-   60,  /*     ACTION => ID */
-   60,  /*      AFTER => ID */
-   60,  /*    ANALYZE => ID */
-   60,  /*        ASC => ID */
-   60,  /*     ATTACH => ID */
-   60,  /*     BEFORE => ID */
-   60,  /*         BY => ID */
-   60,  /*    CASCADE => ID */
-   60,  /*       CAST => ID */
-   60,  /*   CONFLICT => ID */
-   60,  /*   DATABASE => ID */
-   60,  /*       DESC => ID */
-   60,  /*     DETACH => ID */
-   60,  /*       EACH => ID */
-   60,  /*       FAIL => ID */
+    9,  /*    WITHOUT => ID */
+    9,  /*      ABORT => ID */
+    9,  /*     ACTION => ID */
+    9,  /*      AFTER => ID */
+    9,  /*    ANALYZE => ID */
+    9,  /*        ASC => ID */
+    9,  /*     ATTACH => ID */
+    9,  /*     BEFORE => ID */
+    9,  /*         BY => ID */
+    9,  /*    CASCADE => ID */
+    9,  /*       CAST => ID */
+    9,  /*   CONFLICT => ID */
+    9,  /*   DATABASE => ID */
+    9,  /*       DESC => ID */
+    9,  /*     DETACH => ID */
+    9,  /*       EACH => ID */
+    9,  /*  EXCLUSIVE => ID */
+    9,  /*       FAIL => ID */
     0,  /*         OR => nothing */
     0,  /*        AND => nothing */
     0,  /*         IS => nothing */
     0,  /*      ISNOT => nothing */
-   60,  /*      MATCH => ID */
-   60,  /*    LIKE_KW => ID */
+    9,  /*      MATCH => ID */
+    9,  /*    LIKE_KW => ID */
     0,  /*    BETWEEN => nothing */
     0,  /*         IN => nothing */
     0,  /*     ISNULL => nothing */
@@ -175110,48 +177774,47 @@ static const YYCODETYPE yyFallback[] = {
     0,  /*         LT => nothing */
     0,  /*         GE => nothing */
     0,  /*     ESCAPE => nothing */
-    0,  /*         ID => nothing */
-   60,  /*   COLUMNKW => ID */
-   60,  /*         DO => ID */
-   60,  /*        FOR => ID */
-   60,  /*     IGNORE => ID */
-   60,  /*  INITIALLY => ID */
-   60,  /*    INSTEAD => ID */
-   60,  /*         NO => ID */
-   60,  /*        KEY => ID */
-   60,  /*         OF => ID */
-   60,  /*     OFFSET => ID */
-   60,  /*     PRAGMA => ID */
-   60,  /*      RAISE => ID */
-   60,  /*  RECURSIVE => ID */
-   60,  /*    REPLACE => ID */
-   60,  /*   RESTRICT => ID */
-   60,  /*        ROW => ID */
-   60,  /*       ROWS => ID */
-   60,  /*    TRIGGER => ID */
-   60,  /*     VACUUM => ID */
-   60,  /*       VIEW => ID */
-   60,  /*    VIRTUAL => ID */
-   60,  /*       WITH => ID */
-   60,  /*      NULLS => ID */
-   60,  /*      FIRST => ID */
-   60,  /*       LAST => ID */
-   60,  /*    CURRENT => ID */
-   60,  /*  FOLLOWING => ID */
-   60,  /*  PARTITION => ID */
-   60,  /*  PRECEDING => ID */
-   60,  /*      RANGE => ID */
-   60,  /*  UNBOUNDED => ID */
-   60,  /*    EXCLUDE => ID */
-   60,  /*     GROUPS => ID */
-   60,  /*     OTHERS => ID */
-   60,  /*       TIES => ID */
-   60,  /*  GENERATED => ID */
-   60,  /*     ALWAYS => ID */
-   60,  /* MATERIALIZED => ID */
-   60,  /*    REINDEX => ID */
-   60,  /*     RENAME => ID */
-   60,  /*   CTIME_KW => ID */
+    9,  /*   COLUMNKW => ID */
+    9,  /*         DO => ID */
+    9,  /*        FOR => ID */
+    9,  /*     IGNORE => ID */
+    9,  /*  INITIALLY => ID */
+    9,  /*    INSTEAD => ID */
+    9,  /*         NO => ID */
+    9,  /*        KEY => ID */
+    9,  /*         OF => ID */
+    9,  /*     OFFSET => ID */
+    9,  /*     PRAGMA => ID */
+    9,  /*      RAISE => ID */
+    9,  /*  RECURSIVE => ID */
+    9,  /*    REPLACE => ID */
+    9,  /*   RESTRICT => ID */
+    9,  /*        ROW => ID */
+    9,  /*       ROWS => ID */
+    9,  /*    TRIGGER => ID */
+    9,  /*     VACUUM => ID */
+    9,  /*       VIEW => ID */
+    9,  /*    VIRTUAL => ID */
+    9,  /*       WITH => ID */
+    9,  /*      NULLS => ID */
+    9,  /*      FIRST => ID */
+    9,  /*       LAST => ID */
+    9,  /*    CURRENT => ID */
+    9,  /*  FOLLOWING => ID */
+    9,  /*  PARTITION => ID */
+    9,  /*  PRECEDING => ID */
+    9,  /*      RANGE => ID */
+    9,  /*  UNBOUNDED => ID */
+    9,  /*    EXCLUDE => ID */
+    9,  /*     GROUPS => ID */
+    9,  /*     OTHERS => ID */
+    9,  /*       TIES => ID */
+    9,  /*  GENERATED => ID */
+    9,  /*     ALWAYS => ID */
+    9,  /* MATERIALIZED => ID */
+    9,  /*    REINDEX => ID */
+    9,  /*     RENAME => ID */
+    9,  /*   CTIME_KW => ID */
     0,  /*        ANY => nothing */
     0,  /*     BITAND => nothing */
     0,  /*      BITOR => nothing */
@@ -175227,6 +177890,7 @@ static const YYCODETYPE yyFallback[] = {
     0,  /*     UMINUS => nothing */
     0,  /*      TRUTH => nothing */
     0,  /*   REGISTER => nothing */
+    0,  /* CONCURRENT => nothing */
     0,  /*     VECTOR => nothing */
     0,  /* SELECT_COLUMN => nothing */
     0,  /* IF_NULL_ROW => nothing */
@@ -175328,7 +177992,7 @@ static const char *const yyTokenName[] = {
   /*    6 */ "TRANSACTION",
   /*    7 */ "DEFERRED",
   /*    8 */ "IMMEDIATE",
-  /*    9 */ "EXCLUSIVE",
+  /*    9 */ "ID",
   /*   10 */ "COMMIT",
   /*   11 */ "END",
   /*   12 */ "ROLLBACK",
@@ -175361,25 +178025,25 @@ static const char *const yyTokenName[] = {
   /*   39 */ "DESC",
   /*   40 */ "DETACH",
   /*   41 */ "EACH",
-  /*   42 */ "FAIL",
-  /*   43 */ "OR",
-  /*   44 */ "AND",
-  /*   45 */ "IS",
-  /*   46 */ "ISNOT",
-  /*   47 */ "MATCH",
-  /*   48 */ "LIKE_KW",
-  /*   49 */ "BETWEEN",
-  /*   50 */ "IN",
-  /*   51 */ "ISNULL",
-  /*   52 */ "NOTNULL",
-  /*   53 */ "NE",
-  /*   54 */ "EQ",
-  /*   55 */ "GT",
-  /*   56 */ "LE",
-  /*   57 */ "LT",
-  /*   58 */ "GE",
-  /*   59 */ "ESCAPE",
-  /*   60 */ "ID",
+  /*   42 */ "EXCLUSIVE",
+  /*   43 */ "FAIL",
+  /*   44 */ "OR",
+  /*   45 */ "AND",
+  /*   46 */ "IS",
+  /*   47 */ "ISNOT",
+  /*   48 */ "MATCH",
+  /*   49 */ "LIKE_KW",
+  /*   50 */ "BETWEEN",
+  /*   51 */ "IN",
+  /*   52 */ "ISNULL",
+  /*   53 */ "NOTNULL",
+  /*   54 */ "NE",
+  /*   55 */ "EQ",
+  /*   56 */ "GT",
+  /*   57 */ "LE",
+  /*   58 */ "LT",
+  /*   59 */ "GE",
+  /*   60 */ "ESCAPE",
   /*   61 */ "COLUMNKW",
   /*   62 */ "DO",
   /*   63 */ "FOR",
@@ -175496,151 +178160,152 @@ static const char *const yyTokenName[] = {
   /*  174 */ "UMINUS",
   /*  175 */ "TRUTH",
   /*  176 */ "REGISTER",
-  /*  177 */ "VECTOR",
-  /*  178 */ "SELECT_COLUMN",
-  /*  179 */ "IF_NULL_ROW",
-  /*  180 */ "ASTERISK",
-  /*  181 */ "SPAN",
-  /*  182 */ "ERROR",
-  /*  183 */ "QNUMBER",
-  /*  184 */ "SPACE",
-  /*  185 */ "ILLEGAL",
-  /*  186 */ "input",
-  /*  187 */ "cmdlist",
-  /*  188 */ "ecmd",
-  /*  189 */ "cmdx",
-  /*  190 */ "explain",
-  /*  191 */ "cmd",
-  /*  192 */ "transtype",
-  /*  193 */ "trans_opt",
-  /*  194 */ "nm",
-  /*  195 */ "savepoint_opt",
-  /*  196 */ "create_table",
-  /*  197 */ "create_table_args",
-  /*  198 */ "createkw",
-  /*  199 */ "temp",
-  /*  200 */ "ifnotexists",
-  /*  201 */ "dbnm",
-  /*  202 */ "columnlist",
-  /*  203 */ "conslist_opt",
-  /*  204 */ "table_option_set",
-  /*  205 */ "select",
-  /*  206 */ "table_option",
-  /*  207 */ "columnname",
-  /*  208 */ "carglist",
-  /*  209 */ "typetoken",
-  /*  210 */ "typename",
-  /*  211 */ "signed",
-  /*  212 */ "plus_num",
-  /*  213 */ "minus_num",
-  /*  214 */ "scanpt",
-  /*  215 */ "scantok",
-  /*  216 */ "ccons",
-  /*  217 */ "term",
-  /*  218 */ "expr",
-  /*  219 */ "onconf",
-  /*  220 */ "sortorder",
-  /*  221 */ "autoinc",
-  /*  222 */ "eidlist_opt",
-  /*  223 */ "refargs",
-  /*  224 */ "defer_subclause",
-  /*  225 */ "generated",
-  /*  226 */ "refarg",
-  /*  227 */ "refact",
-  /*  228 */ "init_deferred_pred_opt",
-  /*  229 */ "conslist",
-  /*  230 */ "tconscomma",
-  /*  231 */ "tcons",
-  /*  232 */ "sortlist",
-  /*  233 */ "eidlist",
-  /*  234 */ "defer_subclause_opt",
-  /*  235 */ "orconf",
-  /*  236 */ "resolvetype",
-  /*  237 */ "raisetype",
-  /*  238 */ "ifexists",
-  /*  239 */ "fullname",
-  /*  240 */ "selectnowith",
-  /*  241 */ "oneselect",
-  /*  242 */ "wqlist",
-  /*  243 */ "multiselect_op",
-  /*  244 */ "distinct",
-  /*  245 */ "selcollist",
-  /*  246 */ "from",
-  /*  247 */ "where_opt",
-  /*  248 */ "groupby_opt",
-  /*  249 */ "having_opt",
-  /*  250 */ "orderby_opt",
-  /*  251 */ "limit_opt",
-  /*  252 */ "window_clause",
-  /*  253 */ "values",
-  /*  254 */ "nexprlist",
-  /*  255 */ "mvalues",
-  /*  256 */ "sclp",
-  /*  257 */ "as",
-  /*  258 */ "seltablist",
-  /*  259 */ "stl_prefix",
-  /*  260 */ "joinop",
-  /*  261 */ "on_using",
-  /*  262 */ "indexed_by",
-  /*  263 */ "exprlist",
-  /*  264 */ "xfullname",
-  /*  265 */ "idlist",
-  /*  266 */ "indexed_opt",
-  /*  267 */ "nulls",
-  /*  268 */ "with",
-  /*  269 */ "where_opt_ret",
-  /*  270 */ "setlist",
-  /*  271 */ "insert_cmd",
-  /*  272 */ "idlist_opt",
-  /*  273 */ "upsert",
-  /*  274 */ "returning",
-  /*  275 */ "filter_over",
-  /*  276 */ "likeop",
-  /*  277 */ "between_op",
-  /*  278 */ "in_op",
-  /*  279 */ "paren_exprlist",
-  /*  280 */ "case_operand",
-  /*  281 */ "case_exprlist",
-  /*  282 */ "case_else",
-  /*  283 */ "uniqueflag",
-  /*  284 */ "collate",
-  /*  285 */ "vinto",
-  /*  286 */ "nmnum",
-  /*  287 */ "trigger_decl",
-  /*  288 */ "trigger_cmd_list",
-  /*  289 */ "trigger_time",
-  /*  290 */ "trigger_event",
-  /*  291 */ "foreach_clause",
-  /*  292 */ "when_clause",
-  /*  293 */ "trigger_cmd",
-  /*  294 */ "trnm",
-  /*  295 */ "tridxby",
-  /*  296 */ "database_kw_opt",
-  /*  297 */ "key_opt",
-  /*  298 */ "add_column_fullname",
-  /*  299 */ "kwcolumn_opt",
-  /*  300 */ "create_vtab",
-  /*  301 */ "vtabarglist",
-  /*  302 */ "vtabarg",
-  /*  303 */ "vtabargtoken",
-  /*  304 */ "lp",
-  /*  305 */ "anylist",
-  /*  306 */ "wqitem",
-  /*  307 */ "wqas",
-  /*  308 */ "withnm",
-  /*  309 */ "windowdefn_list",
-  /*  310 */ "windowdefn",
-  /*  311 */ "window",
-  /*  312 */ "frame_opt",
-  /*  313 */ "part_opt",
-  /*  314 */ "filter_clause",
-  /*  315 */ "over_clause",
-  /*  316 */ "range_or_rows",
-  /*  317 */ "frame_bound",
-  /*  318 */ "frame_bound_s",
-  /*  319 */ "frame_bound_e",
-  /*  320 */ "frame_exclude_opt",
-  /*  321 */ "frame_exclude",
+  /*  177 */ "CONCURRENT",
+  /*  178 */ "VECTOR",
+  /*  179 */ "SELECT_COLUMN",
+  /*  180 */ "IF_NULL_ROW",
+  /*  181 */ "ASTERISK",
+  /*  182 */ "SPAN",
+  /*  183 */ "ERROR",
+  /*  184 */ "QNUMBER",
+  /*  185 */ "SPACE",
+  /*  186 */ "ILLEGAL",
+  /*  187 */ "input",
+  /*  188 */ "cmdlist",
+  /*  189 */ "ecmd",
+  /*  190 */ "cmdx",
+  /*  191 */ "explain",
+  /*  192 */ "cmd",
+  /*  193 */ "transtype",
+  /*  194 */ "trans_opt",
+  /*  195 */ "nm",
+  /*  196 */ "savepoint_opt",
+  /*  197 */ "create_table",
+  /*  198 */ "create_table_args",
+  /*  199 */ "createkw",
+  /*  200 */ "temp",
+  /*  201 */ "ifnotexists",
+  /*  202 */ "dbnm",
+  /*  203 */ "columnlist",
+  /*  204 */ "conslist_opt",
+  /*  205 */ "table_option_set",
+  /*  206 */ "select",
+  /*  207 */ "table_option",
+  /*  208 */ "columnname",
+  /*  209 */ "carglist",
+  /*  210 */ "typetoken",
+  /*  211 */ "typename",
+  /*  212 */ "signed",
+  /*  213 */ "plus_num",
+  /*  214 */ "minus_num",
+  /*  215 */ "scanpt",
+  /*  216 */ "scantok",
+  /*  217 */ "ccons",
+  /*  218 */ "term",
+  /*  219 */ "expr",
+  /*  220 */ "onconf",
+  /*  221 */ "sortorder",
+  /*  222 */ "autoinc",
+  /*  223 */ "eidlist_opt",
+  /*  224 */ "refargs",
+  /*  225 */ "defer_subclause",
+  /*  226 */ "generated",
+  /*  227 */ "refarg",
+  /*  228 */ "refact",
+  /*  229 */ "init_deferred_pred_opt",
+  /*  230 */ "conslist",
+  /*  231 */ "tconscomma",
+  /*  232 */ "tcons",
+  /*  233 */ "sortlist",
+  /*  234 */ "eidlist",
+  /*  235 */ "defer_subclause_opt",
+  /*  236 */ "orconf",
+  /*  237 */ "resolvetype",
+  /*  238 */ "raisetype",
+  /*  239 */ "ifexists",
+  /*  240 */ "fullname",
+  /*  241 */ "selectnowith",
+  /*  242 */ "oneselect",
+  /*  243 */ "wqlist",
+  /*  244 */ "multiselect_op",
+  /*  245 */ "distinct",
+  /*  246 */ "selcollist",
+  /*  247 */ "from",
+  /*  248 */ "where_opt",
+  /*  249 */ "groupby_opt",
+  /*  250 */ "having_opt",
+  /*  251 */ "orderby_opt",
+  /*  252 */ "limit_opt",
+  /*  253 */ "window_clause",
+  /*  254 */ "values",
+  /*  255 */ "nexprlist",
+  /*  256 */ "mvalues",
+  /*  257 */ "sclp",
+  /*  258 */ "as",
+  /*  259 */ "seltablist",
+  /*  260 */ "stl_prefix",
+  /*  261 */ "joinop",
+  /*  262 */ "on_using",
+  /*  263 */ "indexed_by",
+  /*  264 */ "exprlist",
+  /*  265 */ "xfullname",
+  /*  266 */ "idlist",
+  /*  267 */ "indexed_opt",
+  /*  268 */ "nulls",
+  /*  269 */ "with",
+  /*  270 */ "where_opt_ret",
+  /*  271 */ "setlist",
+  /*  272 */ "insert_cmd",
+  /*  273 */ "idlist_opt",
+  /*  274 */ "upsert",
+  /*  275 */ "returning",
+  /*  276 */ "filter_over",
+  /*  277 */ "likeop",
+  /*  278 */ "between_op",
+  /*  279 */ "in_op",
+  /*  280 */ "paren_exprlist",
+  /*  281 */ "case_operand",
+  /*  282 */ "case_exprlist",
+  /*  283 */ "case_else",
+  /*  284 */ "uniqueflag",
+  /*  285 */ "collate",
+  /*  286 */ "vinto",
+  /*  287 */ "nmnum",
+  /*  288 */ "trigger_decl",
+  /*  289 */ "trigger_cmd_list",
+  /*  290 */ "trigger_time",
+  /*  291 */ "trigger_event",
+  /*  292 */ "foreach_clause",
+  /*  293 */ "when_clause",
+  /*  294 */ "trigger_cmd",
+  /*  295 */ "trnm",
+  /*  296 */ "tridxby",
+  /*  297 */ "database_kw_opt",
+  /*  298 */ "key_opt",
+  /*  299 */ "add_column_fullname",
+  /*  300 */ "kwcolumn_opt",
+  /*  301 */ "create_vtab",
+  /*  302 */ "vtabarglist",
+  /*  303 */ "vtabarg",
+  /*  304 */ "vtabargtoken",
+  /*  305 */ "lp",
+  /*  306 */ "anylist",
+  /*  307 */ "wqitem",
+  /*  308 */ "wqas",
+  /*  309 */ "withnm",
+  /*  310 */ "windowdefn_list",
+  /*  311 */ "windowdefn",
+  /*  312 */ "window",
+  /*  313 */ "frame_opt",
+  /*  314 */ "part_opt",
+  /*  315 */ "filter_clause",
+  /*  316 */ "over_clause",
+  /*  317 */ "range_or_rows",
+  /*  318 */ "frame_bound",
+  /*  319 */ "frame_bound_s",
+  /*  320 */ "frame_bound_e",
+  /*  321 */ "frame_exclude_opt",
+  /*  322 */ "frame_exclude",
 };
 #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
 
@@ -175655,7 +178320,7 @@ static const char *const yyRuleName[] = {
  /*   4 */ "transtype ::=",
  /*   5 */ "transtype ::= DEFERRED",
  /*   6 */ "transtype ::= IMMEDIATE",
- /*   7 */ "transtype ::= EXCLUSIVE",
+ /*   7 */ "transtype ::= ID",
  /*   8 */ "cmd ::= COMMIT|END trans_opt",
  /*   9 */ "cmd ::= ROLLBACK trans_opt",
  /*  10 */ "cmd ::= SAVEPOINT nm",
@@ -176180,98 +178845,98 @@ static void yy_destructor(
     ** inside the C code.
     */
 /********* Begin destructor definitions ***************************************/
-    case 205: /* select */
-    case 240: /* selectnowith */
-    case 241: /* oneselect */
-    case 253: /* values */
-    case 255: /* mvalues */
+    case 206: /* select */
+    case 241: /* selectnowith */
+    case 242: /* oneselect */
+    case 254: /* values */
+    case 256: /* mvalues */
 {
-sqlite3SelectDelete(pParse->db, (yypminor->yy555));
-}
-      break;
-    case 217: /* term */
-    case 218: /* expr */
-    case 247: /* where_opt */
-    case 249: /* having_opt */
-    case 269: /* where_opt_ret */
-    case 280: /* case_operand */
-    case 282: /* case_else */
-    case 285: /* vinto */
-    case 292: /* when_clause */
-    case 297: /* key_opt */
-    case 314: /* filter_clause */
+sqlite3SelectDelete(pParse->db, (yypminor->yy637));
+}
+      break;
+    case 218: /* term */
+    case 219: /* expr */
+    case 248: /* where_opt */
+    case 250: /* having_opt */
+    case 270: /* where_opt_ret */
+    case 281: /* case_operand */
+    case 283: /* case_else */
+    case 286: /* vinto */
+    case 293: /* when_clause */
+    case 298: /* key_opt */
+    case 315: /* filter_clause */
 {
-sqlite3ExprDelete(pParse->db, (yypminor->yy454));
-}
-      break;
-    case 222: /* eidlist_opt */
-    case 232: /* sortlist */
-    case 233: /* eidlist */
-    case 245: /* selcollist */
-    case 248: /* groupby_opt */
-    case 250: /* orderby_opt */
-    case 254: /* nexprlist */
-    case 256: /* sclp */
-    case 263: /* exprlist */
-    case 270: /* setlist */
-    case 279: /* paren_exprlist */
-    case 281: /* case_exprlist */
-    case 313: /* part_opt */
+sqlite3ExprDelete(pParse->db, (yypminor->yy590));
+}
+      break;
+    case 223: /* eidlist_opt */
+    case 233: /* sortlist */
+    case 234: /* eidlist */
+    case 246: /* selcollist */
+    case 249: /* groupby_opt */
+    case 251: /* orderby_opt */
+    case 255: /* nexprlist */
+    case 257: /* sclp */
+    case 264: /* exprlist */
+    case 271: /* setlist */
+    case 280: /* paren_exprlist */
+    case 282: /* case_exprlist */
+    case 314: /* part_opt */
 {
-sqlite3ExprListDelete(pParse->db, (yypminor->yy14));
+sqlite3ExprListDelete(pParse->db, (yypminor->yy402));
 }
       break;
-    case 239: /* fullname */
-    case 246: /* from */
-    case 258: /* seltablist */
-    case 259: /* stl_prefix */
-    case 264: /* xfullname */
+    case 240: /* fullname */
+    case 247: /* from */
+    case 259: /* seltablist */
+    case 260: /* stl_prefix */
+    case 265: /* xfullname */
 {
-sqlite3SrcListDelete(pParse->db, (yypminor->yy203));
+sqlite3SrcListDelete(pParse->db, (yypminor->yy563));
 }
       break;
-    case 242: /* wqlist */
+    case 243: /* wqlist */
 {
-sqlite3WithDelete(pParse->db, (yypminor->yy59));
+sqlite3WithDelete(pParse->db, (yypminor->yy125));
 }
       break;
-    case 252: /* window_clause */
-    case 309: /* windowdefn_list */
+    case 253: /* window_clause */
+    case 310: /* windowdefn_list */
 {
-sqlite3WindowListDelete(pParse->db, (yypminor->yy211));
+sqlite3WindowListDelete(pParse->db, (yypminor->yy483));
 }
       break;
-    case 265: /* idlist */
-    case 272: /* idlist_opt */
+    case 266: /* idlist */
+    case 273: /* idlist_opt */
 {
-sqlite3IdListDelete(pParse->db, (yypminor->yy132));
+sqlite3IdListDelete(pParse->db, (yypminor->yy204));
 }
       break;
-    case 275: /* filter_over */
-    case 310: /* windowdefn */
-    case 311: /* window */
-    case 312: /* frame_opt */
-    case 315: /* over_clause */
+    case 276: /* filter_over */
+    case 311: /* windowdefn */
+    case 312: /* window */
+    case 313: /* frame_opt */
+    case 316: /* over_clause */
 {
-sqlite3WindowDelete(pParse->db, (yypminor->yy211));
+sqlite3WindowDelete(pParse->db, (yypminor->yy483));
 }
       break;
-    case 288: /* trigger_cmd_list */
-    case 293: /* trigger_cmd */
+    case 289: /* trigger_cmd_list */
+    case 294: /* trigger_cmd */
 {
-sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427));
+sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy319));
 }
       break;
-    case 290: /* trigger_event */
+    case 291: /* trigger_event */
 {
-sqlite3IdListDelete(pParse->db, (yypminor->yy286).b);
+sqlite3IdListDelete(pParse->db, (yypminor->yy28).b);
 }
       break;
-    case 317: /* frame_bound */
-    case 318: /* frame_bound_s */
-    case 319: /* frame_bound_e */
+    case 318: /* frame_bound */
+    case 319: /* frame_bound_s */
+    case 320: /* frame_bound_e */
 {
-sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr);
+sqlite3ExprDelete(pParse->db, (yypminor->yy205).pExpr);
 }
       break;
 /********* End destructor definitions *****************************************/
@@ -176573,415 +179238,415 @@ static void yy_shift(
 /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
 ** of that rule */
 static const YYCODETYPE yyRuleInfoLhs[] = {
-   190,  /* (0) explain ::= EXPLAIN */
-   190,  /* (1) explain ::= EXPLAIN QUERY PLAN */
-   189,  /* (2) cmdx ::= cmd */
-   191,  /* (3) cmd ::= BEGIN transtype trans_opt */
-   192,  /* (4) transtype ::= */
-   192,  /* (5) transtype ::= DEFERRED */
-   192,  /* (6) transtype ::= IMMEDIATE */
-   192,  /* (7) transtype ::= EXCLUSIVE */
-   191,  /* (8) cmd ::= COMMIT|END trans_opt */
-   191,  /* (9) cmd ::= ROLLBACK trans_opt */
-   191,  /* (10) cmd ::= SAVEPOINT nm */
-   191,  /* (11) cmd ::= RELEASE savepoint_opt nm */
-   191,  /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */
-   196,  /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */
-   198,  /* (14) createkw ::= CREATE */
-   200,  /* (15) ifnotexists ::= */
-   200,  /* (16) ifnotexists ::= IF NOT EXISTS */
-   199,  /* (17) temp ::= TEMP */
-   199,  /* (18) temp ::= */
-   197,  /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */
-   197,  /* (20) create_table_args ::= AS select */
-   204,  /* (21) table_option_set ::= */
-   204,  /* (22) table_option_set ::= table_option_set COMMA table_option */
-   206,  /* (23) table_option ::= WITHOUT nm */
-   206,  /* (24) table_option ::= nm */
-   207,  /* (25) columnname ::= nm typetoken */
-   209,  /* (26) typetoken ::= */
-   209,  /* (27) typetoken ::= typename LP signed RP */
-   209,  /* (28) typetoken ::= typename LP signed COMMA signed RP */
-   210,  /* (29) typename ::= typename ID|STRING */
-   214,  /* (30) scanpt ::= */
-   215,  /* (31) scantok ::= */
-   216,  /* (32) ccons ::= CONSTRAINT nm */
-   216,  /* (33) ccons ::= DEFAULT scantok term */
-   216,  /* (34) ccons ::= DEFAULT LP expr RP */
-   216,  /* (35) ccons ::= DEFAULT PLUS scantok term */
-   216,  /* (36) ccons ::= DEFAULT MINUS scantok term */
-   216,  /* (37) ccons ::= DEFAULT scantok ID|INDEXED */
-   216,  /* (38) ccons ::= NOT NULL onconf */
-   216,  /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */
-   216,  /* (40) ccons ::= UNIQUE onconf */
-   216,  /* (41) ccons ::= CHECK LP expr RP */
-   216,  /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */
-   216,  /* (43) ccons ::= defer_subclause */
-   216,  /* (44) ccons ::= COLLATE ID|STRING */
-   225,  /* (45) generated ::= LP expr RP */
-   225,  /* (46) generated ::= LP expr RP ID */
-   221,  /* (47) autoinc ::= */
-   221,  /* (48) autoinc ::= AUTOINCR */
-   223,  /* (49) refargs ::= */
-   223,  /* (50) refargs ::= refargs refarg */
-   226,  /* (51) refarg ::= MATCH nm */
-   226,  /* (52) refarg ::= ON INSERT refact */
-   226,  /* (53) refarg ::= ON DELETE refact */
-   226,  /* (54) refarg ::= ON UPDATE refact */
-   227,  /* (55) refact ::= SET NULL */
-   227,  /* (56) refact ::= SET DEFAULT */
-   227,  /* (57) refact ::= CASCADE */
-   227,  /* (58) refact ::= RESTRICT */
-   227,  /* (59) refact ::= NO ACTION */
-   224,  /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */
-   224,  /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */
-   228,  /* (62) init_deferred_pred_opt ::= */
-   228,  /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */
-   228,  /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */
-   203,  /* (65) conslist_opt ::= */
-   230,  /* (66) tconscomma ::= COMMA */
-   231,  /* (67) tcons ::= CONSTRAINT nm */
-   231,  /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */
-   231,  /* (69) tcons ::= UNIQUE LP sortlist RP onconf */
-   231,  /* (70) tcons ::= CHECK LP expr RP onconf */
-   231,  /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */
-   234,  /* (72) defer_subclause_opt ::= */
-   219,  /* (73) onconf ::= */
-   219,  /* (74) onconf ::= ON CONFLICT resolvetype */
-   235,  /* (75) orconf ::= */
-   235,  /* (76) orconf ::= OR resolvetype */
-   236,  /* (77) resolvetype ::= IGNORE */
-   236,  /* (78) resolvetype ::= REPLACE */
-   191,  /* (79) cmd ::= DROP TABLE ifexists fullname */
-   238,  /* (80) ifexists ::= IF EXISTS */
-   238,  /* (81) ifexists ::= */
-   191,  /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */
-   191,  /* (83) cmd ::= DROP VIEW ifexists fullname */
-   191,  /* (84) cmd ::= select */
-   205,  /* (85) select ::= WITH wqlist selectnowith */
-   205,  /* (86) select ::= WITH RECURSIVE wqlist selectnowith */
-   205,  /* (87) select ::= selectnowith */
-   240,  /* (88) selectnowith ::= selectnowith multiselect_op oneselect */
-   243,  /* (89) multiselect_op ::= UNION */
-   243,  /* (90) multiselect_op ::= UNION ALL */
-   243,  /* (91) multiselect_op ::= EXCEPT|INTERSECT */
-   241,  /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */
-   241,  /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */
-   253,  /* (94) values ::= VALUES LP nexprlist RP */
-   241,  /* (95) oneselect ::= mvalues */
-   255,  /* (96) mvalues ::= values COMMA LP nexprlist RP */
-   255,  /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */
-   244,  /* (98) distinct ::= DISTINCT */
-   244,  /* (99) distinct ::= ALL */
-   244,  /* (100) distinct ::= */
-   256,  /* (101) sclp ::= */
-   245,  /* (102) selcollist ::= sclp scanpt expr scanpt as */
-   245,  /* (103) selcollist ::= sclp scanpt STAR */
-   245,  /* (104) selcollist ::= sclp scanpt nm DOT STAR */
-   257,  /* (105) as ::= AS nm */
-   257,  /* (106) as ::= */
-   246,  /* (107) from ::= */
-   246,  /* (108) from ::= FROM seltablist */
-   259,  /* (109) stl_prefix ::= seltablist joinop */
-   259,  /* (110) stl_prefix ::= */
-   258,  /* (111) seltablist ::= stl_prefix nm dbnm as on_using */
-   258,  /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */
-   258,  /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */
-   258,  /* (114) seltablist ::= stl_prefix LP select RP as on_using */
-   258,  /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */
-   201,  /* (116) dbnm ::= */
-   201,  /* (117) dbnm ::= DOT nm */
-   239,  /* (118) fullname ::= nm */
-   239,  /* (119) fullname ::= nm DOT nm */
-   264,  /* (120) xfullname ::= nm */
-   264,  /* (121) xfullname ::= nm DOT nm */
-   264,  /* (122) xfullname ::= nm DOT nm AS nm */
-   264,  /* (123) xfullname ::= nm AS nm */
-   260,  /* (124) joinop ::= COMMA|JOIN */
-   260,  /* (125) joinop ::= JOIN_KW JOIN */
-   260,  /* (126) joinop ::= JOIN_KW nm JOIN */
-   260,  /* (127) joinop ::= JOIN_KW nm nm JOIN */
-   261,  /* (128) on_using ::= ON expr */
-   261,  /* (129) on_using ::= USING LP idlist RP */
-   261,  /* (130) on_using ::= */
-   266,  /* (131) indexed_opt ::= */
-   262,  /* (132) indexed_by ::= INDEXED BY nm */
-   262,  /* (133) indexed_by ::= NOT INDEXED */
-   250,  /* (134) orderby_opt ::= */
-   250,  /* (135) orderby_opt ::= ORDER BY sortlist */
-   232,  /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */
-   232,  /* (137) sortlist ::= expr sortorder nulls */
-   220,  /* (138) sortorder ::= ASC */
-   220,  /* (139) sortorder ::= DESC */
-   220,  /* (140) sortorder ::= */
-   267,  /* (141) nulls ::= NULLS FIRST */
-   267,  /* (142) nulls ::= NULLS LAST */
-   267,  /* (143) nulls ::= */
-   248,  /* (144) groupby_opt ::= */
-   248,  /* (145) groupby_opt ::= GROUP BY nexprlist */
-   249,  /* (146) having_opt ::= */
-   249,  /* (147) having_opt ::= HAVING expr */
-   251,  /* (148) limit_opt ::= */
-   251,  /* (149) limit_opt ::= LIMIT expr */
-   251,  /* (150) limit_opt ::= LIMIT expr OFFSET expr */
-   251,  /* (151) limit_opt ::= LIMIT expr COMMA expr */
-   191,  /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret orderby_opt limit_opt */
-   247,  /* (153) where_opt ::= */
-   247,  /* (154) where_opt ::= WHERE expr */
-   269,  /* (155) where_opt_ret ::= */
-   269,  /* (156) where_opt_ret ::= WHERE expr */
-   269,  /* (157) where_opt_ret ::= RETURNING selcollist */
-   269,  /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */
-   191,  /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret orderby_opt limit_opt */
-   270,  /* (160) setlist ::= setlist COMMA nm EQ expr */
-   270,  /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */
-   270,  /* (162) setlist ::= nm EQ expr */
-   270,  /* (163) setlist ::= LP idlist RP EQ expr */
-   191,  /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */
-   191,  /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */
-   273,  /* (166) upsert ::= */
-   273,  /* (167) upsert ::= RETURNING selcollist */
-   273,  /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */
-   273,  /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */
-   273,  /* (170) upsert ::= ON CONFLICT DO NOTHING returning */
-   273,  /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */
-   274,  /* (172) returning ::= RETURNING selcollist */
-   271,  /* (173) insert_cmd ::= INSERT orconf */
-   271,  /* (174) insert_cmd ::= REPLACE */
-   272,  /* (175) idlist_opt ::= */
-   272,  /* (176) idlist_opt ::= LP idlist RP */
-   265,  /* (177) idlist ::= idlist COMMA nm */
-   265,  /* (178) idlist ::= nm */
-   218,  /* (179) expr ::= LP expr RP */
-   218,  /* (180) expr ::= ID|INDEXED|JOIN_KW */
-   218,  /* (181) expr ::= nm DOT nm */
-   218,  /* (182) expr ::= nm DOT nm DOT nm */
-   217,  /* (183) term ::= NULL|FLOAT|BLOB */
-   217,  /* (184) term ::= STRING */
-   217,  /* (185) term ::= INTEGER */
-   218,  /* (186) expr ::= VARIABLE */
-   218,  /* (187) expr ::= expr COLLATE ID|STRING */
-   218,  /* (188) expr ::= CAST LP expr AS typetoken RP */
-   218,  /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
-   218,  /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */
-   218,  /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
-   218,  /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
-   218,  /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */
-   218,  /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
-   217,  /* (195) term ::= CTIME_KW */
-   218,  /* (196) expr ::= LP nexprlist COMMA expr RP */
-   218,  /* (197) expr ::= expr AND expr */
-   218,  /* (198) expr ::= expr OR expr */
-   218,  /* (199) expr ::= expr LT|GT|GE|LE expr */
-   218,  /* (200) expr ::= expr EQ|NE expr */
-   218,  /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
-   218,  /* (202) expr ::= expr PLUS|MINUS expr */
-   218,  /* (203) expr ::= expr STAR|SLASH|REM expr */
-   218,  /* (204) expr ::= expr CONCAT expr */
-   276,  /* (205) likeop ::= NOT LIKE_KW|MATCH */
-   218,  /* (206) expr ::= expr likeop expr */
-   218,  /* (207) expr ::= expr likeop expr ESCAPE expr */
-   218,  /* (208) expr ::= expr ISNULL|NOTNULL */
-   218,  /* (209) expr ::= expr NOT NULL */
-   218,  /* (210) expr ::= expr IS expr */
-   218,  /* (211) expr ::= expr IS NOT expr */
-   218,  /* (212) expr ::= expr IS NOT DISTINCT FROM expr */
-   218,  /* (213) expr ::= expr IS DISTINCT FROM expr */
-   218,  /* (214) expr ::= NOT expr */
-   218,  /* (215) expr ::= BITNOT expr */
-   218,  /* (216) expr ::= PLUS|MINUS expr */
-   218,  /* (217) expr ::= expr PTR expr */
-   277,  /* (218) between_op ::= BETWEEN */
-   277,  /* (219) between_op ::= NOT BETWEEN */
-   218,  /* (220) expr ::= expr between_op expr AND expr */
-   278,  /* (221) in_op ::= IN */
-   278,  /* (222) in_op ::= NOT IN */
-   218,  /* (223) expr ::= expr in_op LP exprlist RP */
-   218,  /* (224) expr ::= LP select RP */
-   218,  /* (225) expr ::= expr in_op LP select RP */
-   218,  /* (226) expr ::= expr in_op nm dbnm paren_exprlist */
-   218,  /* (227) expr ::= EXISTS LP select RP */
-   218,  /* (228) expr ::= CASE case_operand case_exprlist case_else END */
-   281,  /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */
-   281,  /* (230) case_exprlist ::= WHEN expr THEN expr */
-   282,  /* (231) case_else ::= ELSE expr */
-   282,  /* (232) case_else ::= */
-   280,  /* (233) case_operand ::= */
-   263,  /* (234) exprlist ::= */
-   254,  /* (235) nexprlist ::= nexprlist COMMA expr */
-   254,  /* (236) nexprlist ::= expr */
-   279,  /* (237) paren_exprlist ::= */
-   279,  /* (238) paren_exprlist ::= LP exprlist RP */
-   191,  /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
-   283,  /* (240) uniqueflag ::= UNIQUE */
-   283,  /* (241) uniqueflag ::= */
-   222,  /* (242) eidlist_opt ::= */
-   222,  /* (243) eidlist_opt ::= LP eidlist RP */
-   233,  /* (244) eidlist ::= eidlist COMMA nm collate sortorder */
-   233,  /* (245) eidlist ::= nm collate sortorder */
-   284,  /* (246) collate ::= */
-   284,  /* (247) collate ::= COLLATE ID|STRING */
-   191,  /* (248) cmd ::= DROP INDEX ifexists fullname */
-   191,  /* (249) cmd ::= VACUUM vinto */
-   191,  /* (250) cmd ::= VACUUM nm vinto */
-   285,  /* (251) vinto ::= INTO expr */
-   285,  /* (252) vinto ::= */
-   191,  /* (253) cmd ::= PRAGMA nm dbnm */
-   191,  /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */
-   191,  /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */
-   191,  /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */
-   191,  /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */
-   212,  /* (258) plus_num ::= PLUS INTEGER|FLOAT */
-   213,  /* (259) minus_num ::= MINUS INTEGER|FLOAT */
-   191,  /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
-   287,  /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
-   289,  /* (262) trigger_time ::= BEFORE|AFTER */
-   289,  /* (263) trigger_time ::= INSTEAD OF */
-   289,  /* (264) trigger_time ::= */
-   290,  /* (265) trigger_event ::= DELETE|INSERT */
-   290,  /* (266) trigger_event ::= UPDATE */
-   290,  /* (267) trigger_event ::= UPDATE OF idlist */
-   292,  /* (268) when_clause ::= */
-   292,  /* (269) when_clause ::= WHEN expr */
-   288,  /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
-   288,  /* (271) trigger_cmd_list ::= trigger_cmd SEMI */
-   294,  /* (272) trnm ::= nm DOT nm */
-   295,  /* (273) tridxby ::= INDEXED BY nm */
-   295,  /* (274) tridxby ::= NOT INDEXED */
-   293,  /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
-   293,  /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
-   293,  /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
-   293,  /* (278) trigger_cmd ::= scanpt select scanpt */
-   218,  /* (279) expr ::= RAISE LP IGNORE RP */
-   218,  /* (280) expr ::= RAISE LP raisetype COMMA expr RP */
-   237,  /* (281) raisetype ::= ROLLBACK */
-   237,  /* (282) raisetype ::= ABORT */
-   237,  /* (283) raisetype ::= FAIL */
-   191,  /* (284) cmd ::= DROP TRIGGER ifexists fullname */
-   191,  /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
-   191,  /* (286) cmd ::= DETACH database_kw_opt expr */
-   297,  /* (287) key_opt ::= */
-   297,  /* (288) key_opt ::= KEY expr */
-   191,  /* (289) cmd ::= REINDEX */
-   191,  /* (290) cmd ::= REINDEX nm dbnm */
-   191,  /* (291) cmd ::= ANALYZE */
-   191,  /* (292) cmd ::= ANALYZE nm dbnm */
-   191,  /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */
-   191,  /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
-   191,  /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
-   298,  /* (296) add_column_fullname ::= fullname */
-   191,  /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
-   191,  /* (298) cmd ::= create_vtab */
-   191,  /* (299) cmd ::= create_vtab LP vtabarglist RP */
-   300,  /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
-   302,  /* (301) vtabarg ::= */
-   303,  /* (302) vtabargtoken ::= ANY */
-   303,  /* (303) vtabargtoken ::= lp anylist RP */
-   304,  /* (304) lp ::= LP */
-   268,  /* (305) with ::= WITH wqlist */
-   268,  /* (306) with ::= WITH RECURSIVE wqlist */
-   307,  /* (307) wqas ::= AS */
-   307,  /* (308) wqas ::= AS MATERIALIZED */
-   307,  /* (309) wqas ::= AS NOT MATERIALIZED */
-   306,  /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */
-   308,  /* (311) withnm ::= nm */
-   242,  /* (312) wqlist ::= wqitem */
-   242,  /* (313) wqlist ::= wqlist COMMA wqitem */
-   309,  /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */
-   310,  /* (315) windowdefn ::= nm AS LP window RP */
-   311,  /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
-   311,  /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
-   311,  /* (318) window ::= ORDER BY sortlist frame_opt */
-   311,  /* (319) window ::= nm ORDER BY sortlist frame_opt */
-   311,  /* (320) window ::= nm frame_opt */
-   312,  /* (321) frame_opt ::= */
-   312,  /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
-   312,  /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
-   316,  /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */
-   318,  /* (325) frame_bound_s ::= frame_bound */
-   318,  /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */
-   319,  /* (327) frame_bound_e ::= frame_bound */
-   319,  /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */
-   317,  /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */
-   317,  /* (330) frame_bound ::= CURRENT ROW */
-   320,  /* (331) frame_exclude_opt ::= */
-   320,  /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */
-   321,  /* (333) frame_exclude ::= NO OTHERS */
-   321,  /* (334) frame_exclude ::= CURRENT ROW */
-   321,  /* (335) frame_exclude ::= GROUP|TIES */
-   252,  /* (336) window_clause ::= WINDOW windowdefn_list */
-   275,  /* (337) filter_over ::= filter_clause over_clause */
-   275,  /* (338) filter_over ::= over_clause */
-   275,  /* (339) filter_over ::= filter_clause */
-   315,  /* (340) over_clause ::= OVER LP window RP */
-   315,  /* (341) over_clause ::= OVER nm */
-   314,  /* (342) filter_clause ::= FILTER LP WHERE expr RP */
-   217,  /* (343) term ::= QNUMBER */
-   186,  /* (344) input ::= cmdlist */
-   187,  /* (345) cmdlist ::= cmdlist ecmd */
-   187,  /* (346) cmdlist ::= ecmd */
-   188,  /* (347) ecmd ::= SEMI */
-   188,  /* (348) ecmd ::= cmdx SEMI */
-   188,  /* (349) ecmd ::= explain cmdx SEMI */
-   193,  /* (350) trans_opt ::= */
-   193,  /* (351) trans_opt ::= TRANSACTION */
-   193,  /* (352) trans_opt ::= TRANSACTION nm */
-   195,  /* (353) savepoint_opt ::= SAVEPOINT */
-   195,  /* (354) savepoint_opt ::= */
-   191,  /* (355) cmd ::= create_table create_table_args */
-   204,  /* (356) table_option_set ::= table_option */
-   202,  /* (357) columnlist ::= columnlist COMMA columnname carglist */
-   202,  /* (358) columnlist ::= columnname carglist */
-   194,  /* (359) nm ::= ID|INDEXED|JOIN_KW */
-   194,  /* (360) nm ::= STRING */
-   209,  /* (361) typetoken ::= typename */
-   210,  /* (362) typename ::= ID|STRING */
-   211,  /* (363) signed ::= plus_num */
-   211,  /* (364) signed ::= minus_num */
-   208,  /* (365) carglist ::= carglist ccons */
-   208,  /* (366) carglist ::= */
-   216,  /* (367) ccons ::= NULL onconf */
-   216,  /* (368) ccons ::= GENERATED ALWAYS AS generated */
-   216,  /* (369) ccons ::= AS generated */
-   203,  /* (370) conslist_opt ::= COMMA conslist */
-   229,  /* (371) conslist ::= conslist tconscomma tcons */
-   229,  /* (372) conslist ::= tcons */
-   230,  /* (373) tconscomma ::= */
-   234,  /* (374) defer_subclause_opt ::= defer_subclause */
-   236,  /* (375) resolvetype ::= raisetype */
-   240,  /* (376) selectnowith ::= oneselect */
-   241,  /* (377) oneselect ::= values */
-   256,  /* (378) sclp ::= selcollist COMMA */
-   257,  /* (379) as ::= ID|STRING */
-   266,  /* (380) indexed_opt ::= indexed_by */
-   274,  /* (381) returning ::= */
-   218,  /* (382) expr ::= term */
-   276,  /* (383) likeop ::= LIKE_KW|MATCH */
-   280,  /* (384) case_operand ::= expr */
-   263,  /* (385) exprlist ::= nexprlist */
-   286,  /* (386) nmnum ::= plus_num */
-   286,  /* (387) nmnum ::= nm */
-   286,  /* (388) nmnum ::= ON */
-   286,  /* (389) nmnum ::= DELETE */
-   286,  /* (390) nmnum ::= DEFAULT */
-   212,  /* (391) plus_num ::= INTEGER|FLOAT */
-   291,  /* (392) foreach_clause ::= */
-   291,  /* (393) foreach_clause ::= FOR EACH ROW */
-   294,  /* (394) trnm ::= nm */
-   295,  /* (395) tridxby ::= */
-   296,  /* (396) database_kw_opt ::= DATABASE */
-   296,  /* (397) database_kw_opt ::= */
-   299,  /* (398) kwcolumn_opt ::= */
-   299,  /* (399) kwcolumn_opt ::= COLUMNKW */
-   301,  /* (400) vtabarglist ::= vtabarg */
-   301,  /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */
-   302,  /* (402) vtabarg ::= vtabarg vtabargtoken */
-   305,  /* (403) anylist ::= */
-   305,  /* (404) anylist ::= anylist LP anylist RP */
-   305,  /* (405) anylist ::= anylist ANY */
-   268,  /* (406) with ::= */
-   309,  /* (407) windowdefn_list ::= windowdefn */
-   311,  /* (408) window ::= frame_opt */
+   191,  /* (0) explain ::= EXPLAIN */
+   191,  /* (1) explain ::= EXPLAIN QUERY PLAN */
+   190,  /* (2) cmdx ::= cmd */
+   192,  /* (3) cmd ::= BEGIN transtype trans_opt */
+   193,  /* (4) transtype ::= */
+   193,  /* (5) transtype ::= DEFERRED */
+   193,  /* (6) transtype ::= IMMEDIATE */
+   193,  /* (7) transtype ::= ID */
+   192,  /* (8) cmd ::= COMMIT|END trans_opt */
+   192,  /* (9) cmd ::= ROLLBACK trans_opt */
+   192,  /* (10) cmd ::= SAVEPOINT nm */
+   192,  /* (11) cmd ::= RELEASE savepoint_opt nm */
+   192,  /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */
+   197,  /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */
+   199,  /* (14) createkw ::= CREATE */
+   201,  /* (15) ifnotexists ::= */
+   201,  /* (16) ifnotexists ::= IF NOT EXISTS */
+   200,  /* (17) temp ::= TEMP */
+   200,  /* (18) temp ::= */
+   198,  /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */
+   198,  /* (20) create_table_args ::= AS select */
+   205,  /* (21) table_option_set ::= */
+   205,  /* (22) table_option_set ::= table_option_set COMMA table_option */
+   207,  /* (23) table_option ::= WITHOUT nm */
+   207,  /* (24) table_option ::= nm */
+   208,  /* (25) columnname ::= nm typetoken */
+   210,  /* (26) typetoken ::= */
+   210,  /* (27) typetoken ::= typename LP signed RP */
+   210,  /* (28) typetoken ::= typename LP signed COMMA signed RP */
+   211,  /* (29) typename ::= typename ID|STRING */
+   215,  /* (30) scanpt ::= */
+   216,  /* (31) scantok ::= */
+   217,  /* (32) ccons ::= CONSTRAINT nm */
+   217,  /* (33) ccons ::= DEFAULT scantok term */
+   217,  /* (34) ccons ::= DEFAULT LP expr RP */
+   217,  /* (35) ccons ::= DEFAULT PLUS scantok term */
+   217,  /* (36) ccons ::= DEFAULT MINUS scantok term */
+   217,  /* (37) ccons ::= DEFAULT scantok ID|INDEXED */
+   217,  /* (38) ccons ::= NOT NULL onconf */
+   217,  /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */
+   217,  /* (40) ccons ::= UNIQUE onconf */
+   217,  /* (41) ccons ::= CHECK LP expr RP */
+   217,  /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */
+   217,  /* (43) ccons ::= defer_subclause */
+   217,  /* (44) ccons ::= COLLATE ID|STRING */
+   226,  /* (45) generated ::= LP expr RP */
+   226,  /* (46) generated ::= LP expr RP ID */
+   222,  /* (47) autoinc ::= */
+   222,  /* (48) autoinc ::= AUTOINCR */
+   224,  /* (49) refargs ::= */
+   224,  /* (50) refargs ::= refargs refarg */
+   227,  /* (51) refarg ::= MATCH nm */
+   227,  /* (52) refarg ::= ON INSERT refact */
+   227,  /* (53) refarg ::= ON DELETE refact */
+   227,  /* (54) refarg ::= ON UPDATE refact */
+   228,  /* (55) refact ::= SET NULL */
+   228,  /* (56) refact ::= SET DEFAULT */
+   228,  /* (57) refact ::= CASCADE */
+   228,  /* (58) refact ::= RESTRICT */
+   228,  /* (59) refact ::= NO ACTION */
+   225,  /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */
+   225,  /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */
+   229,  /* (62) init_deferred_pred_opt ::= */
+   229,  /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */
+   229,  /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */
+   204,  /* (65) conslist_opt ::= */
+   231,  /* (66) tconscomma ::= COMMA */
+   232,  /* (67) tcons ::= CONSTRAINT nm */
+   232,  /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */
+   232,  /* (69) tcons ::= UNIQUE LP sortlist RP onconf */
+   232,  /* (70) tcons ::= CHECK LP expr RP onconf */
+   232,  /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */
+   235,  /* (72) defer_subclause_opt ::= */
+   220,  /* (73) onconf ::= */
+   220,  /* (74) onconf ::= ON CONFLICT resolvetype */
+   236,  /* (75) orconf ::= */
+   236,  /* (76) orconf ::= OR resolvetype */
+   237,  /* (77) resolvetype ::= IGNORE */
+   237,  /* (78) resolvetype ::= REPLACE */
+   192,  /* (79) cmd ::= DROP TABLE ifexists fullname */
+   239,  /* (80) ifexists ::= IF EXISTS */
+   239,  /* (81) ifexists ::= */
+   192,  /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */
+   192,  /* (83) cmd ::= DROP VIEW ifexists fullname */
+   192,  /* (84) cmd ::= select */
+   206,  /* (85) select ::= WITH wqlist selectnowith */
+   206,  /* (86) select ::= WITH RECURSIVE wqlist selectnowith */
+   206,  /* (87) select ::= selectnowith */
+   241,  /* (88) selectnowith ::= selectnowith multiselect_op oneselect */
+   244,  /* (89) multiselect_op ::= UNION */
+   244,  /* (90) multiselect_op ::= UNION ALL */
+   244,  /* (91) multiselect_op ::= EXCEPT|INTERSECT */
+   242,  /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */
+   242,  /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */
+   254,  /* (94) values ::= VALUES LP nexprlist RP */
+   242,  /* (95) oneselect ::= mvalues */
+   256,  /* (96) mvalues ::= values COMMA LP nexprlist RP */
+   256,  /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */
+   245,  /* (98) distinct ::= DISTINCT */
+   245,  /* (99) distinct ::= ALL */
+   245,  /* (100) distinct ::= */
+   257,  /* (101) sclp ::= */
+   246,  /* (102) selcollist ::= sclp scanpt expr scanpt as */
+   246,  /* (103) selcollist ::= sclp scanpt STAR */
+   246,  /* (104) selcollist ::= sclp scanpt nm DOT STAR */
+   258,  /* (105) as ::= AS nm */
+   258,  /* (106) as ::= */
+   247,  /* (107) from ::= */
+   247,  /* (108) from ::= FROM seltablist */
+   260,  /* (109) stl_prefix ::= seltablist joinop */
+   260,  /* (110) stl_prefix ::= */
+   259,  /* (111) seltablist ::= stl_prefix nm dbnm as on_using */
+   259,  /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */
+   259,  /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */
+   259,  /* (114) seltablist ::= stl_prefix LP select RP as on_using */
+   259,  /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */
+   202,  /* (116) dbnm ::= */
+   202,  /* (117) dbnm ::= DOT nm */
+   240,  /* (118) fullname ::= nm */
+   240,  /* (119) fullname ::= nm DOT nm */
+   265,  /* (120) xfullname ::= nm */
+   265,  /* (121) xfullname ::= nm DOT nm */
+   265,  /* (122) xfullname ::= nm DOT nm AS nm */
+   265,  /* (123) xfullname ::= nm AS nm */
+   261,  /* (124) joinop ::= COMMA|JOIN */
+   261,  /* (125) joinop ::= JOIN_KW JOIN */
+   261,  /* (126) joinop ::= JOIN_KW nm JOIN */
+   261,  /* (127) joinop ::= JOIN_KW nm nm JOIN */
+   262,  /* (128) on_using ::= ON expr */
+   262,  /* (129) on_using ::= USING LP idlist RP */
+   262,  /* (130) on_using ::= */
+   267,  /* (131) indexed_opt ::= */
+   263,  /* (132) indexed_by ::= INDEXED BY nm */
+   263,  /* (133) indexed_by ::= NOT INDEXED */
+   251,  /* (134) orderby_opt ::= */
+   251,  /* (135) orderby_opt ::= ORDER BY sortlist */
+   233,  /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */
+   233,  /* (137) sortlist ::= expr sortorder nulls */
+   221,  /* (138) sortorder ::= ASC */
+   221,  /* (139) sortorder ::= DESC */
+   221,  /* (140) sortorder ::= */
+   268,  /* (141) nulls ::= NULLS FIRST */
+   268,  /* (142) nulls ::= NULLS LAST */
+   268,  /* (143) nulls ::= */
+   249,  /* (144) groupby_opt ::= */
+   249,  /* (145) groupby_opt ::= GROUP BY nexprlist */
+   250,  /* (146) having_opt ::= */
+   250,  /* (147) having_opt ::= HAVING expr */
+   252,  /* (148) limit_opt ::= */
+   252,  /* (149) limit_opt ::= LIMIT expr */
+   252,  /* (150) limit_opt ::= LIMIT expr OFFSET expr */
+   252,  /* (151) limit_opt ::= LIMIT expr COMMA expr */
+   192,  /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret orderby_opt limit_opt */
+   248,  /* (153) where_opt ::= */
+   248,  /* (154) where_opt ::= WHERE expr */
+   270,  /* (155) where_opt_ret ::= */
+   270,  /* (156) where_opt_ret ::= WHERE expr */
+   270,  /* (157) where_opt_ret ::= RETURNING selcollist */
+   270,  /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */
+   192,  /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret orderby_opt limit_opt */
+   271,  /* (160) setlist ::= setlist COMMA nm EQ expr */
+   271,  /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */
+   271,  /* (162) setlist ::= nm EQ expr */
+   271,  /* (163) setlist ::= LP idlist RP EQ expr */
+   192,  /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */
+   192,  /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */
+   274,  /* (166) upsert ::= */
+   274,  /* (167) upsert ::= RETURNING selcollist */
+   274,  /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */
+   274,  /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */
+   274,  /* (170) upsert ::= ON CONFLICT DO NOTHING returning */
+   274,  /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */
+   275,  /* (172) returning ::= RETURNING selcollist */
+   272,  /* (173) insert_cmd ::= INSERT orconf */
+   272,  /* (174) insert_cmd ::= REPLACE */
+   273,  /* (175) idlist_opt ::= */
+   273,  /* (176) idlist_opt ::= LP idlist RP */
+   266,  /* (177) idlist ::= idlist COMMA nm */
+   266,  /* (178) idlist ::= nm */
+   219,  /* (179) expr ::= LP expr RP */
+   219,  /* (180) expr ::= ID|INDEXED|JOIN_KW */
+   219,  /* (181) expr ::= nm DOT nm */
+   219,  /* (182) expr ::= nm DOT nm DOT nm */
+   218,  /* (183) term ::= NULL|FLOAT|BLOB */
+   218,  /* (184) term ::= STRING */
+   218,  /* (185) term ::= INTEGER */
+   219,  /* (186) expr ::= VARIABLE */
+   219,  /* (187) expr ::= expr COLLATE ID|STRING */
+   219,  /* (188) expr ::= CAST LP expr AS typetoken RP */
+   219,  /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
+   219,  /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */
+   219,  /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
+   219,  /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
+   219,  /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */
+   219,  /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
+   218,  /* (195) term ::= CTIME_KW */
+   219,  /* (196) expr ::= LP nexprlist COMMA expr RP */
+   219,  /* (197) expr ::= expr AND expr */
+   219,  /* (198) expr ::= expr OR expr */
+   219,  /* (199) expr ::= expr LT|GT|GE|LE expr */
+   219,  /* (200) expr ::= expr EQ|NE expr */
+   219,  /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
+   219,  /* (202) expr ::= expr PLUS|MINUS expr */
+   219,  /* (203) expr ::= expr STAR|SLASH|REM expr */
+   219,  /* (204) expr ::= expr CONCAT expr */
+   277,  /* (205) likeop ::= NOT LIKE_KW|MATCH */
+   219,  /* (206) expr ::= expr likeop expr */
+   219,  /* (207) expr ::= expr likeop expr ESCAPE expr */
+   219,  /* (208) expr ::= expr ISNULL|NOTNULL */
+   219,  /* (209) expr ::= expr NOT NULL */
+   219,  /* (210) expr ::= expr IS expr */
+   219,  /* (211) expr ::= expr IS NOT expr */
+   219,  /* (212) expr ::= expr IS NOT DISTINCT FROM expr */
+   219,  /* (213) expr ::= expr IS DISTINCT FROM expr */
+   219,  /* (214) expr ::= NOT expr */
+   219,  /* (215) expr ::= BITNOT expr */
+   219,  /* (216) expr ::= PLUS|MINUS expr */
+   219,  /* (217) expr ::= expr PTR expr */
+   278,  /* (218) between_op ::= BETWEEN */
+   278,  /* (219) between_op ::= NOT BETWEEN */
+   219,  /* (220) expr ::= expr between_op expr AND expr */
+   279,  /* (221) in_op ::= IN */
+   279,  /* (222) in_op ::= NOT IN */
+   219,  /* (223) expr ::= expr in_op LP exprlist RP */
+   219,  /* (224) expr ::= LP select RP */
+   219,  /* (225) expr ::= expr in_op LP select RP */
+   219,  /* (226) expr ::= expr in_op nm dbnm paren_exprlist */
+   219,  /* (227) expr ::= EXISTS LP select RP */
+   219,  /* (228) expr ::= CASE case_operand case_exprlist case_else END */
+   282,  /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */
+   282,  /* (230) case_exprlist ::= WHEN expr THEN expr */
+   283,  /* (231) case_else ::= ELSE expr */
+   283,  /* (232) case_else ::= */
+   281,  /* (233) case_operand ::= */
+   264,  /* (234) exprlist ::= */
+   255,  /* (235) nexprlist ::= nexprlist COMMA expr */
+   255,  /* (236) nexprlist ::= expr */
+   280,  /* (237) paren_exprlist ::= */
+   280,  /* (238) paren_exprlist ::= LP exprlist RP */
+   192,  /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
+   284,  /* (240) uniqueflag ::= UNIQUE */
+   284,  /* (241) uniqueflag ::= */
+   223,  /* (242) eidlist_opt ::= */
+   223,  /* (243) eidlist_opt ::= LP eidlist RP */
+   234,  /* (244) eidlist ::= eidlist COMMA nm collate sortorder */
+   234,  /* (245) eidlist ::= nm collate sortorder */
+   285,  /* (246) collate ::= */
+   285,  /* (247) collate ::= COLLATE ID|STRING */
+   192,  /* (248) cmd ::= DROP INDEX ifexists fullname */
+   192,  /* (249) cmd ::= VACUUM vinto */
+   192,  /* (250) cmd ::= VACUUM nm vinto */
+   286,  /* (251) vinto ::= INTO expr */
+   286,  /* (252) vinto ::= */
+   192,  /* (253) cmd ::= PRAGMA nm dbnm */
+   192,  /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */
+   192,  /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */
+   192,  /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */
+   192,  /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */
+   213,  /* (258) plus_num ::= PLUS INTEGER|FLOAT */
+   214,  /* (259) minus_num ::= MINUS INTEGER|FLOAT */
+   192,  /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
+   288,  /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
+   290,  /* (262) trigger_time ::= BEFORE|AFTER */
+   290,  /* (263) trigger_time ::= INSTEAD OF */
+   290,  /* (264) trigger_time ::= */
+   291,  /* (265) trigger_event ::= DELETE|INSERT */
+   291,  /* (266) trigger_event ::= UPDATE */
+   291,  /* (267) trigger_event ::= UPDATE OF idlist */
+   293,  /* (268) when_clause ::= */
+   293,  /* (269) when_clause ::= WHEN expr */
+   289,  /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
+   289,  /* (271) trigger_cmd_list ::= trigger_cmd SEMI */
+   295,  /* (272) trnm ::= nm DOT nm */
+   296,  /* (273) tridxby ::= INDEXED BY nm */
+   296,  /* (274) tridxby ::= NOT INDEXED */
+   294,  /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
+   294,  /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
+   294,  /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
+   294,  /* (278) trigger_cmd ::= scanpt select scanpt */
+   219,  /* (279) expr ::= RAISE LP IGNORE RP */
+   219,  /* (280) expr ::= RAISE LP raisetype COMMA expr RP */
+   238,  /* (281) raisetype ::= ROLLBACK */
+   238,  /* (282) raisetype ::= ABORT */
+   238,  /* (283) raisetype ::= FAIL */
+   192,  /* (284) cmd ::= DROP TRIGGER ifexists fullname */
+   192,  /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
+   192,  /* (286) cmd ::= DETACH database_kw_opt expr */
+   298,  /* (287) key_opt ::= */
+   298,  /* (288) key_opt ::= KEY expr */
+   192,  /* (289) cmd ::= REINDEX */
+   192,  /* (290) cmd ::= REINDEX nm dbnm */
+   192,  /* (291) cmd ::= ANALYZE */
+   192,  /* (292) cmd ::= ANALYZE nm dbnm */
+   192,  /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */
+   192,  /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
+   192,  /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
+   299,  /* (296) add_column_fullname ::= fullname */
+   192,  /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
+   192,  /* (298) cmd ::= create_vtab */
+   192,  /* (299) cmd ::= create_vtab LP vtabarglist RP */
+   301,  /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
+   303,  /* (301) vtabarg ::= */
+   304,  /* (302) vtabargtoken ::= ANY */
+   304,  /* (303) vtabargtoken ::= lp anylist RP */
+   305,  /* (304) lp ::= LP */
+   269,  /* (305) with ::= WITH wqlist */
+   269,  /* (306) with ::= WITH RECURSIVE wqlist */
+   308,  /* (307) wqas ::= AS */
+   308,  /* (308) wqas ::= AS MATERIALIZED */
+   308,  /* (309) wqas ::= AS NOT MATERIALIZED */
+   307,  /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */
+   309,  /* (311) withnm ::= nm */
+   243,  /* (312) wqlist ::= wqitem */
+   243,  /* (313) wqlist ::= wqlist COMMA wqitem */
+   310,  /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */
+   311,  /* (315) windowdefn ::= nm AS LP window RP */
+   312,  /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
+   312,  /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
+   312,  /* (318) window ::= ORDER BY sortlist frame_opt */
+   312,  /* (319) window ::= nm ORDER BY sortlist frame_opt */
+   312,  /* (320) window ::= nm frame_opt */
+   313,  /* (321) frame_opt ::= */
+   313,  /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
+   313,  /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
+   317,  /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */
+   319,  /* (325) frame_bound_s ::= frame_bound */
+   319,  /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */
+   320,  /* (327) frame_bound_e ::= frame_bound */
+   320,  /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */
+   318,  /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */
+   318,  /* (330) frame_bound ::= CURRENT ROW */
+   321,  /* (331) frame_exclude_opt ::= */
+   321,  /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */
+   322,  /* (333) frame_exclude ::= NO OTHERS */
+   322,  /* (334) frame_exclude ::= CURRENT ROW */
+   322,  /* (335) frame_exclude ::= GROUP|TIES */
+   253,  /* (336) window_clause ::= WINDOW windowdefn_list */
+   276,  /* (337) filter_over ::= filter_clause over_clause */
+   276,  /* (338) filter_over ::= over_clause */
+   276,  /* (339) filter_over ::= filter_clause */
+   316,  /* (340) over_clause ::= OVER LP window RP */
+   316,  /* (341) over_clause ::= OVER nm */
+   315,  /* (342) filter_clause ::= FILTER LP WHERE expr RP */
+   218,  /* (343) term ::= QNUMBER */
+   187,  /* (344) input ::= cmdlist */
+   188,  /* (345) cmdlist ::= cmdlist ecmd */
+   188,  /* (346) cmdlist ::= ecmd */
+   189,  /* (347) ecmd ::= SEMI */
+   189,  /* (348) ecmd ::= cmdx SEMI */
+   189,  /* (349) ecmd ::= explain cmdx SEMI */
+   194,  /* (350) trans_opt ::= */
+   194,  /* (351) trans_opt ::= TRANSACTION */
+   194,  /* (352) trans_opt ::= TRANSACTION nm */
+   196,  /* (353) savepoint_opt ::= SAVEPOINT */
+   196,  /* (354) savepoint_opt ::= */
+   192,  /* (355) cmd ::= create_table create_table_args */
+   205,  /* (356) table_option_set ::= table_option */
+   203,  /* (357) columnlist ::= columnlist COMMA columnname carglist */
+   203,  /* (358) columnlist ::= columnname carglist */
+   195,  /* (359) nm ::= ID|INDEXED|JOIN_KW */
+   195,  /* (360) nm ::= STRING */
+   210,  /* (361) typetoken ::= typename */
+   211,  /* (362) typename ::= ID|STRING */
+   212,  /* (363) signed ::= plus_num */
+   212,  /* (364) signed ::= minus_num */
+   209,  /* (365) carglist ::= carglist ccons */
+   209,  /* (366) carglist ::= */
+   217,  /* (367) ccons ::= NULL onconf */
+   217,  /* (368) ccons ::= GENERATED ALWAYS AS generated */
+   217,  /* (369) ccons ::= AS generated */
+   204,  /* (370) conslist_opt ::= COMMA conslist */
+   230,  /* (371) conslist ::= conslist tconscomma tcons */
+   230,  /* (372) conslist ::= tcons */
+   231,  /* (373) tconscomma ::= */
+   235,  /* (374) defer_subclause_opt ::= defer_subclause */
+   237,  /* (375) resolvetype ::= raisetype */
+   241,  /* (376) selectnowith ::= oneselect */
+   242,  /* (377) oneselect ::= values */
+   257,  /* (378) sclp ::= selcollist COMMA */
+   258,  /* (379) as ::= ID|STRING */
+   267,  /* (380) indexed_opt ::= indexed_by */
+   275,  /* (381) returning ::= */
+   219,  /* (382) expr ::= term */
+   277,  /* (383) likeop ::= LIKE_KW|MATCH */
+   281,  /* (384) case_operand ::= expr */
+   264,  /* (385) exprlist ::= nexprlist */
+   287,  /* (386) nmnum ::= plus_num */
+   287,  /* (387) nmnum ::= nm */
+   287,  /* (388) nmnum ::= ON */
+   287,  /* (389) nmnum ::= DELETE */
+   287,  /* (390) nmnum ::= DEFAULT */
+   213,  /* (391) plus_num ::= INTEGER|FLOAT */
+   292,  /* (392) foreach_clause ::= */
+   292,  /* (393) foreach_clause ::= FOR EACH ROW */
+   295,  /* (394) trnm ::= nm */
+   296,  /* (395) tridxby ::= */
+   297,  /* (396) database_kw_opt ::= DATABASE */
+   297,  /* (397) database_kw_opt ::= */
+   300,  /* (398) kwcolumn_opt ::= */
+   300,  /* (399) kwcolumn_opt ::= COLUMNKW */
+   302,  /* (400) vtabarglist ::= vtabarg */
+   302,  /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */
+   303,  /* (402) vtabarg ::= vtabarg vtabargtoken */
+   306,  /* (403) anylist ::= */
+   306,  /* (404) anylist ::= anylist LP anylist RP */
+   306,  /* (405) anylist ::= anylist ANY */
+   269,  /* (406) with ::= */
+   310,  /* (407) windowdefn_list ::= windowdefn */
+   312,  /* (408) window ::= frame_opt */
 };
 
 /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
@@ -176994,7 +179659,7 @@ static const signed char yyRuleInfoNRhs[] = {
     0,  /* (4) transtype ::= */
    -1,  /* (5) transtype ::= DEFERRED */
    -1,  /* (6) transtype ::= IMMEDIATE */
-   -1,  /* (7) transtype ::= EXCLUSIVE */
+   -1,  /* (7) transtype ::= ID */
    -2,  /* (8) cmd ::= COMMIT|END trans_opt */
    -2,  /* (9) cmd ::= ROLLBACK trans_opt */
    -2,  /* (10) cmd ::= SAVEPOINT nm */
@@ -177447,16 +180112,27 @@ static YYACTIONTYPE yy_reduce(
 { sqlite3FinishCoding(pParse); }
         break;
       case 3: /* cmd ::= BEGIN transtype trans_opt */
-{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);}
+{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy502);}
         break;
       case 4: /* transtype ::= */
-{yymsp[1].minor.yy144 = TK_DEFERRED;}
+{yymsp[1].minor.yy502 = TK_DEFERRED;}
         break;
       case 5: /* transtype ::= DEFERRED */
       case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6);
-      case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7);
       case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324);
-{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/}
+{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/}
+        break;
+      case 7: /* transtype ::= ID */
+{
+   Token *p = &yymsp[0].minor.yy0;
+   if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){
+     yymsp[0].minor.yy502 = TK_EXCLUSIVE;
+   }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){
+     yymsp[0].minor.yy502 = TK_CONCURRENT;  /*A-overwrites-X*/
+   }else{
+     parserSyntaxError(pParse, p);
+   }
+}
         break;
       case 8: /* cmd ::= COMMIT|END trans_opt */
       case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9);
@@ -177479,7 +180155,7 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */
 {
-   sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144);
+   sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy502,0,0,yymsp[-2].minor.yy502);
 }
         break;
       case 14: /* createkw ::= CREATE */
@@ -177493,38 +180169,38 @@ static YYACTIONTYPE yy_reduce(
       case 81: /* ifexists ::= */ yytestcase(yyruleno==81);
       case 100: /* distinct ::= */ yytestcase(yyruleno==100);
       case 246: /* collate ::= */ yytestcase(yyruleno==246);
-{yymsp[1].minor.yy144 = 0;}
+{yymsp[1].minor.yy502 = 0;}
         break;
       case 16: /* ifnotexists ::= IF NOT EXISTS */
-{yymsp[-2].minor.yy144 = 1;}
+{yymsp[-2].minor.yy502 = 1;}
         break;
       case 17: /* temp ::= TEMP */
-{yymsp[0].minor.yy144 = pParse->db->init.busy==0;}
+{yymsp[0].minor.yy502 = pParse->db->init.busy==0;}
         break;
       case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */
 {
-  sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0);
+  sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy9,0);
 }
         break;
       case 20: /* create_table_args ::= AS select */
 {
-  sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555);
-  sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555);
+  sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy637);
+  sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637);
 }
         break;
       case 21: /* table_option_set ::= */
-{yymsp[1].minor.yy391 = 0;}
+{yymsp[1].minor.yy9 = 0;}
         break;
       case 22: /* table_option_set ::= table_option_set COMMA table_option */
-{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;}
-  yymsp[-2].minor.yy391 = yylhsminor.yy391;
+{yylhsminor.yy9 = yymsp[-2].minor.yy9|yymsp[0].minor.yy9;}
+  yymsp[-2].minor.yy9 = yylhsminor.yy9;
         break;
       case 23: /* table_option ::= WITHOUT nm */
 {
   if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){
-    yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid;
+    yymsp[-1].minor.yy9 = TF_WithoutRowid | TF_NoVisibleRowid;
   }else{
-    yymsp[-1].minor.yy391 = 0;
+    yymsp[-1].minor.yy9 = 0;
     sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z);
   }
 }
@@ -177532,13 +180208,13 @@ static YYACTIONTYPE yy_reduce(
       case 24: /* table_option ::= nm */
 {
   if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){
-    yylhsminor.yy391 = TF_Strict;
+    yylhsminor.yy9 = TF_Strict;
   }else{
-    yylhsminor.yy391 = 0;
+    yylhsminor.yy9 = 0;
     sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z);
   }
 }
-  yymsp[0].minor.yy391 = yylhsminor.yy391;
+  yymsp[0].minor.yy9 = yylhsminor.yy9;
         break;
       case 25: /* columnname ::= nm typetoken */
 {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);}
@@ -177564,7 +180240,7 @@ static YYACTIONTYPE yy_reduce(
       case 30: /* scanpt ::= */
 {
   assert( yyLookahead!=YYNOCODE );
-  yymsp[1].minor.yy168 = yyLookaheadToken.z;
+  yymsp[1].minor.yy342 = yyLookaheadToken.z;
 }
         break;
       case 31: /* scantok ::= */
@@ -177578,17 +180254,17 @@ static YYACTIONTYPE yy_reduce(
 {pParse->constraintName = yymsp[0].minor.yy0;}
         break;
       case 33: /* ccons ::= DEFAULT scantok term */
-{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);}
+{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);}
         break;
       case 34: /* ccons ::= DEFAULT LP expr RP */
-{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);}
+{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);}
         break;
       case 35: /* ccons ::= DEFAULT PLUS scantok term */
-{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);}
+{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);}
         break;
       case 36: /* ccons ::= DEFAULT MINUS scantok term */
 {
-  Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0);
+  Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy590, 0);
   sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);
 }
         break;
@@ -177603,151 +180279,151 @@ static YYACTIONTYPE yy_reduce(
 }
         break;
       case 38: /* ccons ::= NOT NULL onconf */
-{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);}
+{sqlite3AddNotNull(pParse, yymsp[0].minor.yy502);}
         break;
       case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */
-{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);}
+{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy502,yymsp[0].minor.yy502,yymsp[-2].minor.yy502);}
         break;
       case 40: /* ccons ::= UNIQUE onconf */
-{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0,
+{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy502,0,0,0,0,
                                    SQLITE_IDXTYPE_UNIQUE);}
         break;
       case 41: /* ccons ::= CHECK LP expr RP */
-{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);}
+{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);}
         break;
       case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */
-{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);}
+{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy402,yymsp[0].minor.yy502);}
         break;
       case 43: /* ccons ::= defer_subclause */
-{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);}
+{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy502);}
         break;
       case 44: /* ccons ::= COLLATE ID|STRING */
 {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);}
         break;
       case 45: /* generated ::= LP expr RP */
-{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);}
+{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy590,0);}
         break;
       case 46: /* generated ::= LP expr RP ID */
-{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);}
+{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy590,&yymsp[0].minor.yy0);}
         break;
       case 48: /* autoinc ::= AUTOINCR */
-{yymsp[0].minor.yy144 = 1;}
+{yymsp[0].minor.yy502 = 1;}
         break;
       case 49: /* refargs ::= */
-{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */}
+{ yymsp[1].minor.yy502 = OE_None*0x0101; /* EV: R-19803-45884 */}
         break;
       case 50: /* refargs ::= refargs refarg */
-{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; }
+{ yymsp[-1].minor.yy502 = (yymsp[-1].minor.yy502 & ~yymsp[0].minor.yy481.mask) | yymsp[0].minor.yy481.value; }
         break;
       case 51: /* refarg ::= MATCH nm */
-{ yymsp[-1].minor.yy383.value = 0;     yymsp[-1].minor.yy383.mask = 0x000000; }
+{ yymsp[-1].minor.yy481.value = 0;     yymsp[-1].minor.yy481.mask = 0x000000; }
         break;
       case 52: /* refarg ::= ON INSERT refact */
-{ yymsp[-2].minor.yy383.value = 0;     yymsp[-2].minor.yy383.mask = 0x000000; }
+{ yymsp[-2].minor.yy481.value = 0;     yymsp[-2].minor.yy481.mask = 0x000000; }
         break;
       case 53: /* refarg ::= ON DELETE refact */
-{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144;     yymsp[-2].minor.yy383.mask = 0x0000ff; }
+{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502;     yymsp[-2].minor.yy481.mask = 0x0000ff; }
         break;
       case 54: /* refarg ::= ON UPDATE refact */
-{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8;  yymsp[-2].minor.yy383.mask = 0x00ff00; }
+{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502<<8;  yymsp[-2].minor.yy481.mask = 0x00ff00; }
         break;
       case 55: /* refact ::= SET NULL */
-{ yymsp[-1].minor.yy144 = OE_SetNull;  /* EV: R-33326-45252 */}
+{ yymsp[-1].minor.yy502 = OE_SetNull;  /* EV: R-33326-45252 */}
         break;
       case 56: /* refact ::= SET DEFAULT */
-{ yymsp[-1].minor.yy144 = OE_SetDflt;  /* EV: R-33326-45252 */}
+{ yymsp[-1].minor.yy502 = OE_SetDflt;  /* EV: R-33326-45252 */}
         break;
       case 57: /* refact ::= CASCADE */
-{ yymsp[0].minor.yy144 = OE_Cascade;  /* EV: R-33326-45252 */}
+{ yymsp[0].minor.yy502 = OE_Cascade;  /* EV: R-33326-45252 */}
         break;
       case 58: /* refact ::= RESTRICT */
-{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */}
+{ yymsp[0].minor.yy502 = OE_Restrict; /* EV: R-33326-45252 */}
         break;
       case 59: /* refact ::= NO ACTION */
-{ yymsp[-1].minor.yy144 = OE_None;     /* EV: R-33326-45252 */}
+{ yymsp[-1].minor.yy502 = OE_None;     /* EV: R-33326-45252 */}
         break;
       case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */
-{yymsp[-2].minor.yy144 = 0;}
+{yymsp[-2].minor.yy502 = 0;}
         break;
       case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */
       case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76);
       case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173);
-{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;}
+{yymsp[-1].minor.yy502 = yymsp[0].minor.yy502;}
         break;
       case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */
       case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80);
       case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219);
       case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222);
       case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247);
-{yymsp[-1].minor.yy144 = 1;}
+{yymsp[-1].minor.yy502 = 1;}
         break;
       case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */
-{yymsp[-1].minor.yy144 = 0;}
+{yymsp[-1].minor.yy502 = 0;}
         break;
       case 66: /* tconscomma ::= COMMA */
 {pParse->constraintName.n = 0;}
         break;
       case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */
-{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);}
+{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy402,yymsp[0].minor.yy502,yymsp[-2].minor.yy502,0);}
         break;
       case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */
-{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0,
+{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy402,yymsp[0].minor.yy502,0,0,0,0,
                                        SQLITE_IDXTYPE_UNIQUE);}
         break;
       case 70: /* tcons ::= CHECK LP expr RP onconf */
-{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);}
+{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy590,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);}
         break;
       case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */
 {
-    sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144);
-    sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144);
+    sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy402, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[-1].minor.yy502);
+    sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy502);
 }
         break;
       case 73: /* onconf ::= */
       case 75: /* orconf ::= */ yytestcase(yyruleno==75);
-{yymsp[1].minor.yy144 = OE_Default;}
+{yymsp[1].minor.yy502 = OE_Default;}
         break;
       case 74: /* onconf ::= ON CONFLICT resolvetype */
-{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;}
+{yymsp[-2].minor.yy502 = yymsp[0].minor.yy502;}
         break;
       case 77: /* resolvetype ::= IGNORE */
-{yymsp[0].minor.yy144 = OE_Ignore;}
+{yymsp[0].minor.yy502 = OE_Ignore;}
         break;
       case 78: /* resolvetype ::= REPLACE */
       case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174);
-{yymsp[0].minor.yy144 = OE_Replace;}
+{yymsp[0].minor.yy502 = OE_Replace;}
         break;
       case 79: /* cmd ::= DROP TABLE ifexists fullname */
 {
-  sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144);
+  sqlite3DropTable(pParse, yymsp[0].minor.yy563, 0, yymsp[-1].minor.yy502);
 }
         break;
       case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */
 {
-  sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144);
+  sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[0].minor.yy637, yymsp[-7].minor.yy502, yymsp[-5].minor.yy502);
 }
         break;
       case 83: /* cmd ::= DROP VIEW ifexists fullname */
 {
-  sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144);
+  sqlite3DropTable(pParse, yymsp[0].minor.yy563, 1, yymsp[-1].minor.yy502);
 }
         break;
       case 84: /* cmd ::= select */
 {
   SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0};
-  sqlite3Select(pParse, yymsp[0].minor.yy555, &dest);
-  sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555);
+  sqlite3Select(pParse, yymsp[0].minor.yy637, &dest);
+  sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637);
 }
         break;
       case 85: /* select ::= WITH wqlist selectnowith */
-{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);}
+{yymsp[-2].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);}
         break;
       case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */
-{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);}
+{yymsp[-3].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);}
         break;
       case 87: /* select ::= selectnowith */
 {
-  Select *p = yymsp[0].minor.yy555;
+  Select *p = yymsp[0].minor.yy637;
   if( p ){
     parserDoubleLinkSelect(pParse, p);
   }
@@ -177755,8 +180431,8 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */
 {
-  Select *pRhs = yymsp[0].minor.yy555;
-  Select *pLhs = yymsp[-2].minor.yy555;
+  Select *pRhs = yymsp[0].minor.yy637;
+  Select *pLhs = yymsp[-2].minor.yy637;
   if( pRhs && pRhs->pPrior ){
     SrcList *pFrom;
     Token x;
@@ -177766,60 +180442,60 @@ static YYACTIONTYPE yy_reduce(
     pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0);
   }
   if( pRhs ){
-    pRhs->op = (u8)yymsp[-1].minor.yy144;
+    pRhs->op = (u8)yymsp[-1].minor.yy502;
     pRhs->pPrior = pLhs;
     if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue;
     pRhs->selFlags &= ~SF_MultiValue;
-    if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1;
+    if( yymsp[-1].minor.yy502!=TK_ALL ) pParse->hasCompound = 1;
   }else{
     sqlite3SelectDelete(pParse->db, pLhs);
   }
-  yymsp[-2].minor.yy555 = pRhs;
+  yymsp[-2].minor.yy637 = pRhs;
 }
         break;
       case 89: /* multiselect_op ::= UNION */
       case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91);
-{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/}
+{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-OP*/}
         break;
       case 90: /* multiselect_op ::= UNION ALL */
-{yymsp[-1].minor.yy144 = TK_ALL;}
+{yymsp[-1].minor.yy502 = TK_ALL;}
         break;
       case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */
 {
-  yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454);
+  yymsp[-8].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy402,yymsp[-5].minor.yy563,yymsp[-4].minor.yy590,yymsp[-3].minor.yy402,yymsp[-2].minor.yy590,yymsp[-1].minor.yy402,yymsp[-7].minor.yy502,yymsp[0].minor.yy590);
 }
         break;
       case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */
 {
-  yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454);
-  if( yymsp[-9].minor.yy555 ){
-    yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211;
+  yymsp[-9].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy402,yymsp[-6].minor.yy563,yymsp[-5].minor.yy590,yymsp[-4].minor.yy402,yymsp[-3].minor.yy590,yymsp[-1].minor.yy402,yymsp[-8].minor.yy502,yymsp[0].minor.yy590);
+  if( yymsp[-9].minor.yy637 ){
+    yymsp[-9].minor.yy637->pWinDefn = yymsp[-2].minor.yy483;
   }else{
-    sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211);
+    sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy483);
   }
 }
         break;
       case 94: /* values ::= VALUES LP nexprlist RP */
 {
-  yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0);
+  yymsp[-3].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy402,0,0,0,0,0,SF_Values,0);
 }
         break;
       case 95: /* oneselect ::= mvalues */
 {
-  sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555);
+  sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy637);
 }
         break;
       case 96: /* mvalues ::= values COMMA LP nexprlist RP */
       case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97);
 {
-  yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14);
+  yymsp[-4].minor.yy637 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy637, yymsp[-1].minor.yy402);
 }
         break;
       case 98: /* distinct ::= DISTINCT */
-{yymsp[0].minor.yy144 = SF_Distinct;}
+{yymsp[0].minor.yy502 = SF_Distinct;}
         break;
       case 99: /* distinct ::= ALL */
-{yymsp[0].minor.yy144 = SF_All;}
+{yymsp[0].minor.yy502 = SF_All;}
         break;
       case 101: /* sclp ::= */
       case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134);
@@ -177827,20 +180503,20 @@ static YYACTIONTYPE yy_reduce(
       case 234: /* exprlist ::= */ yytestcase(yyruleno==234);
       case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237);
       case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242);
-{yymsp[1].minor.yy14 = 0;}
+{yymsp[1].minor.yy402 = 0;}
         break;
       case 102: /* selcollist ::= sclp scanpt expr scanpt as */
 {
-   yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454);
-   if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1);
-   sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168);
+   yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[-2].minor.yy590);
+   if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[0].minor.yy0, 1);
+   sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy402,yymsp[-3].minor.yy342,yymsp[-1].minor.yy342);
 }
         break;
       case 103: /* selcollist ::= sclp scanpt STAR */
 {
   Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0);
   sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail));
-  yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p);
+  yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy402, p);
 }
         break;
       case 104: /* selcollist ::= sclp scanpt nm DOT STAR */
@@ -177850,7 +180526,7 @@ static YYACTIONTYPE yy_reduce(
   sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail));
   pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0);
   pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight);
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, pDot);
 }
         break;
       case 105: /* as ::= AS nm */
@@ -177861,50 +180537,50 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 107: /* from ::= */
       case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110);
-{yymsp[1].minor.yy203 = 0;}
+{yymsp[1].minor.yy563 = 0;}
         break;
       case 108: /* from ::= FROM seltablist */
 {
-  yymsp[-1].minor.yy203 = yymsp[0].minor.yy203;
-  sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203);
+  yymsp[-1].minor.yy563 = yymsp[0].minor.yy563;
+  sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy563);
 }
         break;
       case 109: /* stl_prefix ::= seltablist joinop */
 {
-   if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144;
+   if( ALWAYS(yymsp[-1].minor.yy563 && yymsp[-1].minor.yy563->nSrc>0) ) yymsp[-1].minor.yy563->a[yymsp[-1].minor.yy563->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy502;
 }
         break;
       case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */
 {
-  yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269);
+  yymsp[-4].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy563,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421);
 }
         break;
       case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */
 {
-  yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269);
-  sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0);
+  yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy421);
+  sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy563, &yymsp[-1].minor.yy0);
 }
         break;
       case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */
 {
-  yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269);
-  sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14);
+  yymsp[-7].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy563,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421);
+  sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy563, yymsp[-3].minor.yy402);
 }
         break;
       case 114: /* seltablist ::= stl_prefix LP select RP as on_using */
 {
-    yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269);
+    yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy637,&yymsp[0].minor.yy421);
   }
         break;
       case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */
 {
-    if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){
-      yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203;
-    }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){
-      yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269);
-      if( yymsp[-5].minor.yy203 ){
-        SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1];
-        SrcItem *pOld = yymsp[-3].minor.yy203->a;
+    if( yymsp[-5].minor.yy563==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy421.pOn==0 && yymsp[0].minor.yy421.pUsing==0 ){
+      yymsp[-5].minor.yy563 = yymsp[-3].minor.yy563;
+    }else if( ALWAYS(yymsp[-3].minor.yy563!=0) && yymsp[-3].minor.yy563->nSrc==1 ){
+      yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421);
+      if( yymsp[-5].minor.yy563 ){
+        SrcItem *pNew = &yymsp[-5].minor.yy563->a[yymsp[-5].minor.yy563->nSrc-1];
+        SrcItem *pOld = yymsp[-3].minor.yy563->a;
         assert( pOld->fg.fixedSchema==0 );
         pNew->zName = pOld->zName;
         assert( pOld->fg.fixedSchema==0 );
@@ -177929,12 +180605,12 @@ static YYACTIONTYPE yy_reduce(
         }
         pOld->zName = 0;
       }
-      sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203);
+      sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy563);
     }else{
       Select *pSubquery;
-      sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203);
-      pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0);
-      yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269);
+      sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy563);
+      pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy563,0,0,0,0,SF_NestedFrom,0);
+      yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy421);
     }
   }
         break;
@@ -177944,56 +180620,56 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 118: /* fullname ::= nm */
 {
-  yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0);
-  if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0);
+  yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0);
+  if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0);
 }
-  yymsp[0].minor.yy203 = yylhsminor.yy203;
+  yymsp[0].minor.yy563 = yylhsminor.yy563;
         break;
       case 119: /* fullname ::= nm DOT nm */
 {
-  yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);
-  if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0);
+  yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);
+  if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0);
 }
-  yymsp[-2].minor.yy203 = yylhsminor.yy203;
+  yymsp[-2].minor.yy563 = yylhsminor.yy563;
         break;
       case 120: /* xfullname ::= nm */
-{yymsp[0].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/}
+{yymsp[0].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/}
         break;
       case 121: /* xfullname ::= nm DOT nm */
-{yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/}
+{yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/}
         break;
       case 122: /* xfullname ::= nm DOT nm AS nm */
 {
-   yymsp[-4].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/
-   if( yymsp[-4].minor.yy203 ) yymsp[-4].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0);
+   yymsp[-4].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/
+   if( yymsp[-4].minor.yy563 ) yymsp[-4].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0);
 }
         break;
       case 123: /* xfullname ::= nm AS nm */
 {
-   yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/
-   if( yymsp[-2].minor.yy203 ) yymsp[-2].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0);
+   yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/
+   if( yymsp[-2].minor.yy563 ) yymsp[-2].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0);
 }
         break;
       case 124: /* joinop ::= COMMA|JOIN */
-{ yymsp[0].minor.yy144 = JT_INNER; }
+{ yymsp[0].minor.yy502 = JT_INNER; }
         break;
       case 125: /* joinop ::= JOIN_KW JOIN */
-{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0);  /*X-overwrites-A*/}
+{yymsp[-1].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0);  /*X-overwrites-A*/}
         break;
       case 126: /* joinop ::= JOIN_KW nm JOIN */
-{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/}
+{yymsp[-2].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/}
         break;
       case 127: /* joinop ::= JOIN_KW nm nm JOIN */
-{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/}
+{yymsp[-3].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/}
         break;
       case 128: /* on_using ::= ON expr */
-{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;}
+{yymsp[-1].minor.yy421.pOn = yymsp[0].minor.yy590; yymsp[-1].minor.yy421.pUsing = 0;}
         break;
       case 129: /* on_using ::= USING LP idlist RP */
-{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;}
+{yymsp[-3].minor.yy421.pOn = 0; yymsp[-3].minor.yy421.pUsing = yymsp[-1].minor.yy204;}
         break;
       case 130: /* on_using ::= */
-{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;}
+{yymsp[1].minor.yy421.pOn = 0; yymsp[1].minor.yy421.pUsing = 0;}
         break;
       case 132: /* indexed_by ::= INDEXED BY nm */
 {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;}
@@ -178003,35 +180679,35 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 135: /* orderby_opt ::= ORDER BY sortlist */
       case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145);
-{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;}
+{yymsp[-2].minor.yy402 = yymsp[0].minor.yy402;}
         break;
       case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */
 {
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454);
-  sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402,yymsp[-2].minor.yy590);
+  sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502);
 }
         break;
       case 137: /* sortlist ::= expr sortorder nulls */
 {
-  yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/
-  sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144);
+  yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy590); /*A-overwrites-Y*/
+  sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502);
 }
         break;
       case 138: /* sortorder ::= ASC */
-{yymsp[0].minor.yy144 = SQLITE_SO_ASC;}
+{yymsp[0].minor.yy502 = SQLITE_SO_ASC;}
         break;
       case 139: /* sortorder ::= DESC */
-{yymsp[0].minor.yy144 = SQLITE_SO_DESC;}
+{yymsp[0].minor.yy502 = SQLITE_SO_DESC;}
         break;
       case 140: /* sortorder ::= */
       case 143: /* nulls ::= */ yytestcase(yyruleno==143);
-{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;}
+{yymsp[1].minor.yy502 = SQLITE_SO_UNDEFINED;}
         break;
       case 141: /* nulls ::= NULLS FIRST */
-{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;}
+{yymsp[-1].minor.yy502 = SQLITE_SO_ASC;}
         break;
       case 142: /* nulls ::= NULLS LAST */
-{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;}
+{yymsp[-1].minor.yy502 = SQLITE_SO_DESC;}
         break;
       case 146: /* having_opt ::= */
       case 148: /* limit_opt ::= */ yytestcase(yyruleno==148);
@@ -178040,48 +180716,48 @@ static YYACTIONTYPE yy_reduce(
       case 232: /* case_else ::= */ yytestcase(yyruleno==232);
       case 233: /* case_operand ::= */ yytestcase(yyruleno==233);
       case 252: /* vinto ::= */ yytestcase(yyruleno==252);
-{yymsp[1].minor.yy454 = 0;}
+{yymsp[1].minor.yy590 = 0;}
         break;
       case 147: /* having_opt ::= HAVING expr */
       case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154);
       case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156);
       case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231);
       case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251);
-{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;}
+{yymsp[-1].minor.yy590 = yymsp[0].minor.yy590;}
         break;
       case 149: /* limit_opt ::= LIMIT expr */
-{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);}
+{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,0);}
         break;
       case 150: /* limit_opt ::= LIMIT expr OFFSET expr */
-{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);}
+{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);}
         break;
       case 151: /* limit_opt ::= LIMIT expr COMMA expr */
-{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);}
+{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,yymsp[-2].minor.yy590);}
         break;
       case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret orderby_opt limit_opt */
 {
-  sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy203, &yymsp[-3].minor.yy0);
+  sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy563, &yymsp[-3].minor.yy0);
 #ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-  if( yymsp[-1].minor.yy14 || yymsp[0].minor.yy454 ){
-    updateDeleteLimitError(pParse,yymsp[-1].minor.yy14,yymsp[0].minor.yy454);
-    yymsp[-1].minor.yy14 = 0;
-    yymsp[0].minor.yy454 = 0;
+  if( yymsp[-1].minor.yy402 || yymsp[0].minor.yy590 ){
+    updateDeleteLimitError(pParse,yymsp[-1].minor.yy402,yymsp[0].minor.yy590);
+    yymsp[-1].minor.yy402 = 0;
+    yymsp[0].minor.yy590 = 0;
   }
 #endif
-  sqlite3DeleteFrom(pParse,yymsp[-4].minor.yy203,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[0].minor.yy454);
+  sqlite3DeleteFrom(pParse,yymsp[-4].minor.yy563,yymsp[-2].minor.yy590,yymsp[-1].minor.yy402,yymsp[0].minor.yy590);
 }
         break;
       case 157: /* where_opt_ret ::= RETURNING selcollist */
-{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;}
+{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-1].minor.yy590 = 0;}
         break;
       case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */
-{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;}
+{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-3].minor.yy590 = yymsp[-2].minor.yy590;}
         break;
       case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret orderby_opt limit_opt */
 {
-  sqlite3SrcListIndexedBy(pParse, yymsp[-7].minor.yy203, &yymsp[-6].minor.yy0);
-  if( yymsp[-3].minor.yy203 ){
-    SrcList *pFromClause = yymsp[-3].minor.yy203;
+  sqlite3SrcListIndexedBy(pParse, yymsp[-7].minor.yy563, &yymsp[-6].minor.yy0);
+  if( yymsp[-3].minor.yy563 ){
+    SrcList *pFromClause = yymsp[-3].minor.yy563;
     if( pFromClause->nSrc>1 ){
       Select *pSubquery;
       Token as;
@@ -178090,98 +180766,98 @@ static YYACTIONTYPE yy_reduce(
       as.z = 0;
       pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0);
     }
-    yymsp[-7].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-7].minor.yy203, pFromClause);
+    yymsp[-7].minor.yy563 = sqlite3SrcListAppendList(pParse, yymsp[-7].minor.yy563, pFromClause);
   }
-  sqlite3ExprListCheckLength(pParse,yymsp[-4].minor.yy14,"set list");
+  sqlite3ExprListCheckLength(pParse,yymsp[-4].minor.yy402,"set list");
 #ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-  if( yymsp[-1].minor.yy14 || yymsp[0].minor.yy454 ){
-    updateDeleteLimitError(pParse,yymsp[-1].minor.yy14,yymsp[0].minor.yy454);
-    yymsp[-1].minor.yy14 = 0;
-    yymsp[0].minor.yy454 = 0;
+  if( yymsp[-1].minor.yy402 || yymsp[0].minor.yy590 ){
+    updateDeleteLimitError(pParse,yymsp[-1].minor.yy402,yymsp[0].minor.yy590);
+    yymsp[-1].minor.yy402 = 0;
+    yymsp[0].minor.yy590 = 0;
   }
 #endif
-  sqlite3Update(pParse,yymsp[-7].minor.yy203,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454,yymsp[-8].minor.yy144,yymsp[-1].minor.yy14,yymsp[0].minor.yy454,0);
+  sqlite3Update(pParse,yymsp[-7].minor.yy563,yymsp[-4].minor.yy402,yymsp[-2].minor.yy590,yymsp[-8].minor.yy502,yymsp[-1].minor.yy402,yymsp[0].minor.yy590,0);
 }
         break;
       case 160: /* setlist ::= setlist COMMA nm EQ expr */
 {
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454);
-  sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[0].minor.yy590);
+  sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, 1);
 }
         break;
       case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */
 {
-  yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454);
+  yymsp[-6].minor.yy402 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy402, yymsp[-3].minor.yy204, yymsp[0].minor.yy590);
 }
         break;
       case 162: /* setlist ::= nm EQ expr */
 {
-  yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454);
-  sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1);
+  yylhsminor.yy402 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy590);
+  sqlite3ExprListSetName(pParse, yylhsminor.yy402, &yymsp[-2].minor.yy0, 1);
 }
-  yymsp[-2].minor.yy14 = yylhsminor.yy14;
+  yymsp[-2].minor.yy402 = yylhsminor.yy402;
         break;
       case 163: /* setlist ::= LP idlist RP EQ expr */
 {
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy204, yymsp[0].minor.yy590);
 }
         break;
       case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */
 {
-  sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122);
+  sqlite3Insert(pParse, yymsp[-3].minor.yy563, yymsp[-1].minor.yy637, yymsp[-2].minor.yy204, yymsp[-5].minor.yy502, yymsp[0].minor.yy403);
 }
         break;
       case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */
 {
-  sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0);
+  sqlite3Insert(pParse, yymsp[-4].minor.yy563, 0, yymsp[-3].minor.yy204, yymsp[-6].minor.yy502, 0);
 }
         break;
       case 166: /* upsert ::= */
-{ yymsp[1].minor.yy122 = 0; }
+{ yymsp[1].minor.yy403 = 0; }
         break;
       case 167: /* upsert ::= RETURNING selcollist */
-{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); }
+{ yymsp[-1].minor.yy403 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy402); }
         break;
       case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */
-{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);}
+{ yymsp[-11].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy402,yymsp[-6].minor.yy590,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,yymsp[0].minor.yy403);}
         break;
       case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */
-{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); }
+{ yymsp[-8].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy402,yymsp[-3].minor.yy590,0,0,yymsp[0].minor.yy403); }
         break;
       case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */
-{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); }
+{ yymsp[-4].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); }
         break;
       case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */
-{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);}
+{ yymsp[-7].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,0);}
         break;
       case 172: /* returning ::= RETURNING selcollist */
-{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);}
+{sqlite3AddReturning(pParse,yymsp[0].minor.yy402);}
         break;
       case 175: /* idlist_opt ::= */
-{yymsp[1].minor.yy132 = 0;}
+{yymsp[1].minor.yy204 = 0;}
         break;
       case 176: /* idlist_opt ::= LP idlist RP */
-{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;}
+{yymsp[-2].minor.yy204 = yymsp[-1].minor.yy204;}
         break;
       case 177: /* idlist ::= idlist COMMA nm */
-{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);}
+{yymsp[-2].minor.yy204 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy204,&yymsp[0].minor.yy0);}
         break;
       case 178: /* idlist ::= nm */
-{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/}
+{yymsp[0].minor.yy204 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/}
         break;
       case 179: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;}
+{yymsp[-2].minor.yy590 = yymsp[-1].minor.yy590;}
         break;
       case 180: /* expr ::= ID|INDEXED|JOIN_KW */
-{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/}
+{yymsp[0].minor.yy590=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/}
         break;
       case 181: /* expr ::= nm DOT nm */
 {
   Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0);
   Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0);
-  yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2);
+  yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2);
 }
-  yymsp[-2].minor.yy454 = yylhsminor.yy454;
+  yymsp[-2].minor.yy590 = yylhsminor.yy590;
         break;
       case 182: /* expr ::= nm DOT nm DOT nm */
 {
@@ -178192,27 +180868,27 @@ static YYACTIONTYPE yy_reduce(
   if( IN_RENAME_OBJECT ){
     sqlite3RenameTokenRemap(pParse, 0, temp1);
   }
-  yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4);
+  yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4);
 }
-  yymsp[-4].minor.yy454 = yylhsminor.yy454;
+  yymsp[-4].minor.yy590 = yylhsminor.yy590;
         break;
       case 183: /* term ::= NULL|FLOAT|BLOB */
       case 184: /* term ::= STRING */ yytestcase(yyruleno==184);
-{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/}
+{yymsp[0].minor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/}
         break;
       case 185: /* term ::= INTEGER */
 {
-  yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1);
-  if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail);
+  yylhsminor.yy590 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1);
+  if( yylhsminor.yy590 ) yylhsminor.yy590->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail);
 }
-  yymsp[0].minor.yy454 = yylhsminor.yy454;
+  yymsp[0].minor.yy590 = yylhsminor.yy590;
         break;
       case 186: /* expr ::= VARIABLE */
 {
   if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){
     u32 n = yymsp[0].minor.yy0.n;
-    yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0);
-    sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n);
+    yymsp[0].minor.yy590 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0);
+    sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy590, n);
   }else{
     /* When doing a nested parse, one can include terms in an expression
     ** that look like this:   #1 #2 ...  These terms refer to registers
@@ -178220,81 +180896,81 @@ static YYACTIONTYPE yy_reduce(
     Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/
     assert( t.n>=2 );
     if( pParse->nested==0 ){
-      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
-      yymsp[0].minor.yy454 = 0;
+      parserSyntaxError(pParse, &t);
+      yymsp[0].minor.yy590 = 0;
     }else{
-      yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
-      if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable);
+      yymsp[0].minor.yy590 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
+      if( yymsp[0].minor.yy590 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy590->iTable);
     }
   }
 }
         break;
       case 187: /* expr ::= expr COLLATE ID|STRING */
 {
-  yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1);
+  yymsp[-2].minor.yy590 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy590, &yymsp[0].minor.yy0, 1);
 }
         break;
       case 188: /* expr ::= CAST LP expr AS typetoken RP */
 {
-  yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1);
-  sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0);
+  yymsp[-5].minor.yy590 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1);
+  sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy590, yymsp[-3].minor.yy590, 0);
 }
         break;
       case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy502);
 }
-  yymsp[-4].minor.yy454 = yylhsminor.yy454;
+  yymsp[-4].minor.yy590 = yylhsminor.yy590;
         break;
       case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144);
-  sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy402, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy502);
+  sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-1].minor.yy402);
 }
-  yymsp[-7].minor.yy454 = yylhsminor.yy454;
+  yymsp[-7].minor.yy590 = yylhsminor.yy590;
         break;
       case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0);
 }
-  yymsp[-3].minor.yy454 = yylhsminor.yy454;
+  yymsp[-3].minor.yy590 = yylhsminor.yy590;
         break;
       case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144);
-  sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy402, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy502);
+  sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483);
 }
-  yymsp[-5].minor.yy454 = yylhsminor.yy454;
+  yymsp[-5].minor.yy590 = yylhsminor.yy590;
         break;
       case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144);
-  sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211);
-  sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy402, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy502);
+  sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483);
+  sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-2].minor.yy402);
 }
-  yymsp[-8].minor.yy454 = yylhsminor.yy454;
+  yymsp[-8].minor.yy590 = yylhsminor.yy590;
         break;
       case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0);
-  sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0);
+  sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483);
 }
-  yymsp[-4].minor.yy454 = yylhsminor.yy454;
+  yymsp[-4].minor.yy590 = yylhsminor.yy590;
         break;
       case 195: /* term ::= CTIME_KW */
 {
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0);
 }
-  yymsp[0].minor.yy454 = yylhsminor.yy454;
+  yymsp[0].minor.yy590 = yylhsminor.yy590;
         break;
       case 196: /* expr ::= LP nexprlist COMMA expr RP */
 {
-  ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454);
-  yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
-  if( yymsp[-4].minor.yy454 ){
-    yymsp[-4].minor.yy454->x.pList = pList;
+  ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590);
+  yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
+  if( yymsp[-4].minor.yy590 ){
+    yymsp[-4].minor.yy590->x.pList = pList;
     if( ALWAYS(pList->nExpr) ){
-      yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate;
+      yymsp[-4].minor.yy590->flags |= pList->a[0].pExpr->flags & EP_Propagate;
     }
   }else{
     sqlite3ExprListDelete(pParse->db, pList);
@@ -178302,7 +180978,7 @@ static YYACTIONTYPE yy_reduce(
 }
         break;
       case 197: /* expr ::= expr AND expr */
-{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);}
+{yymsp[-2].minor.yy590=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);}
         break;
       case 198: /* expr ::= expr OR expr */
       case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199);
@@ -178311,7 +180987,7 @@ static YYACTIONTYPE yy_reduce(
       case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202);
       case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203);
       case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204);
-{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);}
+{yymsp[-2].minor.yy590=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);}
         break;
       case 205: /* likeop ::= NOT LIKE_KW|MATCH */
 {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/}
@@ -178321,11 +180997,11 @@ static YYACTIONTYPE yy_reduce(
   ExprList *pList;
   int bNot = yymsp[-1].minor.yy0.n & 0x80000000;
   yymsp[-1].minor.yy0.n &= 0x7fffffff;
-  pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454);
-  pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454);
-  yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0);
-  if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0);
-  if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc;
+  pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy590);
+  pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy590);
+  yymsp[-2].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0);
+  if( bNot ) yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy590, 0);
+  if( yymsp[-2].minor.yy590 ) yymsp[-2].minor.yy590->flags |= EP_InfixFunc;
 }
         break;
       case 207: /* expr ::= expr likeop expr ESCAPE expr */
@@ -178333,91 +181009,91 @@ static YYACTIONTYPE yy_reduce(
   ExprList *pList;
   int bNot = yymsp[-3].minor.yy0.n & 0x80000000;
   yymsp[-3].minor.yy0.n &= 0x7fffffff;
-  pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454);
-  pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454);
-  pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454);
-  yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0);
-  if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0);
-  if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc;
+  pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590);
+  pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy590);
+  pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590);
+  yymsp[-4].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0);
+  if( bNot ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0);
+  if( yymsp[-4].minor.yy590 ) yymsp[-4].minor.yy590->flags |= EP_InfixFunc;
 }
         break;
       case 208: /* expr ::= expr ISNULL|NOTNULL */
-{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy454,0);}
+{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy590,0);}
         break;
       case 209: /* expr ::= expr NOT NULL */
-{yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy454,0);}
+{yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy590,0);}
         break;
       case 210: /* expr ::= expr IS expr */
 {
-  yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);
-  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-2].minor.yy454, TK_ISNULL);
+  yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);
+  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-2].minor.yy590, TK_ISNULL);
 }
         break;
       case 211: /* expr ::= expr IS NOT expr */
 {
-  yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy454,yymsp[0].minor.yy454);
-  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-3].minor.yy454, TK_NOTNULL);
+  yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy590,yymsp[0].minor.yy590);
+  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-3].minor.yy590, TK_NOTNULL);
 }
         break;
       case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */
 {
-  yymsp[-5].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy454,yymsp[0].minor.yy454);
-  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-5].minor.yy454, TK_ISNULL);
+  yymsp[-5].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy590,yymsp[0].minor.yy590);
+  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-5].minor.yy590, TK_ISNULL);
 }
         break;
       case 213: /* expr ::= expr IS DISTINCT FROM expr */
 {
-  yymsp[-4].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy454,yymsp[0].minor.yy454);
-  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-4].minor.yy454, TK_NOTNULL);
+  yymsp[-4].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy590,yymsp[0].minor.yy590);
+  binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-4].minor.yy590, TK_NOTNULL);
 }
         break;
       case 214: /* expr ::= NOT expr */
       case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215);
-{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/}
+{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy590, 0);/*A-overwrites-B*/}
         break;
       case 216: /* expr ::= PLUS|MINUS expr */
 {
-  Expr *p = yymsp[0].minor.yy454;
+  Expr *p = yymsp[0].minor.yy590;
   u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS);
   assert( TK_UPLUS>TK_PLUS );
   assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) );
   if( p && p->op==TK_UPLUS ){
     p->op = op;
-    yymsp[-1].minor.yy454 = p;
+    yymsp[-1].minor.yy590 = p;
   }else{
-    yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0);
+    yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, op, p, 0);
     /*A-overwrites-B*/
   }
 }
         break;
       case 217: /* expr ::= expr PTR expr */
 {
-  ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454);
-  pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454);
-  yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0);
+  ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy590);
+  pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy590);
+  yylhsminor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0);
 }
-  yymsp[-2].minor.yy454 = yylhsminor.yy454;
+  yymsp[-2].minor.yy590 = yylhsminor.yy590;
         break;
       case 218: /* between_op ::= BETWEEN */
       case 221: /* in_op ::= IN */ yytestcase(yyruleno==221);
-{yymsp[0].minor.yy144 = 0;}
+{yymsp[0].minor.yy502 = 0;}
         break;
       case 220: /* expr ::= expr between_op expr AND expr */
 {
-  ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454);
-  pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454);
-  yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0);
-  if( yymsp[-4].minor.yy454 ){
-    yymsp[-4].minor.yy454->x.pList = pList;
+  ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590);
+  pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590);
+  yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy590, 0);
+  if( yymsp[-4].minor.yy590 ){
+    yymsp[-4].minor.yy590->x.pList = pList;
   }else{
     sqlite3ExprListDelete(pParse->db, pList);
   }
-  if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0);
+  if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0);
 }
         break;
       case 223: /* expr ::= expr in_op LP exprlist RP */
 {
-    if( yymsp[-1].minor.yy14==0 ){
+    if( yymsp[-1].minor.yy402==0 ){
       /* Expressions of the form
       **
       **      expr1 IN ()
@@ -178426,110 +181102,110 @@ static YYACTIONTYPE yy_reduce(
       ** simplify to constants 0 (false) and 1 (true), respectively,
       ** regardless of the value of expr1.
       */
-      sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454);
-      yymsp[-4].minor.yy454 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false");
-      if( yymsp[-4].minor.yy454 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy454);
-    }else{
-      Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr;
-      if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){
-        yymsp[-1].minor.yy14->a[0].pExpr = 0;
-        sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14);
+      sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy590);
+      yymsp[-4].minor.yy590 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy502 ? "true" : "false");
+      if( yymsp[-4].minor.yy590 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy590);
+    }else{
+      Expr *pRHS = yymsp[-1].minor.yy402->a[0].pExpr;
+      if( yymsp[-1].minor.yy402->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy590->op!=TK_VECTOR ){
+        yymsp[-1].minor.yy402->a[0].pExpr = 0;
+        sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402);
         pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0);
-        yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS);
-      }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){
-        yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0);
-        sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect);
+        yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy590, pRHS);
+      }else if( yymsp[-1].minor.yy402->nExpr==1 && pRHS->op==TK_SELECT ){
+        yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0);
+        sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pRHS->x.pSelect);
         pRHS->x.pSelect = 0;
-        sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14);
-      }else{
-        yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0);
-        if( yymsp[-4].minor.yy454==0 ){
-          sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14);
-        }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){
-          int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr;
-          Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14);
+        sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402);
+      }else{
+        yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0);
+        if( yymsp[-4].minor.yy590==0 ){
+          sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402);
+        }else if( yymsp[-4].minor.yy590->pLeft->op==TK_VECTOR ){
+          int nExpr = yymsp[-4].minor.yy590->pLeft->x.pList->nExpr;
+          Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy402);
           if( pSelectRHS ){
             parserDoubleLinkSelect(pParse, pSelectRHS);
-            sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS);
+            sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelectRHS);
           }
         }else{
-          yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14;
-          sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454);
+          yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy402;
+          sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590);
         }
       }
-      if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0);
+      if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0);
     }
   }
         break;
       case 224: /* expr ::= LP select RP */
 {
-    yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
-    sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555);
+    yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
+    sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy590, yymsp[-1].minor.yy637);
   }
         break;
       case 225: /* expr ::= expr in_op LP select RP */
 {
-    yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0);
-    sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555);
-    if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0);
+    yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0);
+    sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, yymsp[-1].minor.yy637);
+    if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0);
   }
         break;
       case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */
 {
     SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);
     Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0);
-    if( yymsp[0].minor.yy14 )  sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14);
-    yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0);
-    sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect);
-    if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0);
+    if( yymsp[0].minor.yy402 )  sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy402);
+    yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0);
+    sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelect);
+    if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0);
   }
         break;
       case 227: /* expr ::= EXISTS LP select RP */
 {
     Expr *p;
-    p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0);
-    sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555);
+    p = yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0);
+    sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy637);
   }
         break;
       case 228: /* expr ::= CASE case_operand case_exprlist case_else END */
 {
-  yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0);
-  if( yymsp[-4].minor.yy454 ){
-    yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14;
-    sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454);
+  yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy590, 0);
+  if( yymsp[-4].minor.yy590 ){
+    yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy590 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590) : yymsp[-2].minor.yy402;
+    sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590);
   }else{
-    sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14);
-    sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454);
+    sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy402);
+    sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590);
   }
 }
         break;
       case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */
 {
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454);
-  yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[-2].minor.yy590);
+  yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[0].minor.yy590);
 }
         break;
       case 230: /* case_exprlist ::= WHEN expr THEN expr */
 {
-  yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454);
-  yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454);
+  yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590);
+  yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy402, yymsp[0].minor.yy590);
 }
         break;
       case 235: /* nexprlist ::= nexprlist COMMA expr */
-{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);}
+{yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[0].minor.yy590);}
         break;
       case 236: /* nexprlist ::= expr */
-{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/}
+{yymsp[0].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy590); /*A-overwrites-Y*/}
         break;
       case 238: /* paren_exprlist ::= LP exprlist RP */
       case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243);
-{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;}
+{yymsp[-2].minor.yy402 = yymsp[-1].minor.yy402;}
         break;
       case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
 {
   sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0,
-                     sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144,
-                      &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF);
+                     sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy402, yymsp[-10].minor.yy502,
+                      &yymsp[-11].minor.yy0, yymsp[0].minor.yy590, SQLITE_SO_ASC, yymsp[-8].minor.yy502, SQLITE_IDXTYPE_APPDEF);
   if( IN_RENAME_OBJECT && pParse->pNewIndex ){
     sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0);
   }
@@ -178537,29 +181213,29 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 240: /* uniqueflag ::= UNIQUE */
       case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282);
-{yymsp[0].minor.yy144 = OE_Abort;}
+{yymsp[0].minor.yy502 = OE_Abort;}
         break;
       case 241: /* uniqueflag ::= */
-{yymsp[1].minor.yy144 = OE_None;}
+{yymsp[1].minor.yy502 = OE_None;}
         break;
       case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */
 {
-  yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144);
+  yymsp[-4].minor.yy402 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502);
 }
         break;
       case 245: /* eidlist ::= nm collate sortorder */
 {
-  yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/
+  yymsp[-2].minor.yy402 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); /*A-overwrites-Y*/
 }
         break;
       case 248: /* cmd ::= DROP INDEX ifexists fullname */
-{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);}
+{sqlite3DropIndex(pParse, yymsp[0].minor.yy563, yymsp[-1].minor.yy502);}
         break;
       case 249: /* cmd ::= VACUUM vinto */
-{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);}
+{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy590);}
         break;
       case 250: /* cmd ::= VACUUM nm vinto */
-{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);}
+{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy590);}
         break;
       case 253: /* cmd ::= PRAGMA nm dbnm */
 {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);}
@@ -178581,50 +181257,50 @@ static YYACTIONTYPE yy_reduce(
   Token all;
   all.z = yymsp[-3].minor.yy0.z;
   all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n;
-  sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all);
+  sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy319, &all);
 }
         break;
       case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
 {
-  sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144);
+  sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy502, yymsp[-4].minor.yy28.a, yymsp[-4].minor.yy28.b, yymsp[-2].minor.yy563, yymsp[0].minor.yy590, yymsp[-10].minor.yy502, yymsp[-8].minor.yy502);
   yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/
 }
         break;
       case 262: /* trigger_time ::= BEFORE|AFTER */
-{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ }
+{ yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/ }
         break;
       case 263: /* trigger_time ::= INSTEAD OF */
-{ yymsp[-1].minor.yy144 = TK_INSTEAD;}
+{ yymsp[-1].minor.yy502 = TK_INSTEAD;}
         break;
       case 264: /* trigger_time ::= */
-{ yymsp[1].minor.yy144 = TK_BEFORE; }
+{ yymsp[1].minor.yy502 = TK_BEFORE; }
         break;
       case 265: /* trigger_event ::= DELETE|INSERT */
       case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266);
-{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;}
+{yymsp[0].minor.yy28.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy28.b = 0;}
         break;
       case 267: /* trigger_event ::= UPDATE OF idlist */
-{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;}
+{yymsp[-2].minor.yy28.a = TK_UPDATE; yymsp[-2].minor.yy28.b = yymsp[0].minor.yy204;}
         break;
       case 268: /* when_clause ::= */
       case 287: /* key_opt ::= */ yytestcase(yyruleno==287);
-{ yymsp[1].minor.yy454 = 0; }
+{ yymsp[1].minor.yy590 = 0; }
         break;
       case 269: /* when_clause ::= WHEN expr */
       case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288);
-{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; }
+{ yymsp[-1].minor.yy590 = yymsp[0].minor.yy590; }
         break;
       case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
 {
-  assert( yymsp[-2].minor.yy427!=0 );
-  yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427;
-  yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427;
+  assert( yymsp[-2].minor.yy319!=0 );
+  yymsp[-2].minor.yy319->pLast->pNext = yymsp[-1].minor.yy319;
+  yymsp[-2].minor.yy319->pLast = yymsp[-1].minor.yy319;
 }
         break;
       case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */
 {
-  assert( yymsp[-1].minor.yy427!=0 );
-  yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427;
+  assert( yymsp[-1].minor.yy319!=0 );
+  yymsp[-1].minor.yy319->pLast = yymsp[-1].minor.yy319;
 }
         break;
       case 272: /* trnm ::= nm DOT nm */
@@ -178650,58 +181326,58 @@ static YYACTIONTYPE yy_reduce(
 }
         break;
       case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
-{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);}
-  yymsp[-8].minor.yy427 = yylhsminor.yy427;
+{yylhsminor.yy319 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy563, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590, yymsp[-7].minor.yy502, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy342);}
+  yymsp[-8].minor.yy319 = yylhsminor.yy319;
         break;
       case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
 {
-   yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/
+   yylhsminor.yy319 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy204,yymsp[-2].minor.yy637,yymsp[-6].minor.yy502,yymsp[-1].minor.yy403,yymsp[-7].minor.yy342,yymsp[0].minor.yy342);/*yylhsminor.yy319-overwrites-yymsp[-6].minor.yy502*/
 }
-  yymsp[-7].minor.yy427 = yylhsminor.yy427;
+  yymsp[-7].minor.yy319 = yylhsminor.yy319;
         break;
       case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
-{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);}
-  yymsp[-5].minor.yy427 = yylhsminor.yy427;
+{yylhsminor.yy319 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy590, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy342);}
+  yymsp[-5].minor.yy319 = yylhsminor.yy319;
         break;
       case 278: /* trigger_cmd ::= scanpt select scanpt */
-{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/}
-  yymsp[-2].minor.yy427 = yylhsminor.yy427;
+{yylhsminor.yy319 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy637, yymsp[-2].minor.yy342, yymsp[0].minor.yy342); /*yylhsminor.yy319-overwrites-yymsp[-1].minor.yy637*/}
+  yymsp[-2].minor.yy319 = yylhsminor.yy319;
         break;
       case 279: /* expr ::= RAISE LP IGNORE RP */
 {
-  yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0);
-  if( yymsp[-3].minor.yy454 ){
-    yymsp[-3].minor.yy454->affExpr = OE_Ignore;
+  yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, 0, 0);
+  if( yymsp[-3].minor.yy590 ){
+    yymsp[-3].minor.yy590->affExpr = OE_Ignore;
   }
 }
         break;
       case 280: /* expr ::= RAISE LP raisetype COMMA expr RP */
 {
-  yymsp[-5].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy454, 0);
-  if( yymsp[-5].minor.yy454 ) {
-    yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144;
+  yymsp[-5].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy590, 0);
+  if( yymsp[-5].minor.yy590 ) {
+    yymsp[-5].minor.yy590->affExpr = (char)yymsp[-3].minor.yy502;
   }
 }
         break;
       case 281: /* raisetype ::= ROLLBACK */
-{yymsp[0].minor.yy144 = OE_Rollback;}
+{yymsp[0].minor.yy502 = OE_Rollback;}
         break;
       case 283: /* raisetype ::= FAIL */
-{yymsp[0].minor.yy144 = OE_Fail;}
+{yymsp[0].minor.yy502 = OE_Fail;}
         break;
       case 284: /* cmd ::= DROP TRIGGER ifexists fullname */
 {
-  sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144);
+  sqlite3DropTrigger(pParse,yymsp[0].minor.yy563,yymsp[-1].minor.yy502);
 }
         break;
       case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
 {
-  sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454);
+  sqlite3Attach(pParse, yymsp[-3].minor.yy590, yymsp[-1].minor.yy590, yymsp[0].minor.yy590);
 }
         break;
       case 286: /* cmd ::= DETACH database_kw_opt expr */
 {
-  sqlite3Detach(pParse, yymsp[0].minor.yy454);
+  sqlite3Detach(pParse, yymsp[0].minor.yy590);
 }
         break;
       case 289: /* cmd ::= REINDEX */
@@ -178718,7 +181394,7 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */
 {
-  sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0);
+  sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy563,&yymsp[0].minor.yy0);
 }
         break;
       case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
@@ -178729,18 +181405,18 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
 {
-  sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0);
+  sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy563, &yymsp[0].minor.yy0);
 }
         break;
       case 296: /* add_column_fullname ::= fullname */
 {
   disableLookaside(pParse);
-  sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy203);
+  sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy563);
 }
         break;
       case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
 {
-  sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);
+  sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy563, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);
 }
         break;
       case 298: /* cmd ::= create_vtab */
@@ -178751,7 +181427,7 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
 {
-    sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144);
+    sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy502);
 }
         break;
       case 301: /* vtabarg ::= */
@@ -178764,20 +181440,20 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 305: /* with ::= WITH wqlist */
       case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306);
-{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); }
+{ sqlite3WithPush(pParse, yymsp[0].minor.yy125, 1); }
         break;
       case 307: /* wqas ::= AS */
-{yymsp[0].minor.yy462 = M10d_Any;}
+{yymsp[0].minor.yy444 = M10d_Any;}
         break;
       case 308: /* wqas ::= AS MATERIALIZED */
-{yymsp[-1].minor.yy462 = M10d_Yes;}
+{yymsp[-1].minor.yy444 = M10d_Yes;}
         break;
       case 309: /* wqas ::= AS NOT MATERIALIZED */
-{yymsp[-2].minor.yy462 = M10d_No;}
+{yymsp[-2].minor.yy444 = M10d_No;}
         break;
       case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */
 {
-  yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/
+  yymsp[-5].minor.yy361 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy402, yymsp[-1].minor.yy637, yymsp[-3].minor.yy444); /*A-overwrites-X*/
 }
         break;
       case 311: /* withnm ::= nm */
@@ -178785,160 +181461,160 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 312: /* wqlist ::= wqitem */
 {
-  yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/
+  yymsp[0].minor.yy125 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy361); /*A-overwrites-X*/
 }
         break;
       case 313: /* wqlist ::= wqlist COMMA wqitem */
 {
-  yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67);
+  yymsp[-2].minor.yy125 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy125, yymsp[0].minor.yy361);
 }
         break;
       case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */
 {
-  assert( yymsp[0].minor.yy211!=0 );
-  sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211);
-  yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211;
-  yylhsminor.yy211 = yymsp[0].minor.yy211;
+  assert( yymsp[0].minor.yy483!=0 );
+  sqlite3WindowChain(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy483);
+  yymsp[0].minor.yy483->pNextWin = yymsp[-2].minor.yy483;
+  yylhsminor.yy483 = yymsp[0].minor.yy483;
 }
-  yymsp[-2].minor.yy211 = yylhsminor.yy211;
+  yymsp[-2].minor.yy483 = yylhsminor.yy483;
         break;
       case 315: /* windowdefn ::= nm AS LP window RP */
 {
-  if( ALWAYS(yymsp[-1].minor.yy211) ){
-    yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n);
+  if( ALWAYS(yymsp[-1].minor.yy483) ){
+    yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n);
   }
-  yylhsminor.yy211 = yymsp[-1].minor.yy211;
+  yylhsminor.yy483 = yymsp[-1].minor.yy483;
 }
-  yymsp[-4].minor.yy211 = yylhsminor.yy211;
+  yymsp[-4].minor.yy483 = yylhsminor.yy483;
         break;
       case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */
 {
-  yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0);
+  yymsp[-4].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, 0);
 }
         break;
       case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
 {
-  yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0);
+  yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, &yymsp[-5].minor.yy0);
 }
-  yymsp[-5].minor.yy211 = yylhsminor.yy211;
+  yymsp[-5].minor.yy483 = yylhsminor.yy483;
         break;
       case 318: /* window ::= ORDER BY sortlist frame_opt */
 {
-  yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0);
+  yymsp[-3].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, 0);
 }
         break;
       case 319: /* window ::= nm ORDER BY sortlist frame_opt */
 {
-  yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0);
+  yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0);
 }
-  yymsp[-4].minor.yy211 = yylhsminor.yy211;
+  yymsp[-4].minor.yy483 = yylhsminor.yy483;
         break;
       case 320: /* window ::= nm frame_opt */
 {
-  yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0);
+  yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, 0, &yymsp[-1].minor.yy0);
 }
-  yymsp[-1].minor.yy211 = yylhsminor.yy211;
+  yymsp[-1].minor.yy483 = yylhsminor.yy483;
         break;
       case 321: /* frame_opt ::= */
 {
-  yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0);
+  yymsp[1].minor.yy483 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0);
 }
         break;
       case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
 {
-  yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462);
+  yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy502, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy444);
 }
-  yymsp[-2].minor.yy211 = yylhsminor.yy211;
+  yymsp[-2].minor.yy483 = yylhsminor.yy483;
         break;
       case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
 {
-  yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462);
+  yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy502, yymsp[-3].minor.yy205.eType, yymsp[-3].minor.yy205.pExpr, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, yymsp[0].minor.yy444);
 }
-  yymsp[-5].minor.yy211 = yylhsminor.yy211;
+  yymsp[-5].minor.yy483 = yylhsminor.yy483;
         break;
       case 325: /* frame_bound_s ::= frame_bound */
       case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327);
-{yylhsminor.yy509 = yymsp[0].minor.yy509;}
-  yymsp[0].minor.yy509 = yylhsminor.yy509;
+{yylhsminor.yy205 = yymsp[0].minor.yy205;}
+  yymsp[0].minor.yy205 = yylhsminor.yy205;
         break;
       case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */
       case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328);
       case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330);
-{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;}
-  yymsp[-1].minor.yy509 = yylhsminor.yy509;
+{yylhsminor.yy205.eType = yymsp[-1].major; yylhsminor.yy205.pExpr = 0;}
+  yymsp[-1].minor.yy205 = yylhsminor.yy205;
         break;
       case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */
-{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;}
-  yymsp[-1].minor.yy509 = yylhsminor.yy509;
+{yylhsminor.yy205.eType = yymsp[0].major; yylhsminor.yy205.pExpr = yymsp[-1].minor.yy590;}
+  yymsp[-1].minor.yy205 = yylhsminor.yy205;
         break;
       case 331: /* frame_exclude_opt ::= */
-{yymsp[1].minor.yy462 = 0;}
+{yymsp[1].minor.yy444 = 0;}
         break;
       case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */
-{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;}
+{yymsp[-1].minor.yy444 = yymsp[0].minor.yy444;}
         break;
       case 333: /* frame_exclude ::= NO OTHERS */
       case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334);
-{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/}
+{yymsp[-1].minor.yy444 = yymsp[-1].major; /*A-overwrites-X*/}
         break;
       case 335: /* frame_exclude ::= GROUP|TIES */
-{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/}
+{yymsp[0].minor.yy444 = yymsp[0].major; /*A-overwrites-X*/}
         break;
       case 336: /* window_clause ::= WINDOW windowdefn_list */
-{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; }
+{ yymsp[-1].minor.yy483 = yymsp[0].minor.yy483; }
         break;
       case 337: /* filter_over ::= filter_clause over_clause */
 {
-  if( yymsp[0].minor.yy211 ){
-    yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454;
+  if( yymsp[0].minor.yy483 ){
+    yymsp[0].minor.yy483->pFilter = yymsp[-1].minor.yy590;
   }else{
-    sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454);
+    sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590);
   }
-  yylhsminor.yy211 = yymsp[0].minor.yy211;
+  yylhsminor.yy483 = yymsp[0].minor.yy483;
 }
-  yymsp[-1].minor.yy211 = yylhsminor.yy211;
+  yymsp[-1].minor.yy483 = yylhsminor.yy483;
         break;
       case 338: /* filter_over ::= over_clause */
 {
-  yylhsminor.yy211 = yymsp[0].minor.yy211;
+  yylhsminor.yy483 = yymsp[0].minor.yy483;
 }
-  yymsp[0].minor.yy211 = yylhsminor.yy211;
+  yymsp[0].minor.yy483 = yylhsminor.yy483;
         break;
       case 339: /* filter_over ::= filter_clause */
 {
-  yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
-  if( yylhsminor.yy211 ){
-    yylhsminor.yy211->eFrmType = TK_FILTER;
-    yylhsminor.yy211->pFilter = yymsp[0].minor.yy454;
+  yylhsminor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
+  if( yylhsminor.yy483 ){
+    yylhsminor.yy483->eFrmType = TK_FILTER;
+    yylhsminor.yy483->pFilter = yymsp[0].minor.yy590;
   }else{
-    sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454);
+    sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy590);
   }
 }
-  yymsp[0].minor.yy211 = yylhsminor.yy211;
+  yymsp[0].minor.yy483 = yylhsminor.yy483;
         break;
       case 340: /* over_clause ::= OVER LP window RP */
 {
-  yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211;
-  assert( yymsp[-3].minor.yy211!=0 );
+  yymsp[-3].minor.yy483 = yymsp[-1].minor.yy483;
+  assert( yymsp[-3].minor.yy483!=0 );
 }
         break;
       case 341: /* over_clause ::= OVER nm */
 {
-  yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
-  if( yymsp[-1].minor.yy211 ){
-    yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n);
+  yymsp[-1].minor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
+  if( yymsp[-1].minor.yy483 ){
+    yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n);
   }
 }
         break;
       case 342: /* filter_clause ::= FILTER LP WHERE expr RP */
-{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; }
+{ yymsp[-4].minor.yy590 = yymsp[-1].minor.yy590; }
         break;
       case 343: /* term ::= QNUMBER */
 {
-  yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0);
-  sqlite3DequoteNumber(pParse, yylhsminor.yy454);
+  yylhsminor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0);
+  sqlite3DequoteNumber(pParse, yylhsminor.yy590);
 }
-  yymsp[0].minor.yy454 = yylhsminor.yy454;
+  yymsp[0].minor.yy590 = yylhsminor.yy590;
         break;
       default:
       /* (344) input ::= cmdlist */ yytestcase(yyruleno==344);
@@ -184413,7 +187089,7 @@ static int openDatabase(
   db->aDb = db->aDbStatic;
   db->lookaside.bDisable = 1;
   db->lookaside.sz = 0;
-
+  sqlite3FastPrngInit(&db->sPrng);
   assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
   memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
   db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS;
@@ -186133,6 +188809,35 @@ SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb,
+  unsigned int *pnPrior, unsigned int *pnFrame
+){
+  int rc = SQLITE_OK;
+
+#ifndef SQLITE_OMIT_WAL
+  Btree *pBt;
+  int iDb;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+
+  sqlite3_mutex_enter(db->mutex);
+  iDb = sqlite3FindDbName(db, zDb);
+  if( iDb<0 ){
+    return SQLITE_ERROR;
+  }
+  pBt = db->aDb[iDb].pBt;
+  rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame);
+  sqlite3_mutex_leave(db->mutex);
+#endif   /* SQLITE_OMIT_WAL */
+
+  return rc;
+}
+
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 /*
 ** Given the name of a compile-time option, return true if that option
@@ -226312,6 +229017,13 @@ typedef struct SessionInput SessionInput;
 
 #define SESSIONS_ROWID "_rowid_"
 
+/*
+** The three different types of changesets generated.
+*/
+#define SESSIONS_PATCHSET      0
+#define SESSIONS_CHANGESET     1
+#define SESSIONS_FULLCHANGESET 2
+
 static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
 
 typedef struct SessionHook SessionHook;
@@ -226383,6 +229095,7 @@ struct SessionInput {
 struct sqlite3_changeset_iter {
   SessionInput in;                /* Input buffer or stream */
   SessionBuffer tblhdr;           /* Buffer to hold apValue/zTab/abPK/ */
+  int bChangebatch;               /* True for changebatch_add() */
   int bPatchset;                  /* True if this is a patchset */
   int bInvert;                    /* True to invert changeset */
   int bSkipEmpty;                 /* Skip noop UPDATE changes */
@@ -228896,7 +231609,7 @@ static void sessionAppendCol(
 */
 static int sessionAppendUpdate(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int ePatchset,                  /* True for "patchset", 0 for "changeset" */
   sqlite3_stmt *pStmt,            /* Statement handle pointing at new row */
   SessionChange *p,               /* Object containing old values */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -228960,8 +231673,8 @@ static int sessionAppendUpdate(
 
     /* Add a field to the old.* record. This is omitted if this module is
     ** currently generating a patchset. */
-    if( bPatchset==0 ){
-      if( bChanged || abPK[i] ){
+    if( ePatchset!=SESSIONS_PATCHSET ){
+      if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){
         sessionAppendBlob(pBuf, pCsr, nAdvance, &rc);
       }else{
         sessionAppendByte(pBuf, 0, &rc);
@@ -228970,7 +231683,7 @@ static int sessionAppendUpdate(
 
     /* Add a field to the new.* record. Or the only record if currently
     ** generating a patchset.  */
-    if( bChanged || (bPatchset && abPK[i]) ){
+    if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){
       sessionAppendCol(&buf2, pStmt, i, &rc);
     }else{
       sessionAppendByte(&buf2, 0, &rc);
@@ -228996,7 +231709,7 @@ static int sessionAppendUpdate(
 */
 static int sessionAppendDelete(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int eChangeset,                 /* One of SESSIONS_CHANGESET etc. */
   SessionChange *p,               /* Object containing old values */
   int nCol,                       /* Number of columns in table */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -229006,7 +231719,7 @@ static int sessionAppendDelete(
   sessionAppendByte(pBuf, SQLITE_DELETE, &rc);
   sessionAppendByte(pBuf, p->bIndirect, &rc);
 
-  if( bPatchset==0 ){
+  if( eChangeset!=SESSIONS_PATCHSET ){
     sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc);
   }else{
     int i;
@@ -229232,12 +231945,12 @@ static int sessionSelectBind(
 */
 static void sessionAppendTableHdr(
   SessionBuffer *pBuf,            /* Append header to this buffer */
-  int bPatchset,                  /* Use the patchset format if true */
+  int ePatchset,                  /* Use the patchset format if true */
   SessionTable *pTab,             /* Table object to append header for */
   int *pRc                        /* IN/OUT: Error code */
 ){
   /* Write a table header */
-  sessionAppendByte(pBuf, (bPatchset ? 'P' : 'T'), pRc);
+  sessionAppendByte(pBuf, (ePatchset==SESSIONS_PATCHSET) ? 'P' : 'T', pRc);
   sessionAppendVarint(pBuf, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc);
@@ -229255,7 +231968,7 @@ static void sessionAppendTableHdr(
 */
 static int sessionGenerateChangeset(
   sqlite3_session *pSession,      /* Session object */
-  int bPatchset,                  /* True for patchset, false for changeset */
+  int ePatchset,                  /* One of SESSIONS_CHANGESET etc. */
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut,                     /* First argument for xOutput */
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
@@ -229300,7 +232013,7 @@ static int sessionGenerateChangeset(
       }
 
       /* Write a table header */
-      sessionAppendTableHdr(&buf, bPatchset, pTab, &rc);
+      sessionAppendTableHdr(&buf, ePatchset, pTab, &rc);
 
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
@@ -229326,10 +232039,10 @@ static int sessionGenerateChangeset(
               }
             }else{
               assert( pTab->abPK!=0 );
-              rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, pTab->abPK);
+              rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, pTab->abPK);
             }
           }else if( p->op!=SQLITE_INSERT ){
-            rc = sessionAppendDelete(&buf, bPatchset, p, pTab->nCol,pTab->abPK);
+            rc = sessionAppendDelete(&buf, ePatchset, p, pTab->nCol,pTab->abPK);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3_reset(pSel);
@@ -229388,7 +232101,8 @@ SQLITE_API int sqlite3session_changeset(
   int rc;
 
   if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE;
-  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset);
+  rc = sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset);
   assert( rc || pnChangeset==0
        || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize
   );
@@ -229404,7 +232118,8 @@ SQLITE_API int sqlite3session_changeset_strm(
   void *pOut
 ){
   if( xOutput==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -229416,7 +232131,8 @@ SQLITE_API int sqlite3session_patchset_strm(
   void *pOut
 ){
   if( xOutput==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -229432,9 +232148,20 @@ SQLITE_API int sqlite3session_patchset(
   void **ppPatchset               /* OUT: Buffer containing changeset */
 ){
   if( pnPatchset==0 || ppPatchset==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset);
+}
+
+SQLITE_API int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+){
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset);
 }
 
+
 /*
 ** Enable or disable the session object passed as the first argument.
 */
@@ -229587,8 +232314,13 @@ SQLITE_API int sqlite3changeset_start_v2(
   void *pChangeset,               /* Pointer to buffer containing changeset */
   int flags
 ){
+  int rc;
+  sqlite3_changeset_iter *pIter = 0;
   int bInvert = !!(flags & SQLITE_CHANGESETSTART_INVERT);
-  return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset, bInvert, 0);
+  rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInvert, 0);
+  if( pIter && (flags & SQLITE_CHANGESETSTART_FULL) ) pIter->bChangebatch = 1;
+  *pp = pIter;
+  return rc;
 }
 
 /*
@@ -230065,8 +232797,12 @@ static int sessionChangesetNextOne(
     **
     ** Such records are technically corrupt, but the rebaser was at one
     ** point generating them. Under most circumstances this is benign, but
-    ** can cause spurious SQLITE_RANGE errors when applying the changeset. */
-    if( p->bPatchset==0 && p->op==SQLITE_UPDATE){
+    ** can cause spurious SQLITE_RANGE errors when applying the changeset.
+    **
+    ** Update for bedrock branch: Do not do this for changebatch_add() on
+    ** this branch, as changesets generated by sqlite3sessions_fullchangeset()
+    ** also have this property.  */
+    if( p->bChangebatch==0 && p->bPatchset==0 && p->op==SQLITE_UPDATE){
       for(i=0; i<p->nCol; i++){
         if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){
           sqlite3ValueFree(p->apValue[i]);
@@ -232235,10 +234971,11 @@ static int sessionChangegroupOutput(
   ** hash tables attached to the SessionTable objects in list p->pList.
   */
   for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){
+    int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET;
     int i;
     if( pTab->nEntry==0 ) continue;
 
-    sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc);
+    sessionAppendTableHdr(&buf, eChangeset, pTab, &rc);
     for(i=0; i<pTab->nChange; i++){
       SessionChange *p;
       for(p=pTab->apChange[i]; p; p=p->pNext){
@@ -253389,6 +256126,7 @@ static int fts5UpdateMethod(
   }
 
  update_out:
+  sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -254905,7 +257643,7 @@ static void fts5SourceIdFunc(
 ){
   assert( nArg==0 );
   UNUSED_PARAM2(nArg, apUnused);
-  sqlite3_result_text(pCtx, "fts5: 2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e", -1, SQLITE_TRANSIENT);
+  sqlite3_result_text(pCtx, "fts5: 2024-10-21 17:10:05 d2d954d43abe20a3cf10cc12f0f9922ba6b94584e39b1ef1991f1d354058c99a", -1, SQLITE_TRANSIENT);
 }
 
 /*
diff --git a/deps/sqlite3/sqlite3.h b/deps/sqlite3/sqlite3.h
index eaffd1ec..0bdc9bd7 100644
--- a/deps/sqlite3/sqlite3.h
+++ b/deps/sqlite3/sqlite3.h
@@ -148,7 +148,7 @@ extern "C" {
 */
 #define SQLITE_VERSION        "3.47.0"
 #define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e"
+#define SQLITE_SOURCE_ID      "2024-10-21 17:10:05 d2d954d43abe20a3cf10cc12f0f9922ba6b94584e39b1ef1991f1d354058c99a"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -10715,6 +10715,31 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** CAPI3REF: Wal related information regarding the most recent COMMIT
+** EXPERIMENTAL
+**
+** This function reports on the state of the wal file (if any) for database
+** zDb, which should be "main", "temp", or the name of the attached database.
+** Its results - the values written to the output parameters - are only
+** defined if the most recent SQL command on the connection was a successful
+** COMMIT that wrote data to wal-mode database zDb.
+**
+** Assuming the above conditions are met, output parameter (*pnFrame) is set
+** to the total number of frames in the wal file. Parameter (*pnPrior) is
+** set to the number of frames that were present in the wal file before the
+** most recent transaction was committed. So that the number of frames written
+** by the most recent transaction is (*pnFrame)-(*pnPrior).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It
+** is not an error if this function is called at a time when the results
+** are undefined.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb,
+  unsigned int *pnPrior, unsigned int *pnFrame
+);
+
 /*
 ** CAPI3REF: Serialize a database
 **
@@ -10858,6 +10883,98 @@ SQLITE_API int sqlite3_deserialize(
 #define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
 #define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */
 
+/*
+** Access details of recent COMMIT commands. This function allows various
+** details related to the most recent COMMIT command to be accessed.
+** The requested value is always returned via output parameter (*piVal).
+** The specific value requested is identified by parameter op (see
+** below).
+**
+** SQLITE_OK is returned if successful, or SQLITE_ERROR if the "op" or
+** "zDb" paramters are unrecognized.
+*/
+SQLITE_API int sqlite3_commit_status(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Name of database - "main" etc. */
+  int op,                         /* SQLITE_COMMIT_XXX constant */
+  unsigned int *piVal             /* OUT: Write requested value here */
+);
+
+/*
+** The following describes the five requests supported by
+** sqlite3_commit_status(), each identified by an SQLITE_COMMIT_XXX
+** constant:
+**
+** SQLITE_COMMIT_FIRSTFRAME:
+**   In this case argument zDb must be "main", or "temp", or else the name of
+**   an attached database. If zDb does not correspond to any attached database,
+**   SQLITE_ERROR is returned.
+**
+**   The final value of (*piVal) for this request is only defined if (a) the
+**   most recent attempt to write to the database connection was successful,
+**   (b) the most recent attempt to write to the database did write to database
+**   zDb, and (c) zDb is a wal mode database.
+**
+**   If the above conditions are true, then output parameter (*piVal) is
+**   set to the frame number of the first frame written by the recent
+**   transaction. In wal mode, or in wal2 mode when a transaction is
+**   written into the *-wal file, the frame number indicates the frame's
+**   position in the wal file - frames are numbered starting from 1. In
+**   wal2 mode, when a transaction is written to the *-wal2 file, the frame
+**   number is the frame's position in the *-wal2 file, plus (1 << 31).
+**
+**   Note: Although the a database may have up to (1<<32) pages, each wal
+**   file may contain at most (1<<31) frames.
+**
+** SQLITE_COMMIT_NFRAME:
+**   zDb is interpreted in the same way as, and the final value of (*piVal)
+**   is undefined, for SQLITE_COMMIT_FIRSTFRAME.
+**
+**   Otherwise, (*piVal) is set to the number of frames written by the
+**   recent transaction.
+**
+** SQLITE_COMMIT_CONFLICT_DB:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the most recent attempt to write to
+**   the database handle was a BEGIN CONCURRENT transaction that
+**   failed with an SQLITE_BUSY_SNAPSHOT error.
+**
+**   In other cases, (*piVal) is set to the index of the database
+**   on which the SQLITE_BUSY_SNAPSHOT error occurred (0 for main,
+**   a value of 2 or greater for an attached database). This value
+**   may be used with the sqlite3_db_name() API to find the name
+**   of the conflicting database.
+**
+** SQLITE_COMMIT_CONFLICT_FRAME:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the most recent attempt to write to
+**   the database handle was a BEGIN CONCURRENT transaction that
+**   failed with an SQLITE_BUSY_SNAPSHOT error.
+**
+**   (*piVal) is set to the frame number of the conflicting frame for
+**   the recent SQLITE_BUSY_SNAPSHOT error. The conflicting transaction may
+**   be found by comparing this value with the FIRSTFRAME and
+**   NFRAME values for recent succesfully committed transactions on
+**   the same db. If the CONFLICT_FRAME value is F, then the conflicting
+**   transaction is the most recent successful commit for which
+**   (FIRSTFRAME <= F <= FIRSTFRAME+NFRAME) is true.
+**
+** SQLITE_COMMIT_CONFLICT_PGNO:
+**   Parameter zDb is ignored for this request. The results of this
+**   request are only defined if the previous attempt to write to
+**   the database using database handle db failed with
+**   SQLITE_BUSY_SNAPSHOT.
+**
+**   Return the page number of the conflicting page for the most
+**   recent SQLITE_BUSY_SNAPSHOT error.
+*/
+#define SQLITE_COMMIT_FIRSTFRAME     0
+#define SQLITE_COMMIT_NFRAME         1
+#define SQLITE_COMMIT_CONFLICT_DB    2
+#define SQLITE_COMMIT_CONFLICT_FRAME 3
+#define SQLITE_COMMIT_CONFLICT_PGNO  4
+
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -11371,6 +11488,32 @@ SQLITE_API int sqlite3session_changeset(
   void **ppChangeset              /* OUT: Buffer containing changeset */
 );
 
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+SQLITE_API int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+SQLITE_API int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
 /*
 ** CAPI3REF: Return An Upper-limit For The Size Of The Changeset
 ** METHOD: sqlite3_session
@@ -11571,12 +11714,16 @@ SQLITE_API int sqlite3changeset_start_v2(
 ** The following flags may passed via the 4th parameter to
 ** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]:
 **
-** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
+** <dt>SQLITE_CHANGESETSTART_INVERT <dd>
 **   Invert the changeset while iterating through it. This is equivalent to
 **   inverting a changeset using sqlite3changeset_invert() before applying it.
 **   It is an error to specify this flag with a patchset.
+**
+** <dt>SQLITE_CHANGESETSTART_FULL <dd>
+**   Do not trim extra fields added to fullchangeset changesets.
 */
 #define SQLITE_CHANGESETSTART_INVERT        0x0002
+#define SQLITE_CHANGESETSTART_FULL          0x0004
 
 
 /*
diff --git a/docs/compilation.md b/docs/compilation.md
index e6974a78..360ae0d3 100644
--- a/docs/compilation.md
+++ b/docs/compilation.md
@@ -43,7 +43,7 @@ If you're using a SQLite3 encryption extension that is a drop-in replacement for
 
 # Bundled configuration
 
-By default, this distribution currently uses SQLite3 **version 3.47.0** with the following [compilation options](https://www.sqlite.org/compile.html):
+By default, this distribution currently uses SQLite3 **version ..** with the following [compilation options](https://www.sqlite.org/compile.html):
 
 ```
 HAVE_INT16_T=1
@@ -81,4 +81,5 @@ SQLITE_SOUNDEX
 SQLITE_THREADSAFE=2
 SQLITE_TRACE_SIZE_LIMIT=32
 SQLITE_USE_URI=0
+SQLITE_OMIT_LOAD_EXTENSION
 ```
diff --git a/install.js b/install.js
new file mode 100644
index 00000000..4b84985e
--- /dev/null
+++ b/install.js
@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+/**
+ * Install script for @rocicorp/zero-sqlite3
+ * 
+ * Supports skipping the build process via ZERO_SKIP_SQLITE3_BUILD environment variable.
+ * This is useful in Docker builds and CI/CD pipelines where the native module
+ * might be pre-built or not needed.
+ */
+
+const { execSync } = require('child_process');
+
+if (process.env.ZERO_SKIP_SQLITE3_BUILD === 'true') {
+  console.log('[zero-sqlite3] Skipping native build (ZERO_SKIP_SQLITE3_BUILD=true)');
+  console.log('[zero-sqlite3] Warning: Native module will not be available unless pre-built');
+  process.exit(0);
+}
+
+console.log('[zero-sqlite3] Building native module...');
+
+try {
+  // Try to download pre-built binaries first
+  execSync('prebuild-install', { stdio: 'inherit' });
+  console.log('[zero-sqlite3] Successfully installed pre-built binaries');
+} catch (error) {
+  // Fall back to building from source
+  console.log('[zero-sqlite3] Pre-built binaries not available, building from source...');
+  try {
+    execSync('node-gyp rebuild --release', { stdio: 'inherit' });
+    console.log('[zero-sqlite3] Successfully built from source');
+  } catch (buildError) {
+    console.error('[zero-sqlite3] Failed to build native module');
+    process.exit(1);
+  }
+}
\ No newline at end of file
diff --git a/lib/database.js b/lib/database.js
index 14e05739..4bf6c56d 100644
--- a/lib/database.js
+++ b/lib/database.js
@@ -80,7 +80,6 @@ Database.prototype.serialize = require('./methods/serialize');
 Database.prototype.function = require('./methods/function');
 Database.prototype.aggregate = require('./methods/aggregate');
 Database.prototype.table = require('./methods/table');
-Database.prototype.loadExtension = wrappers.loadExtension;
 Database.prototype.exec = wrappers.exec;
 Database.prototype.close = wrappers.close;
 Database.prototype.defaultSafeIntegers = wrappers.defaultSafeIntegers;
diff --git a/lib/index.d.ts b/lib/index.d.ts
new file mode 100644
index 00000000..dc6eff86
--- /dev/null
+++ b/lib/index.d.ts
@@ -0,0 +1,151 @@
+/// <reference types="node" />
+
+// FIXME: Is this `any` really necessary?
+type VariableArgFunction = (...params: any[]) => unknown;
+type ArgumentTypes<F extends VariableArgFunction> = F extends (...args: infer A) => unknown ? A : never;
+type ElementOf<T> = T extends Array<infer E> ? E : T;
+
+declare namespace BetterSqlite3 {
+    interface Statement<BindParameters extends unknown[], Result = unknown> {
+        database: Database;
+        source: string;
+        reader: boolean;
+        readonly: boolean;
+        busy: boolean;
+
+        run(...params: BindParameters): Database.RunResult;
+        get(...params: BindParameters): Result | undefined;
+        all(...params: BindParameters): Result[];
+        iterate(...params: BindParameters): IterableIterator<Result>;
+        pluck(toggleState?: boolean): this;
+        expand(toggleState?: boolean): this;
+        raw(toggleState?: boolean): this;
+        bind(...params: BindParameters): this;
+        columns(): ColumnDefinition[];
+        safeIntegers(toggleState?: boolean): this;
+    }
+
+    interface ColumnDefinition {
+        name: string;
+        column: string | null;
+        table: string | null;
+        database: string | null;
+        type: string | null;
+    }
+
+    interface Transaction<F extends VariableArgFunction> {
+        (...params: ArgumentTypes<F>): ReturnType<F>;
+        default(...params: ArgumentTypes<F>): ReturnType<F>;
+        deferred(...params: ArgumentTypes<F>): ReturnType<F>;
+        immediate(...params: ArgumentTypes<F>): ReturnType<F>;
+        exclusive(...params: ArgumentTypes<F>): ReturnType<F>;
+    }
+
+    interface VirtualTableOptions {
+        rows: (...params: unknown[]) => Generator;
+        columns: string[];
+        parameters?: string[] | undefined;
+        safeIntegers?: boolean | undefined;
+        directOnly?: boolean | undefined;
+    }
+
+    interface Database {
+        memory: boolean;
+        readonly: boolean;
+        name: string;
+        open: boolean;
+        inTransaction: boolean;
+
+        prepare<BindParameters extends unknown[] | {} = unknown[], Result = unknown>(
+            source: string,
+        ): BindParameters extends unknown[] ? Statement<BindParameters, Result> : Statement<[BindParameters], Result>;
+        transaction<F extends VariableArgFunction>(fn: F): Transaction<F>;
+        exec(source: string): this;
+        pragma(source: string, options?: Database.PragmaOptions): unknown;
+        function(name: string, cb: (...params: unknown[]) => unknown): this;
+        function(name: string, options: Database.RegistrationOptions, cb: (...params: unknown[]) => unknown): this;
+        aggregate<T>(
+            name: string,
+            options: Database.RegistrationOptions & {
+                start?: T | (() => T);
+                // eslint-disable-next-line @typescript-eslint/no-invalid-void-type
+                step: (total: T, next: ElementOf<T>) => T | void;
+                inverse?: ((total: T, dropped: T) => T) | undefined;
+                result?: ((total: T) => unknown) | undefined;
+            },
+        ): this;
+        loadExtension(path: string): this;
+        close(): this;
+        defaultSafeIntegers(toggleState?: boolean): this;
+        backup(destinationFile: string, options?: Database.BackupOptions): Promise<Database.BackupMetadata>;
+        table(name: string, options: VirtualTableOptions): this;
+        unsafeMode(unsafe?: boolean): this;
+        serialize(options?: Database.SerializeOptions): Buffer;
+    }
+
+    interface DatabaseConstructor {
+        new(filename?: string | Buffer, options?: Database.Options): Database;
+        (filename?: string, options?: Database.Options): Database;
+        prototype: Database;
+
+        SqliteError: typeof SqliteError;
+    }
+}
+
+declare class SqliteError extends Error {
+    name: string;
+    message: string;
+    code: string;
+    constructor(message: string, code: string);
+}
+
+declare namespace Database {
+    interface RunResult {
+        changes: number;
+        lastInsertRowid: number | bigint;
+    }
+
+    interface Options {
+        readonly?: boolean | undefined;
+        fileMustExist?: boolean | undefined;
+        timeout?: number | undefined;
+        verbose?: ((message?: unknown, ...additionalArgs: unknown[]) => void) | undefined;
+        nativeBinding?: string | undefined;
+    }
+
+    interface SerializeOptions {
+        attached?: string;
+    }
+
+    interface PragmaOptions {
+        simple?: boolean | undefined;
+    }
+
+    interface RegistrationOptions {
+        varargs?: boolean | undefined;
+        deterministic?: boolean | undefined;
+        safeIntegers?: boolean | undefined;
+        directOnly?: boolean | undefined;
+    }
+
+    type AggregateOptions = Parameters<BetterSqlite3.Database["aggregate"]>[1];
+
+    interface BackupMetadata {
+        totalPages: number;
+        remainingPages: number;
+    }
+    interface BackupOptions {
+        progress: (info: BackupMetadata) => number;
+    }
+
+    type SqliteError = typeof SqliteError;
+    type Statement<BindParameters extends unknown[] | {} = unknown[], Result = unknown> = BindParameters extends
+        unknown[] ? BetterSqlite3.Statement<BindParameters, Result>
+        : BetterSqlite3.Statement<[BindParameters], Result>;
+    type ColumnDefinition = BetterSqlite3.ColumnDefinition;
+    type Transaction<T extends VariableArgFunction = VariableArgFunction> = BetterSqlite3.Transaction<T>;
+    type Database = BetterSqlite3.Database;
+}
+
+declare const Database: BetterSqlite3.DatabaseConstructor;
+export = Database;
\ No newline at end of file
diff --git a/lib/methods/wrappers.js b/lib/methods/wrappers.js
index 1d4673e5..0914ed3e 100644
--- a/lib/methods/wrappers.js
+++ b/lib/methods/wrappers.js
@@ -15,11 +15,6 @@ exports.close = function close() {
 	return this;
 };
 
-exports.loadExtension = function loadExtension(...args) {
-	this[cppdb].loadExtension(...args);
-	return this;
-};
-
 exports.defaultSafeIntegers = function defaultSafeIntegers(...args) {
 	this[cppdb].defaultSafeIntegers(...args);
 	return this;
diff --git a/package.json b/package.json
index 02ab0218..bca179e6 100644
--- a/package.json
+++ b/package.json
@@ -1,24 +1,35 @@
 {
-  "name": "better-sqlite3",
-  "version": "11.5.0",
-  "description": "The fastest and simplest library for SQLite3 in Node.js.",
-  "homepage": "http://github.com/WiseLibs/better-sqlite3",
-  "author": "Joshua Wise <joshuathomaswise@gmail.com>",
+  "name": "@rocicorp/zero-sqlite3",
+  "version": "1.0.8",
+  "description": "better-sqlite3 on bedrock",
+  "homepage": "https://github.com/rocicorp/zero-sqlite3",
+  "author": "Rocicorp",
   "repository": {
     "type": "git",
-    "url": "git://github.com/WiseLibs/better-sqlite3.git"
+    "url": "git://github.com/rocicorp/zero-sqlite3.git"
   },
   "main": "lib/index.js",
+  "bin": {
+    "zero-sqlite3": "./shell.ps1"
+  },
   "files": [
     "binding.gyp",
     "src/*.[ch]pp",
     "lib/**",
-    "deps/**"
+    "deps/**",
+    "shell.sh",
+    "install.js"
   ],
+  "types": "lib/index.d.ts",
   "dependencies": {
     "bindings": "^1.5.0",
     "prebuild-install": "^7.1.1"
   },
+  "overrides": {
+    "prebuild": {
+      "node-abi": "4.11.0"
+    }
+  },
   "devDependencies": {
     "chai": "^4.3.8",
     "cli-color": "^2.0.3",
@@ -30,7 +41,7 @@
     "sqlite3": "^5.1.6"
   },
   "scripts": {
-    "install": "prebuild-install || node-gyp rebuild --release",
+    "install": "node install.js",
     "build-release": "node-gyp rebuild --release",
     "build-debug": "node-gyp rebuild --debug",
     "rebuild-release": "npm run lzz && npm run build-release",
diff --git a/shell.ps1 b/shell.ps1
new file mode 100755
index 00000000..a3359514
--- /dev/null
+++ b/shell.ps1
@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+echo --% >/dev/null;: ' | out-null
+<#'
+
+
+#
+# sh part
+#
+"$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/build/Release/zero_sqlite3" "$@"
+# end bash part
+
+exit #>
+
+
+#
+# powershell part
+#
+$scriptDir = (Get-Item -Path $MyInvocation.MyCommand.Definition).DirectoryName
+& (Join-Path $scriptDir "build" "Release" "zero_sqlite3.exe") @args
+
diff --git a/src/better_sqlite3.cpp b/src/better_sqlite3.cpp
index 26fb9fab..6e596b59 100644
--- a/src/better_sqlite3.cpp
+++ b/src/better_sqlite3.cpp
@@ -342,7 +342,6 @@ v8::Local <v8 :: Function> Database::Init (v8::Isolate * isolate, v8::Local <v8
                 SetPrototypeMethod(isolate, data, t, "function", JS_function);
                 SetPrototypeMethod(isolate, data, t, "aggregate", JS_aggregate);
                 SetPrototypeMethod(isolate, data, t, "table", JS_table);
-                SetPrototypeMethod(isolate, data, t, "loadExtension", JS_loadExtension);
                 SetPrototypeMethod(isolate, data, t, "close", JS_close);
                 SetPrototypeMethod(isolate, data, t, "defaultSafeIntegers", JS_defaultSafeIntegers);
                 SetPrototypeMethod(isolate, data, t, "unsafeMode", JS_unsafeMode);
@@ -680,30 +679,6 @@ void Database::JS_table (v8::FunctionCallbackInfo <v8 :: Value> const & info)
                 }
                 db->busy = false;
 }
-#line 370 "./src/objects/database.lzz"
-void Database::JS_loadExtension (v8::FunctionCallbackInfo <v8 :: Value> const & info)
-#line 370 "./src/objects/database.lzz"
-                                      {
-                Database* db = node :: ObjectWrap :: Unwrap <Database>(info.This());
-                v8::Local<v8::String> entryPoint;
-                if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > filename = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
-                if (info.Length() > 1) { if ( info . Length ( ) <= ( 1 ) || ! info [ 1 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "second" " argument to be " "a string" ) ; entryPoint = ( info [ 1 ] . As < v8 :: String > ( ) ) ; }
-                if ( ! db -> open ) return ThrowTypeError ( "The database connection is not open" ) ;
-                if ( db -> busy ) return ThrowTypeError ( "This database connection is busy executing a query" ) ;
-                if ( db -> iterators ) return ThrowTypeError ( "This database connection is busy executing a query" ) ;
-                v8 :: Isolate * isolate = info . GetIsolate ( ) ;
-                char* error;
-                int status = sqlite3_load_extension(
-                        db->db_handle,
-                        *v8::String::Utf8Value(isolate, filename),
-                        entryPoint.IsEmpty() ? NULL : *v8::String::Utf8Value(isolate, entryPoint),
-                        &error
-                );
-                if (status != SQLITE_OK) {
-                        ThrowSqliteError(db->addon, error, status);
-                }
-                sqlite3_free(error);
-}
 #line 392 "./src/objects/database.lzz"
 void Database::JS_close (v8::FunctionCallbackInfo <v8 :: Value> const & info)
 #line 392 "./src/objects/database.lzz"
diff --git a/src/better_sqlite3.hpp b/src/better_sqlite3.hpp
index 92317ba2..eb887f28 100644
--- a/src/better_sqlite3.hpp
+++ b/src/better_sqlite3.hpp
@@ -264,8 +264,6 @@ class Database : public node::ObjectWrap
   static void JS_aggregate (v8::FunctionCallbackInfo <v8 :: Value> const & info);
 #line 350 "./src/objects/database.lzz"
   static void JS_table (v8::FunctionCallbackInfo <v8 :: Value> const & info);
-#line 370 "./src/objects/database.lzz"
-  static void JS_loadExtension (v8::FunctionCallbackInfo <v8 :: Value> const & info);
 #line 392 "./src/objects/database.lzz"
   static void JS_close (v8::FunctionCallbackInfo <v8 :: Value> const & info);
 #line 402 "./src/objects/database.lzz"
diff --git a/src/objects/database.lzz b/src/objects/database.lzz
index 28294fa3..96a84f8c 100644
--- a/src/objects/database.lzz
+++ b/src/objects/database.lzz
@@ -10,7 +10,6 @@ public:
 		SetPrototypeMethod(isolate, data, t, "function", JS_function);
 		SetPrototypeMethod(isolate, data, t, "aggregate", JS_aggregate);
 		SetPrototypeMethod(isolate, data, t, "table", JS_table);
-		SetPrototypeMethod(isolate, data, t, "loadExtension", JS_loadExtension);
 		SetPrototypeMethod(isolate, data, t, "close", JS_close);
 		SetPrototypeMethod(isolate, data, t, "defaultSafeIntegers", JS_defaultSafeIntegers);
 		SetPrototypeMethod(isolate, data, t, "unsafeMode", JS_unsafeMode);
@@ -367,28 +366,6 @@ private:
 		db->busy = false;
 	}
 
-	NODE_METHOD(JS_loadExtension) {
-		Database* db = Unwrap<Database>(info.This());
-		v8::Local<v8::String> entryPoint;
-		REQUIRE_ARGUMENT_STRING(first, v8::Local<v8::String> filename);
-		if (info.Length() > 1) { REQUIRE_ARGUMENT_STRING(second, entryPoint); }
-		REQUIRE_DATABASE_OPEN(db);
-		REQUIRE_DATABASE_NOT_BUSY(db);
-		REQUIRE_DATABASE_NO_ITERATORS(db);
-		UseIsolate;
-		char* error;
-		int status = sqlite3_load_extension(
-			db->db_handle,
-			*v8::String::Utf8Value(isolate, filename),
-			entryPoint.IsEmpty() ? NULL : *v8::String::Utf8Value(isolate, entryPoint),
-			&error
-		);
-		if (status != SQLITE_OK) {
-			ThrowSqliteError(db->addon, error, status);
-		}
-		sqlite3_free(error);
-	}
-
 	NODE_METHOD(JS_close) {
 		Database* db = Unwrap<Database>(info.This());
 		if (db->open) {
diff --git a/test/35.database.load-extension.js b/test/35.database.load-extension.js
deleted file mode 100644
index e9d81818..00000000
--- a/test/35.database.load-extension.js
+++ /dev/null
@@ -1,75 +0,0 @@
-'use strict';
-const fs = require('fs');
-const path = require('path');
-const Database = require('../.');
-
-describe('Database#loadExtension()', function () {
-	let filepath;
-	before(function () {
-		const releaseFilepath = path.join(__dirname, '..', 'build', 'Release', 'test_extension.node');
-		const debugFilepath = path.join(__dirname, '..', 'build', 'Debug', 'test_extension.node');
-		try {
-			fs.accessSync(releaseFilepath);
-			filepath = releaseFilepath;
-		} catch (_) {
-			fs.accessSync(debugFilepath);
-			filepath = debugFilepath;
-		}
-	});
-	beforeEach(function () {
-		this.db = new Database(util.next());
-	});
-	afterEach(function () {
-		this.db.close();
-	});
-
-	it('should throw an exception if a string argument is not given', function () {
-		expect(() => this.db.loadExtension()).to.throw(TypeError);
-		expect(() => this.db.loadExtension(undefined)).to.throw(TypeError);
-		expect(() => this.db.loadExtension(null)).to.throw(TypeError);
-		expect(() => this.db.loadExtension(123)).to.throw(TypeError);
-		expect(() => this.db.loadExtension(new String(filepath))).to.throw(TypeError);
-		expect(() => this.db.loadExtension([filepath])).to.throw(TypeError);
-	});
-	it('should throw an exception if the database is busy', function () {
-		let invoked = false;
-		for (const value of this.db.prepare('select 555').pluck().iterate()) {
-			expect(value).to.equal(555);
-			expect(() => this.db.loadExtension(filepath)).to.throw(TypeError);
-			invoked = true;
-		}
-		expect(invoked).to.be.true;
-	});
-	it('should throw an exception if the extension is not found', function () {
-		try {
-			this.db.loadExtension(filepath + 'x');
-		} catch (err) {
-			expect(err).to.be.an.instanceof(Database.SqliteError);
-			expect(err.message).to.be.a('string');
-			expect(err.message.length).to.be.above(0);
-			expect(err.message).to.not.equal('not an error');
-			expect(err.code).to.equal('SQLITE_ERROR');
-			return;
-		}
-		throw new Error('This code should not have been reached');
-	});
-	it('should register the specified extension', function () {
-		expect(this.db.loadExtension(filepath)).to.equal(this.db);
-		expect(this.db.prepare('SELECT testExtensionFunction(NULL, 123, 99, 2)').pluck().get()).to.equal(4);
-		expect(this.db.prepare('SELECT testExtensionFunction(NULL, 2)').pluck().get()).to.equal(2);
-	});
-	it('should not allow registering extensions with SQL', function () {
-		expect(() => this.db.prepare('SELECT load_extension(?)').get(filepath)).to.throw(Database.SqliteError);
-		expect(this.db.loadExtension(filepath)).to.equal(this.db);
-		expect(() => this.db.prepare('SELECT load_extension(?)').get(filepath)).to.throw(Database.SqliteError);
-		this.db.close();
-		this.db = new Database(util.next());
-		try {
-			this.db.loadExtension(filepath + 'x');
-		} catch (err) {
-			expect(() => this.db.prepare('SELECT load_extension(?)').get(filepath)).to.throw(Database.SqliteError);
-			return;
-		}
-		throw new Error('This code should not have been reached');
-	});
-});
diff --git a/test/42.integrity.js b/test/42.integrity.js
index ffa7009c..fa99c2cb 100644
--- a/test/42.integrity.js
+++ b/test/42.integrity.js
@@ -187,33 +187,6 @@ describe('integrity checks', function () {
 		});
 	});
 
-	describe('Database#loadExtension()', function () {
-		let filepath;
-		before(function () {
-			const releaseFilepath = path.join(__dirname, '..', 'build', 'Release', 'test_extension.node');
-			const debugFilepath = path.join(__dirname, '..', 'build', 'Debug', 'test_extension.node');
-			try {
-				fs.accessSync(releaseFilepath);
-				filepath = releaseFilepath;
-			} catch (_) {
-				fs.accessSync(debugFilepath);
-				filepath = debugFilepath;
-			}
-		});
-
-		specify('while iterating (blocked)', function () {
-			whileIterating(this, blocked(() => this.db.loadExtension(filepath)));
-			normally(allowed(() => this.db.loadExtension(filepath)));
-		});
-		specify('while busy (blocked)', function () {
-			whileBusy(this, blocked(() => this.db.loadExtension(filepath)));
-			normally(allowed(() => this.db.loadExtension(filepath)));
-		});
-		specify('while closed (blocked)', function () {
-			whileClosed(this, blocked(() => this.db.loadExtension(filepath)));
-		});
-	});
-
 	describe('Database#close()', function () {
 		specify('while iterating (blocked)', function () {
 			whileIterating(this, blocked(() => this.db.close()));