diff --git a/src/.vuepress/sidebar/V1.3.x/en.ts b/src/.vuepress/sidebar/V1.3.x/en.ts index 828ee130c..d91f910b0 100644 --- a/src/.vuepress/sidebar/V1.3.x/en.ts +++ b/src/.vuepress/sidebar/V1.3.x/en.ts @@ -158,9 +158,11 @@ export const enSidebar = { { text: 'Data Import Export(V1.3.0/1/2)', link: 'Data-Import-Export-Tool' }, { text: 'TsFile Import Export(V1.3.0/1/2)', link: 'TsFile-Import-Export-Tool' }, { text: 'Data Import(V1.3.3)', link: 'Data-Import-Tool' }, - { text: 'Data Export(V1.3.3)', link: 'Data-Export-Tool' }, + { text: 'Data Import(V1.3.4)', link: 'Data-Import-Tool-1-3-4' }, + { text: 'Data Export(V1.3.3)', link: 'Data-Export-Tool' }, + { text: 'Data Export(V1.3.4)', link: 'Data-Export-Tool-1-3-4' }, { text: 'Full Backup Tool(V1.3.2)', link: 'Backup-Tool' }, - { text: 'Health Check Tool(V1.3.2)', link: 'Health-Check-Tool' }, + { text: 'Health Check Tool(V1.3.2)', link: 'Health-Check-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar/V1.3.x/zh.ts b/src/.vuepress/sidebar/V1.3.x/zh.ts index 65ac38299..e85b9eff9 100644 --- a/src/.vuepress/sidebar/V1.3.x/zh.ts +++ b/src/.vuepress/sidebar/V1.3.x/zh.ts @@ -145,9 +145,11 @@ export const zhSidebar = { { text: '数据导入导出(V1.3.0/1/2)', link: 'Data-Import-Export-Tool' }, { text: 'TsFile导入导出(V1.3.0/1/2)', link: 'TsFile-Import-Export-Tool' }, { text: '数据导入(V1.3.3)', link: 'Data-Import-Tool' }, - { text: '数据导出(V1.3.3)', link: 'Data-Export-Tool' }, + { text: '数据导入(V1.3.4)', link: 'Data-Import-Tool-1-3-4' }, + { text: '数据导出(V1.3.3)', link: 'Data-Export-Tool' }, + { text: '数据导出(V1.3.4)', link: 'Data-Export-Tool-1-3-4' }, { text: '全量备份工具(V1.3.2)', link: 'Backup-Tool' }, - { text: '健康检查工具(V1.3.2)', link: 'Health-Check-Tool' }, + { text: '健康检查工具(V1.3.2)', link: 'Health-Check-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V1.3.x/en.ts b/src/.vuepress/sidebar_timecho/V1.3.x/en.ts index e10c5233f..8ee86628c 100644 --- a/src/.vuepress/sidebar_timecho/V1.3.x/en.ts +++ b/src/.vuepress/sidebar_timecho/V1.3.x/en.ts @@ -172,9 +172,11 @@ export const enSidebar = { { text: 'Data Import Export(V1.3.0/1/2)', link: 'Data-Import-Export-Tool' }, { text: 'TsFile Import Export(V1.3.0/1/2)', link: 'TsFile-Import-Export-Tool' }, { text: 'Data Import(V1.3.3)', link: 'Data-Import-Tool' }, - { text: 'Data Export(V1.3.3)', link: 'Data-Export-Tool' }, + { text: 'Data Import(V1.3.4)', link: 'Data-Import-Tool-1-3-4' }, + { text: 'Data Export(V1.3.3)', link: 'Data-Export-Tool' }, + { text: 'Data Export(V1.3.4)', link: 'Data-Export-Tool-1-3-4' }, { text: 'Full Backup Tool(V1.3.2)', link: 'Backup-Tool' }, - { text: 'Health Check Tool(V1.3.2)', link: 'Health-Check-Tool' }, + { text: 'Health Check Tool(V1.3.2)', link: 'Health-Check-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts b/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts index 7f1b9533b..0e0a980b1 100644 --- a/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts +++ b/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts @@ -155,9 +155,11 @@ export const zhSidebar = { { text: '数据导入导出(V1.3.0/1/2)', link: 'Data-Import-Export-Tool' }, { text: 'TsFile导入导出(V1.3.0/1/2)', link: 'TsFile-Import-Export-Tool' }, { text: '数据导入(V1.3.3)', link: 'Data-Import-Tool' }, - { text: '数据导出(V1.3.3)', link: 'Data-Export-Tool' }, + { text: '数据导入(V1.3.4)', link: 'Data-Import-Tool-1-3-4' }, + { text: '数据导出(V1.3.3)', link: 'Data-Export-Tool' }, + { text: '数据导出(V1.3.4)', link: 'Data-Export-Tool-1-3-4' }, { text: '全量备份工具(V1.3.2)', link: 'Backup-Tool' }, - { text: '健康检查工具(V1.3.2)', link: 'Health-Check-Tool' }, + { text: '健康检查工具(V1.3.2)', link: 'Health-Check-Tool' }, ], }, { diff --git a/src/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md b/src/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md index 75229ae09..3c71156b3 100644 --- a/src/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md +++ b/src/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md @@ -1,218 +1,200 @@ # Data Import -## 1. IoTDB Data Import - -IoTDB currently supports importing data in CSV, SQL, and TsFile (IoTDB's underlying open-time series file format) into the database. The specific functionalities are as follows: +## 1. Overview +IoTDB supports two methods for data import: +* Data Import Tool: Use the import-data.sh (Unix/OS X) or import-data.bat (Windows) script in the tools directory to manually import CSV, SQL, or TsFile (open-source time-series file format) data into IoTDB. +* TsFile Auto-Loading Feature - + - + - - + - - + - - + +
File Format
File Format IoTDB Tool Description
CSVimport-data.sh/batimport-data.sh/bat Can be used for single or batch import of CSV files into IoTDB
SQLimport-data.sh/batSQL Can be used for single or batch import of SQL files into IoTDB
TsFileload-tsfile.sh/batTsFile Can be used for single or batch import of TsFile files into IoTDB
TsFile Active Listening & Loading FeatureAccording to user configuration, it listens for changes in TsFile files in the specified path and loads newly added TsFile files into IoTDBTsFile Auto-Loading FeatureCan automatically monitor a specified directory for newly generated TsFiles and load them into IoTDB.
-## 2. import-data Scripts - -- Supported formats: CSV、SQL - -### 2.1 Command - -```Bash -# Unix/OS X ->tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] - -# Windows ->tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] -``` - -### 2.2 Parameter Introduction - - -| **Parameter** | **Definition** | **Required** | **Default** | -| --------- | ------------------------------------------------------------ | ------------ | ------------------------ | -| -h | Hostname | No | 127.0.0.1 | -| -p | Port | No | 6667 | -| -u | Username | No | root | -| -pw | Password | No | root | -| -s | Specify the data to be imported, here you can specify files or folders. If a folder is specified, all files with suffixes of csv or sql in the folder will be batch imported (In V1.3.2, the parameter is `-f`) | Yes | | -| -fd | Specify the directory for storing failed SQL files. If this parameter is not specified, failed files will be saved in the source data directory. Note: For unsupported SQL, illegal SQL, and failed SQL, they will be put into the failed directory under the failed file (default is the file name with `.failed` suffix) | No |The source filename with `.failed` suffix | -| -aligned | Specify whether to use the `aligned` interface, options are true or false. Note: This parameter is only effective when importing csv files. | No | false | -| -batch | Used to specify the number of data points per batch (minimum value is 1, maximum value is Integer.*MAX_VALUE*). If the program reports the error `org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`, you can appropriately reduce this parameter. | No | 100000 | -| -tp | Specify the time precision, options include `ms` (milliseconds), `ns` (nanoseconds), `us` (microseconds) | No | ms | -| -lpf | Specify the number of data lines written per failed file (In V1.3.2, the parameter is `-linesPerFailedFile`) | No | 10000 | -| -typeInfer | Used to specify type inference rules, such as . Note: Used to specify type inference rules. `srcTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `NaN`. `dstTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `text`. When `srcTsDataType` is `boolean`, `dstTsDataType` can only be `boolean` or `text`. When `srcTsDataType` is `NaN`, `dstTsDataType` can only be `float`, `double`, or `text`. When `srcTsDataType` is a numerical type, the precision of `dstTsDataType` needs to be higher than `srcTsDataType`. For example: `-typeInfer boolean=text,float=double` | No | | - - -### 2.3 Running Example - - -- Import the `dump0_0.sql` data in the current `data` directory to the local IoTDB database. - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.sql -# Windows ->tools/import-data.bat -s ./data/dump0_0.sql -``` - -- Import all data in the current `data` directory in an aligned manner to the local IoTDB database. - -```Bash +## 2. Data Import Tool +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell # Unix/OS X ->tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + # Windows ->tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] ``` -- Import the `dump0_0.csv` data in the current `data` directory to the local IoTDB database. - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ -# Windows ->tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# Error Example +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` -- Import the `dump0_0.csv` data in the current `data` directory in an aligned manner, batch import 100000 lines to the IoTDB database on the host with IP `192.168.100.1`, record failures in the current `failed` directory, with a maximum of 1000 lines per file. +### 2.3 SQL Format +#### 2.3.1 Command -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + # Windows ->tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] ``` +#### 2.3.2 SQL-Specific Parameters -## 3. load-tsfile Script +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | -- Supported formats: TsFile +#### 2.3.3 Examples -### 3.1 Command +```Shell +# Valid Example +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# Error Example +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist -```Bash -# Unix/OS X ->tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] -# Windows ->tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` +### 2.4 TsFile Format -### 3.2 Parameter Introduction - - -| **Parameter** | **Description** | **Required** | **Default** | -| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | -| -h | Hostname | No | root | -| -p | Port | No | root | -| -u | Username | No | 127.0.0.1 | -| -pw | Password | No | 6667 | -| -s | The local directory path of the script file (folder) to be loaded | Yes | | -| -os | none: Do not delete
mv: Move successful files to the target folder
cp: Hard link (copy) successful files to the target folder
delete: Delete | Yes | | -| -sd | When --on_success is mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_success is mv or cp, it is required to fill in Yes | ${EXEC_DIR}/success | -| -of | none: Skip
mv: Move failed files to the target folder
cp: Hard link (copy) failed files to the target folder
delete: Delete | Yes | | -| -fd | When --on_fail is specified as mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_fail is specified as mv or cp, it is required to fill in | ${EXEC_DIR}/fail | -| -tn | Maximum number of parallel threads | Yes | 8 | - - +#### 2.4.1 Command -### 3.3 Running Examples - - -```Bash +```Shell # Unix/OS X -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 - +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + # Windows -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# Error Example +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` - -## 4. TsFile Active Listening & Loading Feature - -The TsFile Active Listening & Loading Feature can actively monitor TsFile file changes in the specified target path (configured by the user) and automatically synchronize TsFile files from the target path to the specified reception path (configured by the user). Through this feature, IoTDB can automatically detect and load these files without the need for any additional manual loading operations. This automated process not only simplifies the user's operational steps but also reduces potential errors that may occur during the operation, effectively reducing the complexity for users during the usage process. - -![](/img/Data-import2.png) - - -### 4.1 Configuration Parameters - -You can enable the TsFile Active Listening & Loading Feature by finding the following parameters in the configuration file template `iotdb-system.properties.template` and adding them to the IoTDB configuration file `iotdb-system.properties`. The complete configuration is as follows: - - -| **Configuration Parameter** | **Description** | **Value Range** | **Required** | **Default Value** | **Loading Method** | -| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | -| load_active_listening_enable | Whether to enable the DataNode's active listening and loading of tsfile functionality (default is enabled). | Boolean: true，false | Optional | true | Hot Loading | -| load_active_listening_dirs | The directories to be listened to (automatically includes subdirectories of the directory), if there are multiple, separate with “,”. The default directory is ext/load/pending (supports hot loading). | String: one or more file directories | Optional | ext/load/pending | Hot Loading | -| load_active_listening_fail_dir | The directory to which files are transferred after the execution of loading tsfile files fails, only one directory can be configured. | String: one file directory | Optional | ext/load/failed | Hot Loading | -| load_active_listening_max_thread_num | The maximum number of threads to perform loading tsfile tasks simultaneously. The default value when the parameter is commented out is max(1, CPU core count / 2). When the user sets a value not in the range [1, CPU core count / 2], it will be set to the default value (1, CPU core count / 2). | Long: [1, Long.MAX_VALUE] | Optional | max(1, CPU core count / 2) | Effective after restart | -| load_active_listening_check_interval_seconds | Active listening polling interval in seconds. The function of actively listening to tsfile is achieved by polling the folder. This configuration specifies the time interval between two checks of load_active_listening_dirs, and the next check will be executed after load_active_listening_check_interval_seconds seconds of each check. When the user sets the polling interval to less than 1, it will be set to the default value of 5 seconds. | Long: [1, Long.MAX_VALUE] | Optional | 5 | Effective after restart | - - -### 4.2 Precautions - -1. If there is a mods file in the files to be loaded, the mods file should be moved to the listening directory first, and then the tsfile files should be moved, with the mods file and the corresponding tsfile file in the same directory. This prevents the loading of tsfile files without the corresponding mods files. -```SQL -FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) - // Move mods files - modsFiles = searchFiles(sourceDirectory, "*mods*") - IF modsFiles IS NOT EMPTY - FOR EACH file IN modsFiles - MOVE(file, listeningDirectory) - END FOR - END IF +## 3. TsFile Auto-Loading - // Move tsfile files - tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") - IF tsfileFiles IS NOT EMPTY - FOR EACH file IN tsfileFiles - MOVE(file, listeningDirectory) - END FOR - END IF -END FUNCTION +This feature enables IoTDB to automatically monitor a specified directory for new TsFiles and load them into the database without manual intervention. -FUNCTION searchFiles(directory, pattern) - matchedFiles = [] - FOR EACH file IN directory.files - IF file.name MATCHES pattern - APPEND file TO matchedFiles - END IF - END FOR - RETURN matchedFiles -END FUNCTION +![](/img/Data-import2.png) -FUNCTION MOVE(sourceFile, targetDirectory) - // Implement the logic of moving files from sourceFile to targetDirectory -END FUNCTION -``` +### 3.1 Configuration -2. Prohibit setting the receiver directory of Pipe, the data directory for storing data, etc., as the listening directory. +Add the following parameters to `iotdb-system.properties` (template: `iotdb-system.properties.template`): -3. Prohibit `load_active_listening_fail_dir` from having the same directory as `load_active_listening_dirs`, or each other's nesting. +| Parameter | Description | Value Range | Required | Default | Hot-Load? | +| ---------------------------------------------------- |---------------------------------------------------------------------------------------| --------------------------------- | ---------- | ----------------------------- | ----------------------- | +| `load_active_listening_enable` | Enable auto-loading. | `true`/`false` | Optional | `true` | Yes | +| `load_active_listening_dirs` | Directories to monitor (subdirectories included). Multiple paths separated by commas. | String | Optional | `ext/load/pending` | Yes | +| `load_active_listening_fail_dir` | Directory to store failed TsFiles. Only can set one. | String | Optional | `ext/load/failed` | Yes | +| `load_active_listening_max_thread_num` | Maximum Threads for TsFile Loading Tasks:The default value for this parameter, when commented out, is max(1, CPU cores / 2). If the value set by the user falls outside the range [1, CPU cores / 2], it will be reset to the default value of max(1, CPU cores / 2). | `1` to `Long.MAX_VALUE` | Optional | `max(1, CPU_CORES / 2)` | No (restart required) | +| `load_active_listening_check_interval_seconds` | Active Listening Polling Interval (in seconds):The active listening feature for TsFiles is implemented through polling the target directory. This configuration specifies the time interval between two consecutive checks of the `load_active_listening_dirs`. After each check, the next check will be performed after `load_active_listening_check_interval_seconds` seconds. If the polling interval set by the user is less than 1, it will be reset to the default value of 5 seconds. | `1` to `Long.MAX_VALUE` | Optional | `5` | No (restart required) | -4. Ensure that the `load_active_listening_dirs` directory has sufficient permissions. After the load is successful, the files will be deleted. If there is no delete permission, it will lead to repeated loading. +### 3.2 Notes +1. ​​**Mods Files**​: If TsFiles have associated `.mods` files, move `.mods` files to the monitored directory ​**before** their corresponding TsFiles. Ensure `.mods` and TsFiles are in the same directory. +2. ​​**Restricted Directories**​: Do NOT set Pipe receiver directories, data directories, or other system paths as monitored directories. +3. ​​**Directory Conflicts**​: Ensure `load_active_listening_fail_dir` does not overlap with `load_active_listening_dirs` or its subdirectories. +4. ​​**Permissions**​: The monitored directory must have write permissions. Files are deleted after successful loading; insufficient permissions may cause duplicate loading. diff --git a/src/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md b/src/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md new file mode 100644 index 000000000..645fa6bd7 --- /dev/null +++ b/src/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md @@ -0,0 +1,200 @@ +# Data Import + +## 1. Overview +IoTDB supports two methods for data import: +* Data Import Tool: Use the import-data.sh (Unix/OS X) or import-data.bat (Windows) script in the tools directory to manually import CSV, SQL, or TsFile (open-source time-series file format) data into IoTDB. +* TsFile Auto-Loading Feature + + + + + + + + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVimport-data.sh/batCan be used for single or batch import of CSV files into IoTDB
SQLCan be used for single or batch import of SQL files into IoTDB
TsFileCan be used for single or batch import of TsFile files into IoTDB
TsFile Auto-Loading FeatureCan automatically monitor a specified directory for newly generated TsFiles and load them into IoTDB.
+ +## 2. Data Import Tool +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# Error Example +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.3 SQL Format + +#### 2.3.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | + +#### 2.3.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# Error Example +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist + + +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# Error Example +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + + +## 3. TsFile Auto-Loading + +This feature enables IoTDB to automatically monitor a specified directory for new TsFiles and load them into the database without manual intervention. + +![](/img/Data-import2.png) + +### 3.1 Configuration + +Add the following parameters to `iotdb-system.properties` (template: `iotdb-system.properties.template`): + +| Parameter | Description | Value Range | Required | Default | Hot-Load? | +| ---------------------------------------------------- |---------------------------------------------------------------------------------------| --------------------------------- | ---------- | ----------------------------- | ----------------------- | +| `load_active_listening_enable` | Enable auto-loading. | `true`/`false` | Optional | `true` | Yes | +| `load_active_listening_dirs` | Directories to monitor (subdirectories included). Multiple paths separated by commas. | String | Optional | `ext/load/pending` | Yes | +| `load_active_listening_fail_dir` | Directory to store failed TsFiles. Only can set one. | String | Optional | `ext/load/failed` | Yes | +| `load_active_listening_max_thread_num` | Maximum Threads for TsFile Loading Tasks:The default value for this parameter, when commented out, is max(1, CPU cores / 2). If the value set by the user falls outside the range [1, CPU cores / 2], it will be reset to the default value of max(1, CPU cores / 2). | `1` to `Long.MAX_VALUE` | Optional | `max(1, CPU_CORES / 2)` | No (restart required) | +| `load_active_listening_check_interval_seconds` | Active Listening Polling Interval (in seconds):The active listening feature for TsFiles is implemented through polling the target directory. This configuration specifies the time interval between two consecutive checks of the `load_active_listening_dirs`. After each check, the next check will be performed after `load_active_listening_check_interval_seconds` seconds. If the polling interval set by the user is less than 1, it will be reset to the default value of 5 seconds. | `1` to `Long.MAX_VALUE` | Optional | `5` | No (restart required) | + +### 3.2 Notes + +1. ​​**Mods Files**​: If TsFiles have associated `.mods` files, move `.mods` files to the monitored directory ​**before** their corresponding TsFiles. Ensure `.mods` and TsFiles are in the same directory. +2. ​​**Restricted Directories**​: Do NOT set Pipe receiver directories, data directories, or other system paths as monitored directories. +3. ​​**Directory Conflicts**​: Ensure `load_active_listening_fail_dir` does not overlap with `load_active_listening_dirs` or its subdirectories. +4. ​​**Permissions**​: The monitored directory must have write permissions. Files are deleted after successful loading; insufficient permissions may cause duplicate loading. diff --git a/src/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md b/src/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md new file mode 100644 index 000000000..645fa6bd7 --- /dev/null +++ b/src/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md @@ -0,0 +1,200 @@ +# Data Import + +## 1. Overview +IoTDB supports two methods for data import: +* Data Import Tool: Use the import-data.sh (Unix/OS X) or import-data.bat (Windows) script in the tools directory to manually import CSV, SQL, or TsFile (open-source time-series file format) data into IoTDB. +* TsFile Auto-Loading Feature + + + + + + + + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVimport-data.sh/batCan be used for single or batch import of CSV files into IoTDB
SQLCan be used for single or batch import of SQL files into IoTDB
TsFileCan be used for single or batch import of TsFile files into IoTDB
TsFile Auto-Loading FeatureCan automatically monitor a specified directory for newly generated TsFiles and load them into IoTDB.
+ +## 2. Data Import Tool +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# Error Example +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.3 SQL Format + +#### 2.3.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | + +#### 2.3.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# Error Example +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist + + +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# Error Example +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + + +## 3. TsFile Auto-Loading + +This feature enables IoTDB to automatically monitor a specified directory for new TsFiles and load them into the database without manual intervention. + +![](/img/Data-import2.png) + +### 3.1 Configuration + +Add the following parameters to `iotdb-system.properties` (template: `iotdb-system.properties.template`): + +| Parameter | Description | Value Range | Required | Default | Hot-Load? | +| ---------------------------------------------------- |---------------------------------------------------------------------------------------| --------------------------------- | ---------- | ----------------------------- | ----------------------- | +| `load_active_listening_enable` | Enable auto-loading. | `true`/`false` | Optional | `true` | Yes | +| `load_active_listening_dirs` | Directories to monitor (subdirectories included). Multiple paths separated by commas. | String | Optional | `ext/load/pending` | Yes | +| `load_active_listening_fail_dir` | Directory to store failed TsFiles. Only can set one. | String | Optional | `ext/load/failed` | Yes | +| `load_active_listening_max_thread_num` | Maximum Threads for TsFile Loading Tasks:The default value for this parameter, when commented out, is max(1, CPU cores / 2). If the value set by the user falls outside the range [1, CPU cores / 2], it will be reset to the default value of max(1, CPU cores / 2). | `1` to `Long.MAX_VALUE` | Optional | `max(1, CPU_CORES / 2)` | No (restart required) | +| `load_active_listening_check_interval_seconds` | Active Listening Polling Interval (in seconds):The active listening feature for TsFiles is implemented through polling the target directory. This configuration specifies the time interval between two consecutive checks of the `load_active_listening_dirs`. After each check, the next check will be performed after `load_active_listening_check_interval_seconds` seconds. If the polling interval set by the user is less than 1, it will be reset to the default value of 5 seconds. | `1` to `Long.MAX_VALUE` | Optional | `5` | No (restart required) | + +### 3.2 Notes + +1. ​​**Mods Files**​: If TsFiles have associated `.mods` files, move `.mods` files to the monitored directory ​**before** their corresponding TsFiles. Ensure `.mods` and TsFiles are in the same directory. +2. ​​**Restricted Directories**​: Do NOT set Pipe receiver directories, data directories, or other system paths as monitored directories. +3. ​​**Directory Conflicts**​: Ensure `load_active_listening_fail_dir` does not overlap with `load_active_listening_dirs` or its subdirectories. +4. ​​**Permissions**​: The monitored directory must have write permissions. Files are deleted after successful loading; insufficient permissions may cause duplicate loading. diff --git a/src/UserGuide/latest/Tools-System/Data-Import-Tool.md b/src/UserGuide/latest/Tools-System/Data-Import-Tool.md index 75229ae09..3c71156b3 100644 --- a/src/UserGuide/latest/Tools-System/Data-Import-Tool.md +++ b/src/UserGuide/latest/Tools-System/Data-Import-Tool.md @@ -1,218 +1,200 @@ # Data Import -## 1. IoTDB Data Import - -IoTDB currently supports importing data in CSV, SQL, and TsFile (IoTDB's underlying open-time series file format) into the database. The specific functionalities are as follows: +## 1. Overview +IoTDB supports two methods for data import: +* Data Import Tool: Use the import-data.sh (Unix/OS X) or import-data.bat (Windows) script in the tools directory to manually import CSV, SQL, or TsFile (open-source time-series file format) data into IoTDB. +* TsFile Auto-Loading Feature - + - + - - + - - + - - + +
File Format
File Format IoTDB Tool Description
CSVimport-data.sh/batimport-data.sh/bat Can be used for single or batch import of CSV files into IoTDB
SQLimport-data.sh/batSQL Can be used for single or batch import of SQL files into IoTDB
TsFileload-tsfile.sh/batTsFile Can be used for single or batch import of TsFile files into IoTDB
TsFile Active Listening & Loading FeatureAccording to user configuration, it listens for changes in TsFile files in the specified path and loads newly added TsFile files into IoTDBTsFile Auto-Loading FeatureCan automatically monitor a specified directory for newly generated TsFiles and load them into IoTDB.
-## 2. import-data Scripts - -- Supported formats: CSV、SQL - -### 2.1 Command - -```Bash -# Unix/OS X ->tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] - -# Windows ->tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] -``` - -### 2.2 Parameter Introduction - - -| **Parameter** | **Definition** | **Required** | **Default** | -| --------- | ------------------------------------------------------------ | ------------ | ------------------------ | -| -h | Hostname | No | 127.0.0.1 | -| -p | Port | No | 6667 | -| -u | Username | No | root | -| -pw | Password | No | root | -| -s | Specify the data to be imported, here you can specify files or folders. If a folder is specified, all files with suffixes of csv or sql in the folder will be batch imported (In V1.3.2, the parameter is `-f`) | Yes | | -| -fd | Specify the directory for storing failed SQL files. If this parameter is not specified, failed files will be saved in the source data directory. Note: For unsupported SQL, illegal SQL, and failed SQL, they will be put into the failed directory under the failed file (default is the file name with `.failed` suffix) | No |The source filename with `.failed` suffix | -| -aligned | Specify whether to use the `aligned` interface, options are true or false. Note: This parameter is only effective when importing csv files. | No | false | -| -batch | Used to specify the number of data points per batch (minimum value is 1, maximum value is Integer.*MAX_VALUE*). If the program reports the error `org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`, you can appropriately reduce this parameter. | No | 100000 | -| -tp | Specify the time precision, options include `ms` (milliseconds), `ns` (nanoseconds), `us` (microseconds) | No | ms | -| -lpf | Specify the number of data lines written per failed file (In V1.3.2, the parameter is `-linesPerFailedFile`) | No | 10000 | -| -typeInfer | Used to specify type inference rules, such as . Note: Used to specify type inference rules. `srcTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `NaN`. `dstTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `text`. When `srcTsDataType` is `boolean`, `dstTsDataType` can only be `boolean` or `text`. When `srcTsDataType` is `NaN`, `dstTsDataType` can only be `float`, `double`, or `text`. When `srcTsDataType` is a numerical type, the precision of `dstTsDataType` needs to be higher than `srcTsDataType`. For example: `-typeInfer boolean=text,float=double` | No | | - - -### 2.3 Running Example - - -- Import the `dump0_0.sql` data in the current `data` directory to the local IoTDB database. - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.sql -# Windows ->tools/import-data.bat -s ./data/dump0_0.sql -``` - -- Import all data in the current `data` directory in an aligned manner to the local IoTDB database. - -```Bash +## 2. Data Import Tool +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell # Unix/OS X ->tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + # Windows ->tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] ``` -- Import the `dump0_0.csv` data in the current `data` directory to the local IoTDB database. - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ -# Windows ->tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# Error Example +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` -- Import the `dump0_0.csv` data in the current `data` directory in an aligned manner, batch import 100000 lines to the IoTDB database on the host with IP `192.168.100.1`, record failures in the current `failed` directory, with a maximum of 1000 lines per file. +### 2.3 SQL Format +#### 2.3.1 Command -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + # Windows ->tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] ``` +#### 2.3.2 SQL-Specific Parameters -## 3. load-tsfile Script +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | -- Supported formats: TsFile +#### 2.3.3 Examples -### 3.1 Command +```Shell +# Valid Example +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# Error Example +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist -```Bash -# Unix/OS X ->tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] -# Windows ->tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` +### 2.4 TsFile Format -### 3.2 Parameter Introduction - - -| **Parameter** | **Description** | **Required** | **Default** | -| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | -| -h | Hostname | No | root | -| -p | Port | No | root | -| -u | Username | No | 127.0.0.1 | -| -pw | Password | No | 6667 | -| -s | The local directory path of the script file (folder) to be loaded | Yes | | -| -os | none: Do not delete
mv: Move successful files to the target folder
cp: Hard link (copy) successful files to the target folder
delete: Delete | Yes | | -| -sd | When --on_success is mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_success is mv or cp, it is required to fill in Yes | ${EXEC_DIR}/success | -| -of | none: Skip
mv: Move failed files to the target folder
cp: Hard link (copy) failed files to the target folder
delete: Delete | Yes | | -| -fd | When --on_fail is specified as mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_fail is specified as mv or cp, it is required to fill in | ${EXEC_DIR}/fail | -| -tn | Maximum number of parallel threads | Yes | 8 | - - +#### 2.4.1 Command -### 3.3 Running Examples - - -```Bash +```Shell # Unix/OS X -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 - +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + # Windows -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# Error Example +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` - -## 4. TsFile Active Listening & Loading Feature - -The TsFile Active Listening & Loading Feature can actively monitor TsFile file changes in the specified target path (configured by the user) and automatically synchronize TsFile files from the target path to the specified reception path (configured by the user). Through this feature, IoTDB can automatically detect and load these files without the need for any additional manual loading operations. This automated process not only simplifies the user's operational steps but also reduces potential errors that may occur during the operation, effectively reducing the complexity for users during the usage process. - -![](/img/Data-import2.png) - - -### 4.1 Configuration Parameters - -You can enable the TsFile Active Listening & Loading Feature by finding the following parameters in the configuration file template `iotdb-system.properties.template` and adding them to the IoTDB configuration file `iotdb-system.properties`. The complete configuration is as follows: - - -| **Configuration Parameter** | **Description** | **Value Range** | **Required** | **Default Value** | **Loading Method** | -| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | -| load_active_listening_enable | Whether to enable the DataNode's active listening and loading of tsfile functionality (default is enabled). | Boolean: true，false | Optional | true | Hot Loading | -| load_active_listening_dirs | The directories to be listened to (automatically includes subdirectories of the directory), if there are multiple, separate with “,”. The default directory is ext/load/pending (supports hot loading). | String: one or more file directories | Optional | ext/load/pending | Hot Loading | -| load_active_listening_fail_dir | The directory to which files are transferred after the execution of loading tsfile files fails, only one directory can be configured. | String: one file directory | Optional | ext/load/failed | Hot Loading | -| load_active_listening_max_thread_num | The maximum number of threads to perform loading tsfile tasks simultaneously. The default value when the parameter is commented out is max(1, CPU core count / 2). When the user sets a value not in the range [1, CPU core count / 2], it will be set to the default value (1, CPU core count / 2). | Long: [1, Long.MAX_VALUE] | Optional | max(1, CPU core count / 2) | Effective after restart | -| load_active_listening_check_interval_seconds | Active listening polling interval in seconds. The function of actively listening to tsfile is achieved by polling the folder. This configuration specifies the time interval between two checks of load_active_listening_dirs, and the next check will be executed after load_active_listening_check_interval_seconds seconds of each check. When the user sets the polling interval to less than 1, it will be set to the default value of 5 seconds. | Long: [1, Long.MAX_VALUE] | Optional | 5 | Effective after restart | - - -### 4.2 Precautions - -1. If there is a mods file in the files to be loaded, the mods file should be moved to the listening directory first, and then the tsfile files should be moved, with the mods file and the corresponding tsfile file in the same directory. This prevents the loading of tsfile files without the corresponding mods files. -```SQL -FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) - // Move mods files - modsFiles = searchFiles(sourceDirectory, "*mods*") - IF modsFiles IS NOT EMPTY - FOR EACH file IN modsFiles - MOVE(file, listeningDirectory) - END FOR - END IF +## 3. TsFile Auto-Loading - // Move tsfile files - tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") - IF tsfileFiles IS NOT EMPTY - FOR EACH file IN tsfileFiles - MOVE(file, listeningDirectory) - END FOR - END IF -END FUNCTION +This feature enables IoTDB to automatically monitor a specified directory for new TsFiles and load them into the database without manual intervention. -FUNCTION searchFiles(directory, pattern) - matchedFiles = [] - FOR EACH file IN directory.files - IF file.name MATCHES pattern - APPEND file TO matchedFiles - END IF - END FOR - RETURN matchedFiles -END FUNCTION +![](/img/Data-import2.png) -FUNCTION MOVE(sourceFile, targetDirectory) - // Implement the logic of moving files from sourceFile to targetDirectory -END FUNCTION -``` +### 3.1 Configuration -2. Prohibit setting the receiver directory of Pipe, the data directory for storing data, etc., as the listening directory. +Add the following parameters to `iotdb-system.properties` (template: `iotdb-system.properties.template`): -3. Prohibit `load_active_listening_fail_dir` from having the same directory as `load_active_listening_dirs`, or each other's nesting. +| Parameter | Description | Value Range | Required | Default | Hot-Load? | +| ---------------------------------------------------- |---------------------------------------------------------------------------------------| --------------------------------- | ---------- | ----------------------------- | ----------------------- | +| `load_active_listening_enable` | Enable auto-loading. | `true`/`false` | Optional | `true` | Yes | +| `load_active_listening_dirs` | Directories to monitor (subdirectories included). Multiple paths separated by commas. | String | Optional | `ext/load/pending` | Yes | +| `load_active_listening_fail_dir` | Directory to store failed TsFiles. Only can set one. | String | Optional | `ext/load/failed` | Yes | +| `load_active_listening_max_thread_num` | Maximum Threads for TsFile Loading Tasks:The default value for this parameter, when commented out, is max(1, CPU cores / 2). If the value set by the user falls outside the range [1, CPU cores / 2], it will be reset to the default value of max(1, CPU cores / 2). | `1` to `Long.MAX_VALUE` | Optional | `max(1, CPU_CORES / 2)` | No (restart required) | +| `load_active_listening_check_interval_seconds` | Active Listening Polling Interval (in seconds):The active listening feature for TsFiles is implemented through polling the target directory. This configuration specifies the time interval between two consecutive checks of the `load_active_listening_dirs`. After each check, the next check will be performed after `load_active_listening_check_interval_seconds` seconds. If the polling interval set by the user is less than 1, it will be reset to the default value of 5 seconds. | `1` to `Long.MAX_VALUE` | Optional | `5` | No (restart required) | -4. Ensure that the `load_active_listening_dirs` directory has sufficient permissions. After the load is successful, the files will be deleted. If there is no delete permission, it will lead to repeated loading. +### 3.2 Notes +1. ​​**Mods Files**​: If TsFiles have associated `.mods` files, move `.mods` files to the monitored directory ​**before** their corresponding TsFiles. Ensure `.mods` and TsFiles are in the same directory. +2. ​​**Restricted Directories**​: Do NOT set Pipe receiver directories, data directories, or other system paths as monitored directories. +3. ​​**Directory Conflicts**​: Ensure `load_active_listening_fail_dir` does not overlap with `load_active_listening_dirs` or its subdirectories. +4. ​​**Permissions**​: The monitored directory must have write permissions. Files are deleted after successful loading; insufficient permissions may cause duplicate loading. diff --git a/src/zh/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md index 4c14acdf9..2bb4516cf 100644 --- a/src/zh/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md +++ b/src/zh/UserGuide/Master/Tree/Tools-System/Data-Import-Tool.md @@ -1,207 +1,209 @@ # 数据导入 -## 1. IoTDB 数据导入 +## 1. 功能概述 -IoTDB 目前支持将 CSV、SQL、及TsFile（IoTDB底层开放式时间序列文件格式）格式的数据导入数据库。具体功能如下： +IoTDB 支持两种方式进行数据导入 + +* 数据导入工具：tools 目录下的手动数据导入工具 `import-data.sh/bat`，可以将 CSV、SQL、及TsFile（开源时序文件格式）的数据导入 IoTDB。 +* TsFile 自动加载功能 - + - + - - - - + +
文件格式
文件格式 IoTDB工具 具体介绍
CSVimport-data.sh/batimport-data.sh/bat 可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQLimport-data.sh/bat 可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFileload-tsfile.sh/bat 可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
TsFile 主动监听&加载功能根据用户配置，监听指定路径下TsFile文件的变化，将新增加的TsFile文件加载入IoTDBTsFile 自动加载功能 可以监听指定路径下新产生的TsFile文件，并将其加载进IoTDB
-## 2. import-data 脚本 - -- 支持格式：CSV、SQL - -### 2.1 运行命令： - -```Bash -# Unix/OS X ->tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] - -# Windows ->tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] -``` +## 2. 数据导入工具 -### 2.2 参数介绍： +### 2.1 公共参数 -| **参数** | **定义** | **是否必填** | **默认** | -| ---------- | ------------------------------------------------------------ | ------------ | ------------------------- | -| -h | 数据库IP地址 | 否 | 127.0.0.1 | -| -p | 数据库端口 | 否 | 6667 | -| -u | 数据库连接用户名 | 否 | root | -| -pw | 数据库连接密码 | 否 | root | -| -s | 指定想要导入的数据，这里可以指定文件或者文件夹。如果指定的是文件夹，将会把文件夹中所有的后缀为 csv 或者 sql 的文件进行批量导入(V1.3.2版本参数是`-f`) | 是 | | -| -fd | 指定存放失败 SQL 文件的目录，如果未指定这个参数，失败的文件将会被保存到源数据的目录中。 说明：对于不支持的 SQL ，不合法的 SQL ，执行失败的 SQL 都会放到失败目录下的失败文件里（默认为 文件名.failed） | 否 | 源文件名加上`.failed`后缀 | -| -aligned | 指定是否使用`aligned`接口，选项为 true 或者 false 说明：这个参数只在导入文件为csv文件时生效 | 否 | false | -| -batch | 用于指定每一批插入的数据的点数（最小值为1，最大值为 Integer.*MAX_VALUE*）。如果程序报了`org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。 | 否 | 100000 | -| -tp | 指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒） | 否 | ms | -| -lpf | 指定每个导入失败文件写入数据的行数(V1.3.2版本参数是`-linesPerFailedFile`) | 否 | 10000 | -| -typeInfer | 用于指定类型推断规则，如。 说明：用于指定类型推断规则.`srcTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`NaN`.`dstTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`text`.当`srcTsDataType`为`boolean`, `dstTsDataType`只能为`boolean`或`text`.当`srcTsDataType`为`NaN`, `dstTsDataType`只能为`float`, `double`或`text`.当`srcTsDataType`为数值类型, `dstTsDataType`的精度需要高于`srcTsDataType`.例如:`-typeInfer boolean=text,float=double` | 否 | | +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为csv sql tsfile这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +| +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | +| -### 2.3 运行示例： +### 2.2 Csv 格式 -- 导入当前`data`目录下的`dump0_0.sql`数据到本机 IoTDB 数据库中。 +#### 2.2.1 运行命令 -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.sql +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + # Windows ->tools/import-data.bat -s ./data/dump0_0.sql +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] ``` -- 将当前`data`目录下的所有数据以对齐的方式导入到本机 IoTDB 数据库中。 - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true -# Windows ->tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` -- 导入当前`data`目录下的`dump0_0.csv`数据到本机 IoTDB 数据库中。 - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ -# Windows ->tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ -``` +### 2.3 Sql 格式 -- 将当前`data`目录下的`dump0_0.csv`数据以对齐的方式，一批导入100000条导入到`192.168.100.1`IP所在主机的 IoTDB 数据库中，失败的记录记在当前`failed`目录下，每个文件最多记1000条。 +#### 2.2.1 运行命令 -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + # Windows ->tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] ``` -## 3. load-tsfile 脚本 +#### 2.2.2 私有参数 -- 支持格式：TsFile +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | -### 3.1 运行命令 +#### 2.2.3 运行示例 -```Bash -# Unix/OS X ->tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist -# Windows ->tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] -``` -### 3.2 参数介绍 +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` -| **参数** | **定义** | **是否必填** | **默认** | -| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | -| -h | 主机名 | 否 | root | -| -p | 端口号 | 否 | root | -| -u | 用户名 | 否 | 127.0.0.1 | -| -pw | 密码 | 否 | 6667 | -| -s | 待加载的脚本文件(夹)的本地目录路径 | 是 | | -| -os | none：不删除
mv：移动成功的文件到目标文件夹
cp：硬连接（拷贝）成功的文件到目标文件夹
delete：删除 | 是 | | -| -sd | 当--on_succcess为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_succcess为mv或cp时需要填写是 | ${EXEC_DIR}/success | -| -of | none：跳过
mv：移动失败的文件到目标文件夹
cp：硬连接（拷贝）失败的文件到目标文件夹
delete：删除 | 是 | | -| -fd | 当--on_fail指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_fail指定为mv或cp时需要填写 | ${EXEC_DIR}/fail | -| -tn | 最大并行线程数 | 是 | 8 | +### 2.4 TsFile 格式 -### 3.3 运行示例： +#### 2.4.1 运行命令 -```Bash +```Shell # Unix/OS X -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 - +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + # Windows -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] ``` -## 4. TsFile 主动监听&加载功能 +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为mv或cp时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile非远程导入：-tp 指定tsfile文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定tsfile文件的时间精度 pipe自动校验时间戳精度是否一致 不一致返回pipe报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# 异常示例 +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +## 3. TsFile 自动加载功能 -TsFile 主动监听&加载功能能够主动监听指定目标路径（用户配置）下TsFile的文件变化，并将目标路径下的TsFile文件自动同步至指定接收路径（用户配置）。通过此功能，IoTDB 能自动检测并加载这些文件，无需手动执行任何额外的加载操作。这种自动化流程不仅简化了用户的操作步骤，还减少了操作过程中可能出现的错误，有效降低了用户在使用过程中的复杂性。 +本功能允许 IoTDB 主动监听指定目录下的新增 TsFile，并将 TsFile 自动加载至 IoTDB 中。通过此功能，IoTDB 能自动检测并加载 TsFile，无需手动执行任何额外的加载操作。 ![](/img/Data-import1.png) +### 3.1 配置参数 -### 4.1 配置参数 - -可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启TsFile 主动监听&加载功能。完整配置如下：： +可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启 TsFile 自动加载功能。完整配置如下： +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -------------------- | ------------------------ | -------------------- | +| load\_active\_listening\_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load\_active\_listening\_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 `ext/load/pending`（支持热装载） | String: 一个或多个文件目录 | 选填 | `ext/load/pending` | 热加载 | +| load\_active\_listening\_fail\_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | `ext/load/failed` | 热加载 | +| load\_active\_listening\_max\_thread\_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX\_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load\_active\_listening\_check\_interval\_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 `load_active_listening_dirs` 的时间间隔，每次检查完成 `load_active_listening_check_interval_seconds` 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX\_VALUE] | 选填 | 5 | 重启后生效 | -| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | -| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | -| load_active_listening_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | -| load_active_listening_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 ext/load/pending（支持热装载） | String: 一个或多个文件目录 | 选填 | ext/load/pending | 热加载 | -| load_active_listening_fail_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | ext/load/failed | 热加载 | -| load_active_listening_max_thread_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | -| load_active_listening_check_interval_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 load_active_listening_dirs 的时间间隔，每次检查完成 load_active_listening_check_interval_seconds 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX_VALUE] | 选填 | 5 | 重启后生效 | - -### 4.2 注意事项 +### 3.2 注意事项 1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 - -```SQL -FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) - // 移动 mods 文件 - modsFiles = searchFiles(sourceDirectory, "*mods*") - IF modsFiles IS NOT EMPTY - FOR EACH file IN modsFiles - MOVE(file, listeningDirectory) - END FOR - END IF - - // 移动 tsfile 文件 - tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") - IF tsfileFiles IS NOT EMPTY - FOR EACH file IN tsfileFiles - MOVE(file, listeningDirectory) - END FOR - END IF -END FUNCTION - -FUNCTION searchFiles(directory, pattern) - matchedFiles = [] - FOR EACH file IN directory.files - IF file.name MATCHES pattern - APPEND file TO matchedFiles - END IF - END FOR - RETURN matchedFiles -END FUNCTION - -FUNCTION MOVE(sourceFile, targetDirectory) - // 实现文件从 sourceFile 移动到 targetDirectory 的逻辑 -END FUNCTION -``` - 2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 - 3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 - 4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 - diff --git a/src/zh/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md b/src/zh/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md new file mode 100644 index 000000000..2bb4516cf --- /dev/null +++ b/src/zh/UserGuide/V1.3.x/Tools-System/Data-Import-Tool-1-3-4.md @@ -0,0 +1,209 @@ +# 数据导入 + +## 1. 功能概述 + +IoTDB 支持两种方式进行数据导入 + +* 数据导入工具：tools 目录下的手动数据导入工具 `import-data.sh/bat`，可以将 CSV、SQL、及TsFile（开源时序文件格式）的数据导入 IoTDB。 +* TsFile 自动加载功能 + + + + + + + + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVimport-data.sh/bat可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQL可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFile可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
TsFile 自动加载功能 可以监听指定路径下新产生的TsFile文件，并将其加载进IoTDB
+ +## 2. 数据导入工具 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为csv sql tsfile这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +| +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | +| + +### 2.2 Csv 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.3 Sql 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist + + +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` + +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为mv或cp时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile非远程导入：-tp 指定tsfile文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定tsfile文件的时间精度 pipe自动校验时间戳精度是否一致 不一致返回pipe报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# 异常示例 +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +## 3. TsFile 自动加载功能 + +本功能允许 IoTDB 主动监听指定目录下的新增 TsFile，并将 TsFile 自动加载至 IoTDB 中。通过此功能，IoTDB 能自动检测并加载 TsFile，无需手动执行任何额外的加载操作。 + +![](/img/Data-import1.png) + +### 3.1 配置参数 + +可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启 TsFile 自动加载功能。完整配置如下： + +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -------------------- | ------------------------ | -------------------- | +| load\_active\_listening\_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load\_active\_listening\_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 `ext/load/pending`（支持热装载） | String: 一个或多个文件目录 | 选填 | `ext/load/pending` | 热加载 | +| load\_active\_listening\_fail\_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | `ext/load/failed` | 热加载 | +| load\_active\_listening\_max\_thread\_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX\_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load\_active\_listening\_check\_interval\_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 `load_active_listening_dirs` 的时间间隔，每次检查完成 `load_active_listening_check_interval_seconds` 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX\_VALUE] | 选填 | 5 | 重启后生效 | + +### 3.2 注意事项 + +1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 +2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 +3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 +4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 diff --git a/src/zh/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md b/src/zh/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md new file mode 100644 index 000000000..2bb4516cf --- /dev/null +++ b/src/zh/UserGuide/dev-1.3/Tools-System/Data-Import-Tool-1-3-4.md @@ -0,0 +1,209 @@ +# 数据导入 + +## 1. 功能概述 + +IoTDB 支持两种方式进行数据导入 + +* 数据导入工具：tools 目录下的手动数据导入工具 `import-data.sh/bat`，可以将 CSV、SQL、及TsFile（开源时序文件格式）的数据导入 IoTDB。 +* TsFile 自动加载功能 + + + + + + + + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVimport-data.sh/bat可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQL可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFile可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
TsFile 自动加载功能 可以监听指定路径下新产生的TsFile文件，并将其加载进IoTDB
+ +## 2. 数据导入工具 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为csv sql tsfile这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +| +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | +| + +### 2.2 Csv 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.3 Sql 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist + + +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` + +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为mv或cp时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile非远程导入：-tp 指定tsfile文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定tsfile文件的时间精度 pipe自动校验时间戳精度是否一致 不一致返回pipe报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# 异常示例 +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +## 3. TsFile 自动加载功能 + +本功能允许 IoTDB 主动监听指定目录下的新增 TsFile，并将 TsFile 自动加载至 IoTDB 中。通过此功能，IoTDB 能自动检测并加载 TsFile，无需手动执行任何额外的加载操作。 + +![](/img/Data-import1.png) + +### 3.1 配置参数 + +可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启 TsFile 自动加载功能。完整配置如下： + +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -------------------- | ------------------------ | -------------------- | +| load\_active\_listening\_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load\_active\_listening\_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 `ext/load/pending`（支持热装载） | String: 一个或多个文件目录 | 选填 | `ext/load/pending` | 热加载 | +| load\_active\_listening\_fail\_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | `ext/load/failed` | 热加载 | +| load\_active\_listening\_max\_thread\_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX\_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load\_active\_listening\_check\_interval\_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 `load_active_listening_dirs` 的时间间隔，每次检查完成 `load_active_listening_check_interval_seconds` 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX\_VALUE] | 选填 | 5 | 重启后生效 | + +### 3.2 注意事项 + +1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 +2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 +3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 +4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 diff --git a/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md index 4c14acdf9..2bb4516cf 100644 --- a/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md +++ b/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md @@ -1,207 +1,209 @@ # 数据导入 -## 1. IoTDB 数据导入 +## 1. 功能概述 -IoTDB 目前支持将 CSV、SQL、及TsFile（IoTDB底层开放式时间序列文件格式）格式的数据导入数据库。具体功能如下： +IoTDB 支持两种方式进行数据导入 + +* 数据导入工具：tools 目录下的手动数据导入工具 `import-data.sh/bat`，可以将 CSV、SQL、及TsFile（开源时序文件格式）的数据导入 IoTDB。 +* TsFile 自动加载功能 - + - + - - - - + +
文件格式
文件格式 IoTDB工具 具体介绍
CSVimport-data.sh/batimport-data.sh/bat 可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQLimport-data.sh/bat 可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFileload-tsfile.sh/bat 可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
TsFile 主动监听&加载功能根据用户配置，监听指定路径下TsFile文件的变化，将新增加的TsFile文件加载入IoTDBTsFile 自动加载功能 可以监听指定路径下新产生的TsFile文件，并将其加载进IoTDB
-## 2. import-data 脚本 - -- 支持格式：CSV、SQL - -### 2.1 运行命令： - -```Bash -# Unix/OS X ->tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] - -# Windows ->tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] -``` +## 2. 数据导入工具 -### 2.2 参数介绍： +### 2.1 公共参数 -| **参数** | **定义** | **是否必填** | **默认** | -| ---------- | ------------------------------------------------------------ | ------------ | ------------------------- | -| -h | 数据库IP地址 | 否 | 127.0.0.1 | -| -p | 数据库端口 | 否 | 6667 | -| -u | 数据库连接用户名 | 否 | root | -| -pw | 数据库连接密码 | 否 | root | -| -s | 指定想要导入的数据，这里可以指定文件或者文件夹。如果指定的是文件夹，将会把文件夹中所有的后缀为 csv 或者 sql 的文件进行批量导入(V1.3.2版本参数是`-f`) | 是 | | -| -fd | 指定存放失败 SQL 文件的目录，如果未指定这个参数，失败的文件将会被保存到源数据的目录中。 说明：对于不支持的 SQL ，不合法的 SQL ，执行失败的 SQL 都会放到失败目录下的失败文件里（默认为 文件名.failed） | 否 | 源文件名加上`.failed`后缀 | -| -aligned | 指定是否使用`aligned`接口，选项为 true 或者 false 说明：这个参数只在导入文件为csv文件时生效 | 否 | false | -| -batch | 用于指定每一批插入的数据的点数（最小值为1，最大值为 Integer.*MAX_VALUE*）。如果程序报了`org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。 | 否 | 100000 | -| -tp | 指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒） | 否 | ms | -| -lpf | 指定每个导入失败文件写入数据的行数(V1.3.2版本参数是`-linesPerFailedFile`) | 否 | 10000 | -| -typeInfer | 用于指定类型推断规则，如。 说明：用于指定类型推断规则.`srcTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`NaN`.`dstTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`text`.当`srcTsDataType`为`boolean`, `dstTsDataType`只能为`boolean`或`text`.当`srcTsDataType`为`NaN`, `dstTsDataType`只能为`float`, `double`或`text`.当`srcTsDataType`为数值类型, `dstTsDataType`的精度需要高于`srcTsDataType`.例如:`-typeInfer boolean=text,float=double` | 否 | | +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为csv sql tsfile这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +| +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | +| -### 2.3 运行示例： +### 2.2 Csv 格式 -- 导入当前`data`目录下的`dump0_0.sql`数据到本机 IoTDB 数据库中。 +#### 2.2.1 运行命令 -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.sql +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + # Windows ->tools/import-data.bat -s ./data/dump0_0.sql +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] ``` -- 将当前`data`目录下的所有数据以对齐的方式导入到本机 IoTDB 数据库中。 - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true -# Windows ->tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 100 -aligned true -ti "BOOLEAN=text,INT=long,FLOAT=double" + -tp ms -tz +08:00 -batch 5000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft csv -s /non_path +error: Source file or directory /non_path does not exist + +> tools/import-data.sh -ft csv -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. ``` -- 导入当前`data`目录下的`dump0_0.csv`数据到本机 IoTDB 数据库中。 - -```Bash -# Unix/OS X ->tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ -# Windows ->tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ -``` +### 2.3 Sql 格式 -- 将当前`data`目录下的`dump0_0.csv`数据以对齐的方式，一批导入100000条导入到`192.168.100.1`IP所在主机的 IoTDB 数据库中，失败的记录记在当前`failed`目录下，每个文件最多记1000条。 +#### 2.2.1 运行命令 -```Bash +```Shell # Unix/OS X ->tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + # Windows ->tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] ``` -## 3. load-tsfile 脚本 +#### 2.2.2 私有参数 -- 支持格式：TsFile +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | -### 3.1 运行命令 +#### 2.2.3 运行示例 -```Bash -# Unix/OS X ->tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql + -fd /path/failure/dir -lpf 500 -tz +08:00 + -batch 100000 -tn 4 + +# 异常示例 +> tools/import-data.sh -ft sql -s /path/sql -fd /non_path +error: Source file or directory /path/sql does not exist -# Windows ->tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] -``` -### 3.2 参数介绍 +> tools/import-data.sh -ft sql -s /path/sql -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` -| **参数** | **定义** | **是否必填** | **默认** | -| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | -| -h | 主机名 | 否 | root | -| -p | 端口号 | 否 | root | -| -u | 用户名 | 否 | 127.0.0.1 | -| -pw | 密码 | 否 | 6667 | -| -s | 待加载的脚本文件(夹)的本地目录路径 | 是 | | -| -os | none：不删除
mv：移动成功的文件到目标文件夹
cp：硬连接（拷贝）成功的文件到目标文件夹
delete：删除 | 是 | | -| -sd | 当--on_succcess为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_succcess为mv或cp时需要填写是 | ${EXEC_DIR}/success | -| -of | none：跳过
mv：移动失败的文件到目标文件夹
cp：硬连接（拷贝）失败的文件到目标文件夹
delete：删除 | 是 | | -| -fd | 当--on_fail指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_fail指定为mv或cp时需要填写 | ${EXEC_DIR}/fail | -| -tn | 最大并行线程数 | 是 | 8 | +### 2.4 TsFile 格式 -### 3.3 运行示例： +#### 2.4.1 运行命令 -```Bash +```Shell # Unix/OS X -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 -> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 - +> tools/import-data.sh -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + # Windows -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 -> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools\import-data.bat -ft [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] ``` -## 4. TsFile 主动监听&加载功能 +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为mv或cp时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile非远程导入：-tp 指定tsfile文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定tsfile文件的时间精度 pipe自动校验时间戳精度是否一致 不一致返回pipe报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -h 127.0.0.1 -p 6667 -u root -pw root + -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir + -tn 8 -tz +08:00 -tp ms + +# 异常示例 +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -fd /path/failure/dir -tn 8 +error: Missing option --success_dir (or -sd) when --on_success is 'mv' or 'cp' + +> tools/import-data.sh -ft tsfile -s /path/sql -os mv -of cp + -sd /path/success/dir -fd /path/failure/dir -tn 0 +error: Invalid thread number '0'. Please set a positive integer. +``` +## 3. TsFile 自动加载功能 -TsFile 主动监听&加载功能能够主动监听指定目标路径（用户配置）下TsFile的文件变化，并将目标路径下的TsFile文件自动同步至指定接收路径（用户配置）。通过此功能，IoTDB 能自动检测并加载这些文件，无需手动执行任何额外的加载操作。这种自动化流程不仅简化了用户的操作步骤，还减少了操作过程中可能出现的错误，有效降低了用户在使用过程中的复杂性。 +本功能允许 IoTDB 主动监听指定目录下的新增 TsFile，并将 TsFile 自动加载至 IoTDB 中。通过此功能，IoTDB 能自动检测并加载 TsFile，无需手动执行任何额外的加载操作。 ![](/img/Data-import1.png) +### 3.1 配置参数 -### 4.1 配置参数 - -可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启TsFile 主动监听&加载功能。完整配置如下：： +可通过从配置文件模版 `iotdb-system.properties.template` 中找到下列参数，添加到 IoTDB 配置文件 `iotdb-system.properties` 中开启 TsFile 自动加载功能。完整配置如下： +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -------------------- | ------------------------ | -------------------- | +| load\_active\_listening\_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load\_active\_listening\_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 `ext/load/pending`（支持热装载） | String: 一个或多个文件目录 | 选填 | `ext/load/pending` | 热加载 | +| load\_active\_listening\_fail\_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | `ext/load/failed` | 热加载 | +| load\_active\_listening\_max\_thread\_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX\_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load\_active\_listening\_check\_interval\_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 `load_active_listening_dirs` 的时间间隔，每次检查完成 `load_active_listening_check_interval_seconds` 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX\_VALUE] | 选填 | 5 | 重启后生效 | -| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | -| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | -| load_active_listening_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | -| load_active_listening_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 ext/load/pending（支持热装载） | String: 一个或多个文件目录 | 选填 | ext/load/pending | 热加载 | -| load_active_listening_fail_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | ext/load/failed | 热加载 | -| load_active_listening_max_thread_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | -| load_active_listening_check_interval_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 load_active_listening_dirs 的时间间隔，每次检查完成 load_active_listening_check_interval_seconds 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX_VALUE] | 选填 | 5 | 重启后生效 | - -### 4.2 注意事项 +### 3.2 注意事项 1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 - -```SQL -FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) - // 移动 mods 文件 - modsFiles = searchFiles(sourceDirectory, "*mods*") - IF modsFiles IS NOT EMPTY - FOR EACH file IN modsFiles - MOVE(file, listeningDirectory) - END FOR - END IF - - // 移动 tsfile 文件 - tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") - IF tsfileFiles IS NOT EMPTY - FOR EACH file IN tsfileFiles - MOVE(file, listeningDirectory) - END FOR - END IF -END FUNCTION - -FUNCTION searchFiles(directory, pattern) - matchedFiles = [] - FOR EACH file IN directory.files - IF file.name MATCHES pattern - APPEND file TO matchedFiles - END IF - END FOR - RETURN matchedFiles -END FUNCTION - -FUNCTION MOVE(sourceFile, targetDirectory) - // 实现文件从 sourceFile 移动到 targetDirectory 的逻辑 -END FUNCTION -``` - 2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 - 3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 - 4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 -