Single machine download script and downloaded files check

data_prep/arxiv/run_download.py

@@ -51,9 +51,16 @@ def run(self, input_file: str, tgt_dir: pathlib.Path, max_files=-1):
                 break
 
     def __download_file(self, key, tgt_dir: pathlib.Path):
+        filename = pathlib.Path(tgt_dir, key)
         print('\nDownloading s3://arxiv/{} t'
-              'o {}...'.format(key, pathlib.Path(tgt_dir, key)))
-
+              'o {}...'.format(key, filename))
+
+        if filename.exists():
+            print(f'File {filename} already exists, skipping download...')
+            return
+
+        print('\nDownloading s3://arxiv/{} to {}...'.format(key, filename))
+
         try:
             self.s3resource.meta.client.download_file(
                 Bucket='arxiv',

data_prep/arxiv/scripts/arxiv-single-download.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# for download partitions
+# for single machine download
+
+set -e
+
+# Depend on your CPU
+WORKERS=8
+
+export DATA_DIR="./data/arxiv"
+
+# setup partitions
+python run_download.py --aws_config aws_config.ini --workers $WORKERS --target_dir $DATA_DIR --setup
+
+# Function to process a file
+process_file() {
+    INPUT_FILE="$1"
+    echo "Processing input file is ${INPUT_FILE}"
+    python run_download.py --aws_config aws_config.ini --target_dir $DATA_DIR --input $INPUT_FILE
+}
+
+# Export the function to be used by xargs
+export -f process_file
+
+ls ${DATA_DIR}/partitions/*.txt | xargs -I {} -P ${WORKERS} -n 1 bash -c 'process_file "$@"' _ {}
+