Parallelize the downloads and insertions to database

.github/workflows/test.yml

@@ -11,22 +11,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
           allow-prereleases: true
 
       - name: Install dependencies
         run: |
-          python -m pip install -U pip
-          python -m pip install -U pytest
+          python -m pip install -U pip pytest
           python -m pip install -r requirements.txt
 
       - name: Test

.gitignore

@@ -1,5 +1,7 @@
 # Uncompressed sqlite3 database
 pypi.db
+pypi.db-shm
+pypi.db-wal
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,6 +1,6 @@
-FROM python:3.9-slim
+FROM python:3.14-slim
 
 WORKDIR /pypi-data
 COPY . /pypi-data
 RUN python -m pip install -r requirements.txt
-CMD python main.py
+CMD ["python", "main.py"]

README.md

@@ -107,13 +107,13 @@ CREATE TABLE classifiers (
 
 ### Download data
 
-Downloads are grabbed from https://github.com/hugovk/top-pypi-packages but only available for the top 5,000 packages.
+Downloads are grabbed from <https://github.com/hugovk/top-pypi-packages> but only available for the top 15,000 packages.
 
 ## Running locally
 
-```
-$ docker build -t pypi-data .
-$ docker run --rm pypi-data
+```shell
+docker build -t pypi-data .
+docker run --rm pypi-data
 ```
 
 ## License