Implement Graph Database in SQLITE

.github/workflows/testing-coverage.yml

@@ -17,10 +17,15 @@ jobs:
     if: ${{ !(contains(github.event.pull_request.labels.*.name, 'WIP (no-ci)')) && !(contains(github.event.pull_request.labels.*.name, 'WIP (lint-only)')) }}
     name: BEE Integration Test
     strategy:
+      fail-fast: false
       matrix:
         bee_worker: [Slurmrestd, SlurmCommands, Flux]
+        # TODO get neo4j test to work again
+        # gdb_backend: [neo4j, sqlite3]
+        gdb_backend: [sqlite3]
     env:
       BEE_WORKER: ${{ matrix.bee_worker }}
+      GDB_BACKEND: ${{ matrix.gdb_backend }}
     # Note: Needs to run on 22.04 or later since slurmrestd doesn't seem to be
     # available on 20.04
     runs-on: ubuntu-22.04
@@ -37,13 +42,20 @@ jobs:
         run: |
           . ./ci/env.sh
           ./ci/integration_test.sh
-          mv .coverage.integration ".coverage.${{ matrix.bee_worker }}"
+          mv .coverage.integration ".coverage.${{ matrix.bee_worker }}.${{ matrix.gdb_backend }}"
+      - name: Upload bee error logs
+        if: failure()    # only when the job fails
+        uses: actions/upload-artifact@v4
+        with:
+          name: bee-error-logs-${{ matrix.bee_worker }}-${{ matrix.gdb_backend }}.log
+          path: bee-error-*.log
+          if-no-files-found: ignore
       - name: Upload coverage artifact
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.bee_worker }}
+          name: coverage-${{ matrix.bee_worker }}-${{ matrix.gdb_backend }}
           include-hidden-files: true
-          path: ".coverage.${{ matrix.bee_worker }}"
+          path: ".coverage.${{ matrix.bee_worker }}.${{ matrix.gdb_backend }}"
 
   unit-test:
     if: ${{ !(contains(github.event.pull_request.labels.*.name, 'WIP (no-ci)')) && !(contains(github.event.pull_request.labels.*.name, 'WIP (lint-only)')) }}
@@ -93,22 +105,40 @@ jobs:
           name: coverage-unit-tests
           path: ./
 
-      - name: Get Slurmrestd coverage
+      # - name: Get Slurmrestd+neo4j coverage
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: coverage-Slurmrestd-neo4j
+      #     path: ./
+
+      - name: Get Slurmrestd+sqlite3 coverage
         uses: actions/download-artifact@v4
         with:
-          name: coverage-Slurmrestd
+          name: coverage-Slurmrestd-sqlite3
           path: ./
-
-      - name: Get SlurmCommands coverage
+
+      # - name: Get SlurmCommands+neo4j coverage
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: coverage-SlurmCommands-neo4j
+      #     path: ./
+
+      - name: Get SlurmCommands+sqlite3 coverage
         uses: actions/download-artifact@v4
         with:
-          name: coverage-SlurmCommands
+          name: coverage-SlurmCommands-sqlite3
           path: ./
-
-      - name: Get Flux coverage
+
+      # - name: Get Flux+neo4j coverage
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: coverage-Flux-neo4j
+      #     path: ./
+
+      - name: Get Flux+sqlite3 coverage
         uses: actions/download-artifact@v4
         with:
-          name: coverage-Flux
+          name: coverage-Flux-sqlite3
           path: ./
 
       - name: Combine coverage

beeflow/client/core.py

@@ -202,6 +202,11 @@ def need_slurmrestd():
             and not bc.get('slurm', 'use_commands'))
 
 
+def need_neo4j():
+    """Check if neo4j is needed."""
+    return bc.get('graphdb', 'type').lower() == 'neo4j'
+
+
 def init_components(remote=False):
     """Initialize the components and component manager."""
     mgr = ComponentManager()
@@ -255,14 +260,15 @@ def celery():
     # container_path = paths.redis_container()
     # If it exists, we assume that it actually has a valid container
 
-    if not container_manager.check_container_dir('neo4j'):
-        print('Unpacking Neo4j image...')
-        container_manager.create_image('neo4j')
+    if need_neo4j():
+        if not container_manager.check_container_dir('neo4j'):
+            print('Unpacking Neo4j image...')
+            container_manager.create_image('neo4j')
 
-    @mgr.component('neo4j-database', ('wf_manager',))
-    def start_neo4j():
-        """Start the neo4j graph database."""
-        return neo4j_manager.start()
+        @mgr.component('neo4j-database', ('wf_manager',))
+        def start_neo4j():
+            """Start the neo4j graph database."""
+            return neo4j_manager.start()
 
     @mgr.component('redis', ())
     def start_redis():
@@ -673,14 +679,17 @@ def pull_deps(outdir: str = typer.Option('.', '--outdir', '-o',
                                          help='directory to store containers in')):
     """Pull required BEE containers and store in outdir."""
     load_check_charliecloud()
-    neo4j_path = os.path.join(os.path.realpath(outdir), 'neo4j.tar.gz')
-    neo4j_dockerfile = str(Path(REPO_PATH, "beeflow/data/dockerfiles/Dockerfile.neo4j"))
-    build_to_tar('neo4j_image', neo4j_dockerfile, neo4j_path)
+    if need_neo4j():
+        neo4j_path = os.path.join(os.path.realpath(outdir), 'neo4j.tar.gz')
+        neo4j_dockerfile = str(Path(REPO_PATH, "beeflow/data/dockerfiles/Dockerfile.neo4j"))
+        build_to_tar('neo4j_image', neo4j_dockerfile, neo4j_path)
     redis_path = os.path.join(os.path.realpath(outdir), 'redis.tar.gz')
     pull_to_tar('redis', redis_path)
 
-    AlterConfig(changes={'DEFAULT': {'neo4j_image': neo4j_path,
-                                     'redis_image': redis_path}}).save()
+    if need_neo4j():
+        AlterConfig(changes={'DEFAULT': {'neo4j_image': neo4j_path}}).save()
+
+    AlterConfig(changes={'DEFAULT': {'redis_image': redis_path}}).save()
 
     dep_dir = container_manager.get_dep_dir()
     if os.path.isdir(dep_dir):

beeflow/common/config_driver.py

@@ -397,6 +397,8 @@ def validate_chrun_opts(opts):
                  info='extra Charliecloud setup to put in a job script')
 # Graph Database
 VALIDATOR.section('graphdb', info='Main graph database configuration section.')
+VALIDATOR.option('graphdb', 'type', default='sqlite3', choices=('neo4j', 'sqlite3'),
+                 info='type of graph database to use', prompt=False)
 VALIDATOR.option('graphdb', 'hostname', default='localhost', prompt=False,
                  info='hostname of database')
 

beeflow/common/db/bdb.py

@@ -15,9 +15,10 @@ def create_connection(db_file):
     conn = None
     try:
         conn = sqlite3.connect(db_file)
+        conn.execute("PRAGMA foreign_keys = ON;")
         return conn
     except Error as error:
-        print(error)
+        print("Error connecting to database: ", error)
     return conn
 
 
@@ -42,9 +43,24 @@ def run(db_file, stmt, params=None):
                 cursor.execute(stmt)
             conn.commit()
         except Error as error:
+            print("Error running: ", stmt)
             print(error)
 
 
+
+def runscript(db_file, script):
+    """Run the sql script on the database. Doesn't return anything."""
+    with create_connection(db_file) as conn:
+        try:
+            cursor = conn.cursor()
+            cursor.executescript(script)
+            conn.commit()
+        except Error as error:
+            print("Error running script")
+            print(error)
+
+
+
 def getone(db_file, stmt, params=None):
     """Run the sql statement on the database and return the result."""
     with create_connection(db_file) as conn:
@@ -55,7 +71,9 @@ def getone(db_file, stmt, params=None):
             else:
                 cursor.execute(stmt)
             result = cursor.fetchone()
-        except Error:
+        except Error as error:
+            print("Error fetching one: ", stmt)
+            print(error)
             result = None
         return result
 
@@ -70,7 +88,9 @@ def getall(db_file, stmt, params=None):
             else:
                 cursor.execute(stmt)
             result = cursor.fetchall()
-        except Error:
+        except Error as error:
+            print("Error fetching all: ", stmt)
+            print(error)
             result = None
     return result