Add schema discovery and more testing

README.md

@@ -71,7 +71,8 @@ $ ../index.js -c current_schema.json -p previous_schema.json -q -d diff_dot.gv
 
   -q, --quiet              do not output svg to stdout
 
-  -s, --schema=schema      schema(s) to graph and optionally diff, use multiple times to include more than one schema
+  -s, --schema=schema      schema(s) to graph and optionally diff, use multiple times to include more than one schema.
+                           if omitted schemas will be discovered and all schemas found will be included in output
 
   --help                   show CLI help
 

dbFactory.js

@@ -16,7 +16,7 @@ const getSchemaProcessor = (connectionString) => {
   }
 }
 
-exports.generateErd = (current, schema) => {
+exports.generateErd = (current, schema, setSchema) => {
   const schemaProcessor = getSchemaProcessor(current)
-  return schemaProcessor.generateErd(current, schema)
+  return schemaProcessor.generateErd(current, schema, setSchema)
 }

index.js

@@ -180,11 +180,12 @@ class ERD extends Command {
   static description = `generate Entity Relationship Diagram`
   async run() {
     const {flags} = this.parse(ERD)
-    const schema = flags.schema
+    var schema = flags.schema
     let currentErd
     let previousErd
+    const setSchema = (s) => schema = s 
     try {
-      currentErd = await getErd(flags.current, schema)
+      currentErd = await getErd(flags.current, schema, setSchema)
     } catch (err) {
       this.error('uh oh! error loading current schema', {exit: 3})
     }

mysql.js

@@ -5,12 +5,15 @@ const createConnection = async (connectionString) => {
   try {
     connection = await mysql.createConnection(connectionString)
   } catch (err) {
-    console.log('uh oh! error connecting to mysql url')
     throw err
   } 
   return connection
 }
 
+const getAllSchemas = () => {
+  return `SELECT SCHEMA_NAME AS name FROM information_schema.SCHEMATA WHERE SCHEMA_NAME NOT IN ('mysql','information_schema','performance_schema', 'sys');`
+}
+
 const getTableSchemas = (schema) => {
   return schema.map(s => `c.TABLE_SCHEMA = '${s}'`).join(' OR ')
 }
@@ -66,18 +69,30 @@ const getRoutineQuery = (sprocSchemas) => {
 }
 
 exports.MySqlProcessor = {
-  generateErd: async (connectionString, schema) => {
+  generateErd: async (connectionString, schema, setSchema) => {
     const connection = await createConnection(connectionString)
-
-    if (schema.length) {
+    let tableSchemas
+    let sprocSchemas
+    let s
+    if (schema && schema?.length) {
       tableSchemas = getTableSchemas(schema)
       sprocSchemas = getRoutineSchemas(schema)
+    } else {
+      const [schemas] = await connection.query(getAllSchemas())
+      s = schemas.map(i => i.name)
+      if (setSchema instanceof Function)
+        setSchema(s)
+      tableSchemas = getTableSchemas(s)
+      sprocSchemas = getRoutineSchemas(s)
     }
 
     const tablesQuery = getTableQuery(tableSchemas)
     const sprocQuery = getRoutineQuery(sprocSchemas)
     const [tableData] = await connection.query(tablesQuery)
     const [sprocData] = await connection.query(sprocQuery)
+    if (tableData.length == 0 && sprocData.length == 0) {
+      throw new Error ('no data found for schemas: ' + schema || s)
+    }
     const tables = {}
     const sprocs = {}
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "erdiff",
-  "version": "0.9.4",
+  "version": "0.9.5",
   "description": "Database Entity Relation Diagramming tool with diffing support for diagrams and stored procedures",
   "main": "index.js",
   "license": "MIT",

postgres.js

@@ -10,8 +10,12 @@ const createConnection = async (connectionString) => {
   return client
 }
 
+const getAllSchemas = () => {
+  return `SELECT SCHEMA_NAME AS name FROM information_schema.SCHEMATA WHERE SCHEMA_NAME NOT IN ('pg_toast','information_schema','pg_catalog');`
+}
+
 const getTableSchemas = (schema) => {
-  return typeof schema == 'array' ? schema.map(s => `c.table_schema = '${s}'`).join(' OR ') : `c.table_schema = 'public'`
+  return schema ? schema.map(s => `c.table_schema = '${s}'`).join(' OR ') : `c.table_schema = 'public'`
 }
 
 const getRoutineSchemas = (schema) => {
@@ -60,8 +64,6 @@ SELECT
     fk.foreign_column_name as ref_column
   FROM information_schema.columns AS c
     LEFT JOIN information_schema.tables AS t ON (c.table_schema=t.table_schema AND c.table_name = t.table_name)
-
-    --LEFT JOIN information_schema.table_constraints AS uk ON (c.table_schema=uk.table_schema AND c.table_name = uk.table_name AND uk.constraint_type = 'UNIQUE')
     LEFT JOIN information_schema.table_constraints AS pk ON (c.table_schema=pk.table_schema AND c.table_name = pk.table_name AND pk.constraint_type = 'PRIMARY KEY')
     LEFT JOIN fk ON (c.table_schema=fk.table_schema AND c.table_name = fk.table_name AND c.column_name = fk.column_name)
   WHERE
@@ -90,17 +92,32 @@ SELECT
 
 
 exports.PostgresProcessor = {
-  generateErd: async (connectionString, schemas) => {
+  generateErd: async (connectionString, schemas, setSchema) => {
     const connection = await createConnection(connectionString)
-    tableSchemas = getTableSchemas(schemas)
-    sprocSchemas = getRoutineSchemas(schemas)
+    let tableSchemas
+    let sprocSchemas
+    let s
+    if (schemas && schemas?.length > 0) {
+      tableSchemas = getTableSchemas(schemas)
+      sprocSchemas = getRoutineSchemas(schemas)
+    } else {
+      const schema = await connection.query(getAllSchemas())
+      s = schema.rows.map(i => i.name)
+      if (setSchema instanceof Function)
+        setSchema(s)
+      tableSchemas = getTableSchemas(s)
+      sprocSchemas = getRoutineSchemas(s)
+    }
 
     const tablesQuery = getTableQuery(tableSchemas)
     const sprocQuery = getRoutineQuery(sprocSchemas)
     const {rows: tableData} = await connection.query(tablesQuery)
     const {rows: sprocData} = await connection.query(sprocQuery)
     const tables = {}
     const sprocs = {}
+    if (tableData.length == 0 && sprocData.length == 0) {
+      throw new Error ('no data found for schemas: ' + schemas || s)
+    }
     tableData.forEach(row => {
       const name = `${row.schema}.${row.table}`
       if (!tables.hasOwnProperty(name)) {

tests/mysql/test.sh

@@ -2,8 +2,11 @@
 
 docker-compose up -d
 
-sleep 30
+sleep 60
 
 node ../../index.js -s database -c 'mysql://root:rootpassword@127.0.0.1:3306/database' -p 'mysql://root:rootpassword@127.0.0.1:3307/database' > output.html 
+node ../../index.js -c 'mysql://root:rootpassword@127.0.0.1:3306/database' -p 'mysql://root:rootpassword@127.0.0.1:3307/database' > schemaless.html 
 
 docker-compose down
+
+diff output.html schemaless.html > /dev/null && rm output.html schemaless.html

tests/postgres/current/tables.sql

@@ -0,0 +1,48 @@
+CREATE TABLE users (
+  userid SERIAL PRIMARY KEY, 
+  name TEXT, 
+  email TEXT,
+  password TEXT, 
+  created timestamp default current_timestamp,
+  updated timestamp default current_timestamp,
+  deleted timestamp default current_timestamp
+);
+
+CREATE TABLE posts (
+  postid SERIAL PRIMARY KEY, 
+  title TEXT, 
+  body TEXT,
+  userid INT, 
+  inreplyto INT, 
+  created timestamp default current_timestamp,
+  updated timestamp default current_timestamp,
+  deleted timestamp default current_timestamp,
+  FOREIGN KEY(userid) REFERENCES users(userid),
+  FOREIGN KEY(inreplyto) REFERENCES posts(postid)
+);
+
+CREATE TABLE likes (
+  postid INT,
+  userid INT,
+  created timestamp default current_timestamp,
+  deleted timestamp default current_timestamp,
+  PRIMARY KEY(postid,userid),
+  FOREIGN KEY(postid) REFERENCES posts(postid),
+  FOREIGN KEY(userid) REFERENCES users(userid)
+);
+
+
+CREATE PROCEDURE prune_accounts() 
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  UPDATE users SET deleted=NOW() WHERE updated < CURRENT_DATE - '6 months'::interval;
+END;$$ ;
+
+CREATE PROCEDURE prune_posts() 
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  -- this is a bad implementation, need to fix later
+  UPDATE posts SET deleted=NOW() WHERE deleted IS NULL AND userid in (SELECT userid FROM users WHERE deleted > CURRENT_DATE - '1 months'::interval);
+END;$$;

tests/postgres/current/tables2.sql

@@ -0,0 +1,84 @@
+ALTER TABLE users ADD COLUMN nickname TEXT;
+CREATE UNIQUE INDEX unique_emails ON users(email);
+
+CREATE TABLE new_likes (
+  postid INT,
+  userid INT,
+  created timestamp default current_timestamp,
+  PRIMARY KEY(postid,userid),
+  FOREIGN KEY(postid) REFERENCES posts(postid),
+  FOREIGN KEY(userid) REFERENCES users(userid)
+);
+
+INSERT INTO new_likes SELECT postid, userid, created FROM likes;
+
+DROP TABLE likes;
+
+ALTER TABLE new_likes RENAME TO likes;
+
+CREATE VIEW stats AS
+WITH lc AS (
+  SELECT
+    postid,
+    count(userid) AS count
+  FROM likes GROUP BY postid
+),rc AS (
+  SELECT
+    posts.postid,
+    posts.userid,
+    count(replies.postid) AS count
+  FROM posts AS posts
+    LEFT JOIN posts AS replies ON (posts.postid = replies.inreplyto)
+  GROUP BY posts.postid, posts.userid)
+SELECT
+    posts.userid,
+    posts.title,
+    lc.count AS likecount,
+    rc.count AS replycount,
+    rc.count/lc.count AS ratio 
+  FROM posts AS posts
+    LEFT JOIN lc USING(postid)
+    LEFT JOIN rc USING (postid, userid)
+  WHERE
+    posts.inreplyto IS NULL
+;
+
+DROP PROCEDURE IF EXISTS prune_posts;
+CREATE PROCEDURE prune_posts() 
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  UPDATE posts 
+    SET deleted=NOW() 
+    WHERE postid in (
+      SELECT posts.postid AS postid
+        FROM posts
+          JOIN users ON (posts.userid=users.userid)
+        WHERE 
+          posts.deleted IS NULL AND
+          users.deleted > CURRENT_DATE - '1 months'::interval
+    );
+END;$$;
+
+CREATE PROCEDURE top_likers(IN foremail text)
+LANGUAGE plpgsql
+AS $$
+BEGIN
+SELECT
+    count(1) AS num,
+    likers.name
+  FROM users AS source
+    JOIN posts AS posts ON (source.userid=posts.userid)
+    JOIN likes AS liked ON (posts.postid=liked.postid)
+    JOIN users AS likers ON (liked.userid=likers.userid)
+  WHERE
+    liked.deleted IS NULL
+    AND source.email = foremail
+
+  GROUP BY likers.name
+  HAVING num > 1 
+  ORDER BY 1
+  LIMIT 5
+;
+END;$$;
+

tests/postgres/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.7'
+services:
+  previous:
+    image: postgres:alpine
+    volumes:
+    - ./previous:/docker-entrypoint-initdb.d
+    ports:
+      - "5433:5432"  
+    environment:
+      POSTGRES_DB: database
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: rootpassword
+  current:
+    image: postgres:alpine
+    volumes:
+    - ./current:/docker-entrypoint-initdb.d
+    ports:
+      - "5432:5432"  
+    environment:
+      POSTGRES_DB: database
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: rootpassword

tests/postgres/previous/tables.sql

@@ -0,0 +1,48 @@
+CREATE TABLE users (
+  userid SERIAL PRIMARY KEY, 
+  name TEXT, 
+  email TEXT,
+  password TEXT, 
+  created timestamp default current_timestamp,
+  updated timestamp default current_timestamp,
+  deleted timestamp default current_timestamp
+);
+
+CREATE TABLE posts (
+  postid SERIAL PRIMARY KEY, 
+  title TEXT, 
+  body TEXT,
+  userid INT, 
+  inreplyto INT, 
+  created timestamp default current_timestamp,
+  updated timestamp default current_timestamp,
+  deleted timestamp default current_timestamp,
+  FOREIGN KEY(userid) REFERENCES users(userid),
+  FOREIGN KEY(inreplyto) REFERENCES posts(postid)
+);
+
+CREATE TABLE likes (
+  postid INT,
+  userid INT,
+  created timestamp default current_timestamp,
+  deleted timestamp default current_timestamp,
+  PRIMARY KEY(postid,userid),
+  FOREIGN KEY(postid) REFERENCES posts(postid),
+  FOREIGN KEY(userid) REFERENCES users(userid)
+);
+
+
+CREATE PROCEDURE prune_accounts() 
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  UPDATE users SET deleted=NOW() WHERE updated < CURRENT_DATE - '6 months'::interval;
+END;$$ ;
+
+CREATE PROCEDURE prune_posts() 
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  -- this is a bad implementation, need to fix later
+  UPDATE posts SET deleted=NOW() WHERE deleted IS NULL AND userid in (SELECT userid FROM users WHERE deleted > CURRENT_DATE - '1 months'::interval);
+END;$$;

tests/postgres/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+docker-compose up -d
+
+sleep 30
+
+node ../../index.js -s public -c 'postgres://user:rootpassword@127.0.0.1:5432/database' -p 'postgres://user:rootpassword@127.0.0.1:5433/database' > output.html 
+node ../../index.js -c 'postgres://user:rootpassword@127.0.0.1:5432/database' -p 'postgres://user:rootpassword@127.0.0.1:5433/database' > schemaless.html 
+
+docker-compose down
+
+diff output.html schemaless.html > /dev/null && rm output.html schemaless.html

tests/sqlite/test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+rm current.db previous.db
+
 docker-compose up
 
 node ../../index.js -c sqlite://./current.db -p sqlite://./previous.db > test.html

tests/test.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo $(cd sqlite/; ./test.sh) && \
+echo $(cd postgres/; ./test.sh) && \
+echo $(cd mysql/; ./test.sh) && \
+echo success