From ee71226bf15afacc52a5b3310de1c1ab60564c8e Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Wed, 6 Nov 2024 08:52:13 +0100 Subject: [PATCH 01/12] Snapshot version --- build.gradle | 2 +- demos/demo-data-dictionary/gradle.properties | 2 +- demos/demo-data-lineage/gradle.properties | 2 +- demos/demo-erdiagram/gradle.properties | 2 +- demos/demo-erdiagram/pom.xml | 2 +- demos/demo-sqlscript/gradle.properties | 2 +- documentation-generator-gradle/README.md | 2 +- documentation-generator-maven-plugin/README.md | 2 +- documentation-generator-maven-plugin/jreleaser.yml | 2 +- documentation-generator-maven-plugin/pom.xml | 2 +- jreleaser.yml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 331793b..9703e58 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { allprojects { group = 'dk.fust.docgen' - version = '0.0.8' + version = '0.0.9-SNAPSHOT' plugins.withType(JavaPlugin).whenPluginAdded { tasks.withType(Test).configureEach { diff --git a/demos/demo-data-dictionary/gradle.properties b/demos/demo-data-dictionary/gradle.properties index 41a99d3..34489f0 100644 --- a/demos/demo-data-dictionary/gradle.properties +++ b/demos/demo-data-dictionary/gradle.properties @@ -1 +1 @@ -documentationGeneratorVersion = 0.0.8 +documentationGeneratorVersion = 0.0.9-SNAPSHOT diff --git a/demos/demo-data-lineage/gradle.properties b/demos/demo-data-lineage/gradle.properties index 41a99d3..34489f0 100644 --- a/demos/demo-data-lineage/gradle.properties +++ b/demos/demo-data-lineage/gradle.properties @@ -1 +1 @@ -documentationGeneratorVersion = 0.0.8 +documentationGeneratorVersion = 0.0.9-SNAPSHOT diff --git a/demos/demo-erdiagram/gradle.properties b/demos/demo-erdiagram/gradle.properties index 41a99d3..34489f0 100644 --- a/demos/demo-erdiagram/gradle.properties +++ b/demos/demo-erdiagram/gradle.properties @@ -1 +1 @@ -documentationGeneratorVersion = 0.0.8 +documentationGeneratorVersion = 0.0.9-SNAPSHOT diff --git a/demos/demo-erdiagram/pom.xml b/demos/demo-erdiagram/pom.xml index a03ac7a..6a22111 100644 --- a/demos/demo-erdiagram/pom.xml +++ b/demos/demo-erdiagram/pom.xml @@ -35,7 +35,7 @@ dk.fust.docgen documentation-generator-maven-plugin - 0.0.8 + 0.0.9-SNAPSHOT generator-configuration.yml diff --git a/demos/demo-sqlscript/gradle.properties b/demos/demo-sqlscript/gradle.properties index 41a99d3..34489f0 100644 --- a/demos/demo-sqlscript/gradle.properties +++ b/demos/demo-sqlscript/gradle.properties @@ -1 +1 @@ -documentationGeneratorVersion = 0.0.8 +documentationGeneratorVersion = 0.0.9-SNAPSHOT diff --git a/documentation-generator-gradle/README.md b/documentation-generator-gradle/README.md index 3992c68..5fb8673 100644 --- a/documentation-generator-gradle/README.md +++ b/documentation-generator-gradle/README.md @@ -5,7 +5,7 @@ ### gradle.properties Add the version of documentation-generator you want to use in `gradle.properties`. ```groovy -documentationGeneratorVersion = 0.0.8 +documentationGeneratorVersion = 0.0.9-SNAPSHOT ``` ### build.gradle diff --git a/documentation-generator-maven-plugin/README.md b/documentation-generator-maven-plugin/README.md index 8ad98ef..2c354c7 100644 --- a/documentation-generator-maven-plugin/README.md +++ b/documentation-generator-maven-plugin/README.md @@ -12,7 +12,7 @@ In the `build`-section of your pom, you can add the `generateDocumentation` goal dk.fust.docgen documentation-generator-maven-plugin - 0.0.8 + 0.0.9-SNAPSHOT generator-configuration.yml diff --git a/documentation-generator-maven-plugin/jreleaser.yml b/documentation-generator-maven-plugin/jreleaser.yml index 6bb91fd..e565d07 100644 --- a/documentation-generator-maven-plugin/jreleaser.yml +++ b/documentation-generator-maven-plugin/jreleaser.yml @@ -1,7 +1,7 @@ # $schema: "../jreleaser-schema-1,14.0.json" project: name: documentation-generator - version: 0.0.8 + version: 0.0.9-SNAPSHOT description: "Documentation Generator" longDescription: Generates documentation based upon a yaml-file. Describe how your database looks, in a single file (or files) and then generate the corresponding documentation. diff --git a/documentation-generator-maven-plugin/pom.xml b/documentation-generator-maven-plugin/pom.xml index ed91274..6569c0e 100644 --- a/documentation-generator-maven-plugin/pom.xml +++ b/documentation-generator-maven-plugin/pom.xml @@ -5,7 +5,7 @@ dk.fust.docgen documentation-generator-maven-plugin - 0.0.8 + 0.0.9-SNAPSHOT maven-plugin Documentation Generator Maven Plugin diff --git a/jreleaser.yml b/jreleaser.yml index c114417..0381e2b 100644 --- a/jreleaser.yml +++ b/jreleaser.yml @@ -1,6 +1,6 @@ project: name: documentation-generator - version: 0.0.8 + version: 0.0.9-SNAPSHOT description: "Documentation Generator" longDescription: Generates documentation based upon a yaml-file. Describe how your database looks, in a single file (or files) and then generate the corresponding documentation. From c3b51764495ba94815f6bd360a5892b691e1ffe5 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Wed, 6 Nov 2024 08:56:15 +0100 Subject: [PATCH 02/12] Data Dictionary --- .../dk/fust/docgen/model/datadict/Column.java | 4 ++-- .../model/datadict/DataDictionaryFile.java | 4 ++++ .../main/resources/documentation-schema.json | 12 ++++++---- .../java/dk/fust/docgen/ModelValidator.java | 23 +++++++++++++++++++ .../csv/format/table/CSVTableFormatter.java | 3 --- .../datadict/DataDictionaryConfiguration.java | 2 ++ .../datadict/DataDictionaryGenerator.java | 20 +++++++++++++--- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/Column.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/Column.java index c96b0be..715d4ec 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/Column.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/Column.java @@ -33,6 +33,6 @@ public class Column { @Description("Maximum value of the content") private Long maximumValue; - @Description(value = "Is this column a part of the unique key?", defaultBoolean = false) - private Boolean unique = false; + @Description(value = "Is this column a part of the unique key?") + private String keys; } diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java index 1b30163..bbd76ea 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java @@ -17,6 +17,10 @@ public class DataDictionaryFile { @Description(value = "Data dictionary version", required = true) private String version; + @Description("Description of the file") + private String fileDescription; + @Description(value = "The files columns", required = true) private List columns; + } diff --git a/documentation-generator-api/src/main/resources/documentation-schema.json b/documentation-generator-api/src/main/resources/documentation-schema.json index bb3a936..4a4a8c3 100644 --- a/documentation-generator-api/src/main/resources/documentation-schema.json +++ b/documentation-generator-api/src/main/resources/documentation-schema.json @@ -68,6 +68,10 @@ "type" : "string", "description" : "Example on how the content may look. Could be a description as well: 'value between 0 and 1. eg 0,3333'" }, + "keys" : { + "type" : "string", + "description" : "Is this column a part of the unique key?" + }, "mandatory" : { "type" : "boolean", "description" : "If true, then the content may not be null or empty" @@ -83,16 +87,16 @@ "regex" : { "type" : "string", "description" : "Regular expression that the contain must obey. Example: ((\\d{3}[A-Z]?)|0{4})" - }, - "unique" : { - "type" : "boolean", - "description" : "Is this column a part of the unique key?" } }, "required" : [ "columnName", "dataType" ], "description" : "The files columns" } }, + "fileDescription" : { + "type" : "string", + "description" : "Description of the file" + }, "fileName" : { "type" : "string", "description" : "Filename for the data dictionary" diff --git a/documentation-generator-core/src/main/java/dk/fust/docgen/ModelValidator.java b/documentation-generator-core/src/main/java/dk/fust/docgen/ModelValidator.java index 5aa60dc..864cb1a 100644 --- a/documentation-generator-core/src/main/java/dk/fust/docgen/ModelValidator.java +++ b/documentation-generator-core/src/main/java/dk/fust/docgen/ModelValidator.java @@ -6,6 +6,8 @@ import dk.fust.docgen.model.Index; import dk.fust.docgen.model.Table; import dk.fust.docgen.model.View; +import dk.fust.docgen.model.datadict.Column; +import dk.fust.docgen.model.datadict.DataDictionaryFile; import dk.fust.docgen.util.Assert; import java.util.HashSet; @@ -35,6 +37,27 @@ public void validate() { throw new IllegalArgumentException("Documentation cannot be null"); } validateTables(); + validateDataDictionary(); + } + + private void validateDataDictionary() { + if (documentation.getDataDictionary() != null && documentation.getDataDictionary().getDataDictionaryFiles() != null) { + documentation.getDataDictionary().getDataDictionaryFiles().forEach(this::validateDictionaryFile); + } + } + + private void validateDictionaryFile(DataDictionaryFile dataDictionaryFile) { + String filename = dataDictionaryFile.getFileName(); + Assert.isNotNull(filename, "Filename cannot be null"); + Assert.isNotNull(dataDictionaryFile.getVersion(), "File %s has no version".formatted(filename)); + Assert.isNotNull(dataDictionaryFile.getColumns(), "File %s has no columns".formatted(filename)); + dataDictionaryFile.getColumns().forEach(this::validateColumn); + } + + private void validateColumn(Column column) { + String name = column.getColumnName(); + Assert.isNotNull(column.getColumnName(), "Column name cannot be null"); + Assert.isNotNull(column.getDataType(), "Column %s has no data type".formatted(name)); } private void validateTables() { diff --git a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java index 3216f85..432e49b 100644 --- a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java +++ b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java @@ -7,13 +7,10 @@ import lombok.Data; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; import java.io.IOException; import java.io.StringWriter; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * Converts from model of a Table to a CSV representation diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java index 0455ba3..63f7e73 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java @@ -24,6 +24,8 @@ public class DataDictionaryConfiguration implements GeneratorConfiguration { private TableFormatter tableFormatter = new MarkdownTableFormatter(); + private boolean addDescriptionForFile = false; + @Override public void validate() { Assert.isNotNull(destination, "Destination is required"); diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java index b834b96..c38ffe4 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java @@ -28,14 +28,26 @@ public void generate(Documentation documentation, GeneratorConfiguration generat Assert.isTrue(generatorConfiguration instanceof DataDictionaryConfiguration, "Configuration must be DataDictionaryConfiguration"); DataDictionaryConfiguration dataDictionaryConfiguration = (DataDictionaryConfiguration) generatorConfiguration; if (documentation.getDataDictionary() != null && documentation.getDataDictionary().getDataDictionaryFiles() != null) { - sendTableToDestination(generateTable(documentation), dataDictionaryConfiguration); + sendTableToDestination(generateTable(documentation, dataDictionaryConfiguration), dataDictionaryConfiguration); } } - private FormatTable generateTable(Documentation documentation) { + private FormatTable generateTable(Documentation documentation, DataDictionaryConfiguration dataDictionaryConfiguration) { FormatTable formatTable = new FormatTable(); formatTable.getRows().add(createHeaderRow()); for (DataDictionaryFile dataDictionaryFile : documentation.getDataDictionary().getDataDictionaryFiles()) { + if (dataDictionaryConfiguration.isAddDescriptionForFile()) { + Row row = new Row(); + List cells = row.getCells(); + cells.add(new Cell(dataDictionaryFile.getFileName())); + cells.add(new Cell(null)); + cells.add(new Cell(null)); + cells.add(new Cell(null)); + cells.add(null); + cells.add(null); + cells.add(new Cell(dataDictionaryFile.getFileDescription())); + formatTable.getRows().add(row); + } int position = 1; for (Column column : dataDictionaryFile.getColumns()) { Row row = new Row(); @@ -45,7 +57,8 @@ private FormatTable generateTable(Documentation documentation) { cells.add(new Cell(Integer.toString(position++))); cells.add(new Cell(column.getDataType())); cells.add(new Cell(column.getMandatory() ? "Yes" : "No")); - cells.add(new Cell(column.getUnique() ? "Yes" : "No")); + cells.add(new Cell(column.getKeys())); + cells.add(new Cell(column.getColumnDescription())); formatTable.getRows().add(row); } } @@ -61,6 +74,7 @@ private static Row createHeaderRow() { headerCells.add(new Cell(1, 1, "Type", true)); headerCells.add(new Cell(1, 1, "Mandatory", true)); headerCells.add(new Cell(1, 1, "Keys", true)); + headerCells.add(new Cell(1, 1, "Description", true)); return headerRow; } From f5fc94a2ed748f89d2f0785003ca8bb3433e1cea Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Wed, 6 Nov 2024 10:36:12 +0100 Subject: [PATCH 03/12] Added filter to SqlScript --- .../dk/fust/docgen/sqlscript/SqlScriptConfiguration.java | 2 ++ .../java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java index 9ddab6d..14e78ca 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java @@ -18,6 +18,8 @@ public class SqlScriptConfiguration implements GeneratorConfiguration { private Destination destination; + private String filter; + @Override public void validate() { Assert.isNotNull(documentationFile, "documentationFile must not be null"); diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java index 2d447d5..be214f6 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java @@ -9,6 +9,7 @@ import dk.fust.docgen.model.Index; import dk.fust.docgen.model.Table; import dk.fust.docgen.model.View; +import dk.fust.docgen.util.Assert; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -27,8 +28,10 @@ public class SqlScriptGenerator implements Generator { @Override public void generate(Documentation documentation, GeneratorConfiguration generatorConfiguration) throws IOException { log.info("Generating SQL-scripts..."); + Assert.isTrue(generatorConfiguration instanceof SqlScriptConfiguration, "configuration must be an instance of SqlScriptConfiguration"); + SqlScriptConfiguration sqlScriptConfiguration = (SqlScriptConfiguration) generatorConfiguration; String schemaName = documentation.getSchemaName(); - for (Table table : documentation.getTables()) { + for (Table table : documentation.filterTables(sqlScriptConfiguration.getFilter())) { Generation generationForTable = documentation.getGenerationForTable(table); if (table.getCreateTableScript() != null) { StringBuilder sqlScript = new StringBuilder(256); From 2f143e44fa28647df3636d13a3bc59e9cc096b5a Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Wed, 6 Nov 2024 10:37:15 +0100 Subject: [PATCH 04/12] Documentation --- .../README.md | 51 +++++++++++++++++++ .../README.md | 23 +++++++-- .../README.md | 24 ++++++++- .../README.md | 26 +++++++--- 4 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 generators/documentation-generator-data-dictionary/README.md diff --git a/generators/documentation-generator-data-dictionary/README.md b/generators/documentation-generator-data-dictionary/README.md new file mode 100644 index 0000000..628e613 --- /dev/null +++ b/generators/documentation-generator-data-dictionary/README.md @@ -0,0 +1,51 @@ +# Documentation Generator Data Dictionary + +This generator can generate [data dictionaries](https://atlan.com/what-is-a-data-dictionary/). + +## Configuration + +To use the data dictionary generator, you must configure it with `dk.fust.docgen.datadict.DataDictionaryConfiguration` + +| Setting | Type | Description | Default | +|-----------------------|----------------|--------------------------------------------------------------|--------------------------| +| documentationFile | File | Location of the documentation file | | +| destination | Destination | Where to send the generated documentation | | +| key | String | Key to identify it on the destination | | +| tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | +| addDescriptionForFile | boolean | Will add a header row containing the description of the file | false | + +## Examples + +### gradle.build + +```groovy +import dk.fust.docgen.datadict.DataDictionaryConfiguration +import dk.fust.docgen.destination.MarkdownDestination +import dk.fust.docgen.format.table.MarkdownTableFormatter + +new DataDictionaryConfiguration( + documentationFile: new File(projectDir, 'data-dictionary.yml'), + destination: new MarkdownDestination( + file: new File('data-dictionary.md'), + ), + tableFormatter: new MarkdownTableFormatter(), +) +``` + +### generator-configuration.yml + +```yaml +- className: dk.fust.docgen.datadict.DataDictionaryConfiguration + documentationFile: data-dictionary.yml + tableFormatter: + className: dk.fust.docgen.format.table.MarkdownTableFormatter + destination: + className: dk.fust.docgen.destination.MarkdownDestination + file: data-dictionary.md +``` + +--- + +## Demo + +Check the demo in [demo-data-dictionary](../../demos/demo-data-dictionary) diff --git a/generators/documentation-generator-datalineage/README.md b/generators/documentation-generator-datalineage/README.md index 1356926..b5c7f13 100644 --- a/generators/documentation-generator-datalineage/README.md +++ b/generators/documentation-generator-datalineage/README.md @@ -15,7 +15,9 @@ To use the entity-relation diagram generator, you must configure it with `dk.fus | tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | | filter | String | If specified, only tables with this filter is used | | -## Example in a gradle.build +## Examples + +### gradle.build ```groovy import dk.fust.docgen.datalineage.DataLineageConfiguration @@ -24,7 +26,7 @@ import dk.fust.docgen.format.table.MarkdownTableFormatter new DataLineageConfiguration( key: 'my-data-lineage', - documentationFile: new File(projectDir, 'documentation.yaml'), + documentationFile: new File(projectDir, 'documentation.yml'), destination: new MarkdownDestination( file: new File('README.md'), ), @@ -36,6 +38,21 @@ new DataLineageConfiguration( ) ``` +### generator-configuration.yml + +```yaml +- className: dk.fust.docgen.datalineage.DataLineageConfiguration + documentationFile: documentation.yml + key: my-data-lineage + tableFormatter: + className: dk.fust.docgen.format.table.MarkdownTableFormatter + destination: + className: dk.fust.docgen.destination.MarkdownDestination + file: README.md +``` + +--- + ## Demo -Check the demo in [demo-datalineage](../../demos/demo-datalineage) +Check the demo in [demo-data-lineage](../../demos/demo-data-lineage) diff --git a/generators/documentation-generator-erdiagram/README.md b/generators/documentation-generator-erdiagram/README.md index 0586073..56a7682 100644 --- a/generators/documentation-generator-erdiagram/README.md +++ b/generators/documentation-generator-erdiagram/README.md @@ -28,7 +28,10 @@ so you can have several ER-diagram with a sub-set of the tables. Read more about `MarkdownDestination` [here](../../README.md#dkfustdocgendestinationmarkdowndestination-) -## Example in a gradle.build +## Examples + +### gradle.build + ```groovy import dk.fust.docgen.erdiagram.GenerateKey import dk.fust.docgen.erdiagram.generators.UMLGenerator @@ -36,7 +39,7 @@ import dk.fust.docgen.destination.MarkdownDestination import dk.fust.docgen.erdiagram.ERDiagramConfiguration new ERDiagramConfiguration( - documentationFile: new File(projectDir, 'documentation.yaml'), + documentationFile: new File(projectDir, 'documentation.yml'), umlGenerator : UMLGenerator.MERMAID, // Can be omitted because it's default generateKeys: [ // Empty filter means all groups @@ -49,6 +52,23 @@ new ERDiagramConfiguration( ) ``` +### generator-configuration.yml + +```yaml +- className: dk.fust.docgen.erdiagram.ERDiagramConfiguration + documentationFile: documentation.yml + umlGenerator: MERMAID + generateKeys: + - className: dk.fust.docgen.erdiagram.GenerateKey + destinationKey: MODEL_MERMAID_PLACEHOLDER + - className: dk.fust.docgen.erdiagram.GenerateKey + destinationKey: MODEL_MERMAID_GROUP_PLACEHOLDER + filter: my_group + destination: + className: dk.fust.docgen.destination.MarkdownDestination + file: README.md +``` + ## Demo Check the demo in [demo-erdiagram](../../demos/demo-erdiagram) diff --git a/generators/documentation-generator-sqlscript/README.md b/generators/documentation-generator-sqlscript/README.md index cc51971..62adc49 100644 --- a/generators/documentation-generator-sqlscript/README.md +++ b/generators/documentation-generator-sqlscript/README.md @@ -6,19 +6,22 @@ This generator can generate sql files containing create tables, indexes and view To use the sql generator, you must configure it with `dk.fust.docgen.sqlscript.SqlScriptConfiguration` -| Setting | Type | Description | Default | -|-------------------|-------------------|---------------------------------------------------------------------|----------| -| documentationFile | File | Location of the documentation file | | -| destination | Destination | Where to send the generated documentation | | +| Setting | Type | Description | Default | +|-------------------|-------------|----------------------------------------------------|-----------| +| documentationFile | File | Location of the documentation file | | +| destination | Destination | Where to send the generated documentation | | +| filter | String | If specified, only tables with this filter is used | | +## Examples + +### gradle.build -## Example in a gradle.build ```groovy import dk.fust.docgen.sqlscript.SqlScriptConfiguration import dk.fust.docgen.destination.DirectoryDestination new SqlScriptConfiguration( - documentationFile: new File(projectDir, 'documentation.yaml'), + documentationFile: new File(projectDir, 'documentation.yml'), destination: new DirectoryDestination( directory: new File('scripts'), createParentDirectories: true @@ -26,6 +29,17 @@ new SqlScriptConfiguration( ) ``` +### generator-configuration.yml + +```yaml +- className: dk.fust.docgen.sqlscript.SqlScriptConfiguration + documentationFile: documentation.yml + destination: + className: dk.fust.docgen.destination.DirectoryDestination + directory: scripts + createParentDirectories: true +``` + ## Demo Check the demo in [demo-sqlscript](../../demos/demo-sqlscript) From b2261538bc5be13ba08f33808e3e77d44e6623b8 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Wed, 6 Nov 2024 11:34:32 +0100 Subject: [PATCH 05/12] Handling relative dirs --- .../dk/fust/docgen/DocumentationGenerator.java | 16 ++++++++++++++-- .../DocumentationConfigurationLoaderService.java | 6 +++--- .../docgen/service/DocumentationService.java | 3 +++ .../gradle/DocumentationGeneratorPlugin.java | 14 ++++++++++---- .../docgen/confluence/ConfluenceService.java | 6 +++--- .../destination/ConfluenceDestination.java | 4 ++-- .../datalineage/DataLineageConfiguration.java | 1 - .../docgen/erdiagram/ERDiagramConfiguration.java | 1 - 8 files changed, 35 insertions(+), 16 deletions(-) diff --git a/documentation-generator-core/src/main/java/dk/fust/docgen/DocumentationGenerator.java b/documentation-generator-core/src/main/java/dk/fust/docgen/DocumentationGenerator.java index ed213bc..6e629c2 100644 --- a/documentation-generator-core/src/main/java/dk/fust/docgen/DocumentationGenerator.java +++ b/documentation-generator-core/src/main/java/dk/fust/docgen/DocumentationGenerator.java @@ -3,8 +3,10 @@ import dk.fust.docgen.model.Documentation; import dk.fust.docgen.service.DocumentationService; import dk.fust.docgen.util.Assert; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import java.io.File; import java.io.IOException; import java.util.List; @@ -17,6 +19,9 @@ public class DocumentationGenerator { private final DocumentationService documentationService = new DocumentationService(); + @Setter + private File baseDir; + /** * Loops over all the configurations and generates the documentation * @param generatorConfigurations list of configurations to run @@ -36,7 +41,14 @@ public void generate(GeneratorConfiguration generatorConfiguration) { generatorConfiguration.validate(); generatorConfiguration.getDestination().validate(); try { - Documentation documentation = documentationService.loadDocumentation(generatorConfiguration.getDocumentationFile()); + File documentationFile = generatorConfiguration.getDocumentationFile(); + log.debug("DocumentationGenerator: Loading documentation file: {}", documentationFile.getAbsolutePath()); + if (!documentationFile.exists()) { + log.debug("Can't find file -> trying in baseDir"); + documentationFile = new File(baseDir, documentationFile.getPath()); + log.debug("DocumentationGenerator: Loading documentation file in baseDir: {}, exists={}", documentationFile.getAbsolutePath(), documentationFile.exists()); + } + Documentation documentation = documentationService.loadDocumentation(documentationFile); validateModel(documentation); Generator generator = generatorConfiguration.getGenerator(); generator.generate(documentation, generatorConfiguration); @@ -47,7 +59,7 @@ public void generate(GeneratorConfiguration generatorConfiguration) { } private void validateModel(Documentation documentation) { - log.info("Validating documentation..."); + log.debug("Validating documentation model..."); ModelValidator modelValidator = new ModelValidator(documentation); modelValidator.validate(); } diff --git a/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationConfigurationLoaderService.java b/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationConfigurationLoaderService.java index 5c30899..8515950 100644 --- a/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationConfigurationLoaderService.java +++ b/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationConfigurationLoaderService.java @@ -89,13 +89,13 @@ private Object convertObject(JsonNode jsonNode, Class dataType) { // Maybe we can write to it try { Files.writeString(file.toPath(), "something"); - log.info("We could write to the file"); + log.debug("We could write to the file"); file.delete(); } catch (IOException e) { - log.info("Can't find the file or write to it: {}", file.getAbsolutePath()); + log.debug("Can't find the file or write to it: {}", file.getAbsolutePath()); // Maybe it's in the same folder as the configuration file file = new File(configurationFile.getParentFile(), jsonNode.textValue()); - log.info("Trying {} instead", file.getAbsolutePath()); + log.debug("Trying {} instead", file.getAbsolutePath()); } } return file; diff --git a/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationService.java b/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationService.java index e45ce35..0a1d3cd 100644 --- a/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationService.java +++ b/documentation-generator-core/src/main/java/dk/fust/docgen/service/DocumentationService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import dk.fust.docgen.model.Documentation; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.IOException; @@ -13,6 +14,7 @@ /** * Service to read documentation files */ +@Slf4j public class DocumentationService { /** @@ -23,6 +25,7 @@ public class DocumentationService { * @throws IOException an error occurred */ public Documentation loadDocumentation(File file) throws IOException { + log.debug("DocumentationService.loadDocumentation file: {}", file.getAbsolutePath()); ObjectMapper objectMapper = createObjectMapper(file); return objectMapper.readValue(file, Documentation.class); } diff --git a/documentation-generator-gradle/src/main/java/dk/fust/docgen/gradle/DocumentationGeneratorPlugin.java b/documentation-generator-gradle/src/main/java/dk/fust/docgen/gradle/DocumentationGeneratorPlugin.java index 12b36b7..a80500e 100644 --- a/documentation-generator-gradle/src/main/java/dk/fust/docgen/gradle/DocumentationGeneratorPlugin.java +++ b/documentation-generator-gradle/src/main/java/dk/fust/docgen/gradle/DocumentationGeneratorPlugin.java @@ -21,10 +21,11 @@ public class DocumentationGeneratorPlugin implements Plugin { @Override public void apply(Project project) { + String projectDir = project.getProjectDir().getAbsolutePath(); DocumentationGeneratorPluginExtension extension = project.getExtensions().create("documentationGenerator", DocumentationGeneratorPluginExtension.class); Task generateDocumentationTask = project.task("generateDocumentation").doLast(task -> { - log.info("Validating configuration..."); + log.debug("Validating configuration..."); if (extension.getGeneratorConfigurations() == null && extension.getDocumentationConfigurationFile() == null) { throw new IllegalArgumentException("Need either GeneratorConfigurations or DocumentationConfigurationFile"); } @@ -45,11 +46,16 @@ public void apply(Project project) { log.info("Found configuration file: {}", documentationConfigurationFile.getPath());; confs = documentationConfigurationLoaderService.readConfigurations(documentationConfigurationFile); } else { - File projectConfigurationFile = new File(project.getProjectDir(), documentationConfigurationFile.getPath()); - log.info("Could not find configuration file -> trying i project dir: " + projectConfigurationFile.getAbsolutePath()); + File projectConfigurationFile = new File(projectDir, documentationConfigurationFile.getPath()); + log.debug("Could not find configuration file -> trying i project dir: " + projectConfigurationFile.getAbsolutePath()); + log.debug("Exists now: " + projectConfigurationFile.exists()); + if (projectConfigurationFile.exists()) { + log.info("Found project configuration file: {}", projectConfigurationFile.getPath()); + } + documentationGenerator.setBaseDir(new File(projectDir)); confs = documentationConfigurationLoaderService.readConfigurations(projectConfigurationFile); } - log.info("Generating documentation..."); + log.debug("Calling generate..."); documentationGenerator.generate(confs); } catch (IOException e) { throw new RuntimeException("Can't load generator configurations", e); diff --git a/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/ConfluenceService.java b/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/ConfluenceService.java index 12f00e1..de4cfa6 100644 --- a/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/ConfluenceService.java +++ b/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/ConfluenceService.java @@ -76,7 +76,7 @@ public Page updatePage(Page page) throws IOException { String url = "%s/content/%s".formatted(baseUrl, page.getId()); HttpPut put = new HttpPut(url); String json = pageToString(page); - log.info("Updating Confluence with: {}", json); + log.debug("Updating Confluence with: {}", json); put.setEntity(new StringEntity(json)); return callAndParse(put, Page.class); } @@ -93,11 +93,11 @@ public Page getChildWithTitle(String id, String titleForChild) throws IOExceptio HttpGet get = new HttpGet("%s?expand=page&limit=100".formatted(url)); PageSearch pageSearch = callAndParse(get, PageSearch.class); if (pageSearch.getPage() != null && pageSearch.getPage().getResults() != null && !pageSearch.getPage().getResults().isEmpty()) { - log.info("has child pages"); + log.debug("has child pages"); Optional pages = pageSearch.getPage().getResults().stream().filter(page -> page.getTitle().equals(titleForChild)).findFirst(); if (pages.isPresent()) { Page childPageInfo = pages.get(); - log.info("Found child page with title: {}", childPageInfo.getTitle()); + log.debug("Found child page with title: {}", childPageInfo.getTitle()); return getPage(childPageInfo.getId()); } } diff --git a/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/destination/ConfluenceDestination.java b/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/destination/ConfluenceDestination.java index 08e2d12..b8dc42e 100644 --- a/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/destination/ConfluenceDestination.java +++ b/extensions/documentation-generator-confluence/src/main/java/dk/fust/docgen/confluence/destination/ConfluenceDestination.java @@ -53,12 +53,12 @@ public void sendDocumentToDestination(String document, String destination) throw } Page page = getOrCreateChildWithTitle(id, pageTitle, confluenceService); - log.info("Got page: {}", page); + log.debug("Got page: {}", page); String originalHtml = page.getBody().getStorage().getValue(); replaceTable(page, document, destination); if (!originalHtml.equals(page.getBody().getStorage().getValue())) { - log.info("Table has changed -> updating Confluence"); + log.debug("Table has changed -> updating Confluence"); page.getVersion().setNumber(page.getVersion().getNumber() + 1); confluenceService.updatePage(page); log.info("Table is updated"); diff --git a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java index 8b0ecb9..8d5607a 100644 --- a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java +++ b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java @@ -36,7 +36,6 @@ public class DataLineageConfiguration implements GeneratorConfiguration { @Override public void validate() { Assert.isNotNull(documentationFile, "documentationFile must not be null"); - Assert.isTrue(documentationFile.exists(), "documentationFile must exist"); Assert.isNotNull(destination, "destination must not be null"); Assert.isNotNull(key, "key must not be null"); Assert.isNotNull(tableFormatter, "tableFormatter must not be null"); diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramConfiguration.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramConfiguration.java index 0cb4405..ec9a086 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramConfiguration.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramConfiguration.java @@ -27,7 +27,6 @@ public class ERDiagramConfiguration implements GeneratorConfiguration { @Override public void validate() { Assert.isNotNull(documentationFile, "documentationFile must not be null"); - Assert.isTrue(documentationFile.exists(), "documentationFile must exist: " + documentationFile.getAbsolutePath()); Assert.isNotNull(umlGenerator, "umlGenerator must not be null"); Assert.isNotNull(generateKeys, "generateKeys must not be null"); } From db5e68a493b75af47c2fb46a6423235168d4221f Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Thu, 7 Nov 2024 14:08:53 +0100 Subject: [PATCH 06/12] Bug: Didn't use expected delimeter --- .../java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java index 432e49b..6db6b52 100644 --- a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java +++ b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java @@ -25,7 +25,7 @@ public String formatTable(FormatTable formatTable) { StringWriter sw = new StringWriter(); CSVFormat csvFormat = CSVFormat.DEFAULT.builder() - .setDelimiter(';') + .setDelimiter(delimiter) .build(); try (final CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { From e434371b6830d00e95ea3227211db86e0af8f56f Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Fri, 8 Nov 2024 16:51:41 +0100 Subject: [PATCH 07/12] Added filter to data-dictionaries --- README.md | 80 ++++++++++++++++--- .../dk/fust/docgen/model/Documentation.java | 16 +++- .../main/java/dk/fust/docgen/model/Table.java | 1 + .../model/datadict/DataDictionaryFile.java | 3 + .../README.md | 27 +++++++ .../documentation-generator-csv/README.md | 10 +++ .../README.md | 3 +- .../datadict/DataDictionaryConfiguration.java | 2 + .../datadict/DataDictionaryGenerator.java | 3 +- 9 files changed, 132 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d6d94c8..00e0b35 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The generators acts as plugin - Just implement `GeneratorConfiguration`. title: Basic idea --- flowchart TD + docFile["documentation file"] docgen["Documentation Generator"] generator1["Generator 1"] generator1Generate[[Generates SQL-scripts]] @@ -52,10 +53,16 @@ flowchart TD destination4["Destination 4"] csv(["CSV file"]) + generatorX["Generator X"] + destinationX["Destination X"] + something(["Something else"]) + + docFile --> docgen docgen --> generator1 docgen --> generator2 docgen --> generator3 docgen --> generator4 + docgen --> generatorX generator1 --> generator1Generate generator1Generate --> destination1 @@ -69,6 +76,9 @@ flowchart TD generator4 --> destination4 destination4 --> csv + + generatorX --> destinationX + destinationX --> something ``` --- @@ -240,18 +250,68 @@ Everything in between will be substituted. Class name: `dk.fust.docgen.confluence.destination.ConfluenceDestination` -In order to use Confluence as destination, you'll need to add the module: -[documentation-generator-confluence](extensions/documentation-generator-confluence/) +In order to use Confluence as destination, you'll need to add the extension `documentation-generator-confluence`. + +Read the documentation [here](extensions/documentation-generator-confluence) + +--- + +## Table format + +Some generators use tables and needs a formatter to create a string representation of the table. +The table formatter you want to use, is configured when you configure the generator. +Example: +```yaml +- className: dk.fust.docgen.datadict.DataDictionaryConfiguration + documentationFile: data-dictionary.yml + tableFormatter: + className: dk.fust.docgen.csv.format.table.CSVTableFormatter + destination: + className: dk.fust.docgen.destination.FileDestination + file: data_dictionary-output.csv +``` + +### MarkdownTableFormatter + +Class name: `dk.fust.docgen.format.table.MarkdownTableFormatter` + +Default table formatter for most generators. + +Generates the table in a format that can be used i Markdown files. + +Use it together with [MarkdownDestination](#markdowndestination). + +### HTMLTableFormatter + +Class name: 'dk.fust.docgen.format.table.HTMLTableFormatter' + +| Setting | Type | Description | Default | +|--------------|---------------------|-----------------------------------------|---------| +| dataFields | Map | Data fields to be appended to the table | | +| columnWidths | List | Setting column withs | | + +#### Example + +```yaml +tableFormatter: + className: dk.fust.docgen.format.table.HTMLTableFormatter + dataFields: + table-width: "1800" + columnWidths: + - "255" + - "328" + - "114" + - "115" + - "149" + - "242" +``` + +### CSVTableFormatter + +Class name: `dk.fust.docgen.csv.format.table.CSVTableFormatter` -| Setting | Type | Description | Exanple | -|---------------------|--------|-------------------------|-----------------------------------------| -| baseUrl | String | Base URL to Confluence. | https://xxx.atlassian.net/wiki/rest/api | -| documentationPageId | String | Id of the parent page. | 123456 | -| spaceKey | String | Confluence space key | ABC | -| parentPageTitle | String | | My Parent Page | -| pageTitle | String | | My Page | +Read the documentation [here](extensions/documentation-generator-csv) -Username and personal access token for Confluence will be promptet . --- diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java index ddb61c6..a3b5d9d 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import dk.fust.docgen.model.datadict.DataDictionary; +import dk.fust.docgen.model.datadict.DataDictionaryFile; import lombok.Data; import java.util.ArrayList; @@ -97,7 +98,7 @@ private void mergeGenerations(Generation from, Generation to) { } /** - * Return those tables that have a tag that is equal to the filter + * Returns those tables that have a tag that is equal to the filter * @param filter filter to search for * @return only matching tables */ @@ -109,6 +110,19 @@ public List filterTables(String filter) { return tables; } + /** + * Returns those data dictionaries that have a tag that i equal to the filter + * @param filter filter to search for + * @return only matching data dictionaries + */ + public List filterDataDictionaryFiles(String filter) { + if (filter != null && !filter.isEmpty() && dataDictionary.getDataDictionaryFiles() != null) { + // Only those with the filer + return dataDictionary.getDataDictionaryFiles().stream().filter(t -> t.getTags() != null && t.getTags().contains(filter)).toList(); + } + return dataDictionary.getDataDictionaryFiles(); + } + /** * Finds the first table with the table name * @param tableName searching for table name diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Table.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Table.java index 5c2d976..280ccc1 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Table.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Table.java @@ -16,6 +16,7 @@ public class Table { @Description("Comment to be added to the table") private String comment; + @Description("List of tags that may be used in a filter") private List tags; @Description(value = "Filename of sql-script-file to be generated. If left out, no files are generated") diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java index bbd76ea..d7c5527 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/datadict/DataDictionaryFile.java @@ -23,4 +23,7 @@ public class DataDictionaryFile { @Description(value = "The files columns", required = true) private List columns; + @Description("List of tags that may be used in a filter") + private List tags; + } diff --git a/extensions/documentation-generator-confluence/README.md b/extensions/documentation-generator-confluence/README.md index cdb3fd8..319ce43 100644 --- a/extensions/documentation-generator-confluence/README.md +++ b/extensions/documentation-generator-confluence/README.md @@ -1,3 +1,30 @@ # Confluence If you add this module, you can send documentation to Confluence. + +## ConfluenceDestination + +Class name: `dk.fust.docgen.confluence.destination.ConfluenceDestination` + +### Setup +In order to use Confluence as destination, you'll need to add this module in your build.gradle: +```groovy +buildscript { + dependencies { + classpath("dk.fust.docgen:documentation-generator-gradle:${documentationGeneratorVersion}") + classpath("dk.fust.docgen:documentation-generator-confluence:${documentationGeneratorVersion}") + } +} +``` + +### Configuration of destination + +| Setting | Type | Description | Exanple | +|---------------------|--------|-------------------------|-----------------------------------------| +| baseUrl | String | Base URL to Confluence. | https://xxx.atlassian.net/wiki/rest/api | +| documentationPageId | String | Id of the parent page. | 123456 | +| spaceKey | String | Confluence space key | ABC | +| parentPageTitle | String | | My Parent Page | +| pageTitle | String | | My Page | + +Username and personal access token for Confluence will be promptet. diff --git a/extensions/documentation-generator-csv/README.md b/extensions/documentation-generator-csv/README.md index 56eeac0..1fec879 100644 --- a/extensions/documentation-generator-csv/README.md +++ b/extensions/documentation-generator-csv/README.md @@ -1,3 +1,13 @@ # CSV If you add this module, you can generate documentation as CSV-files + +## CSVTableFormatter + +Class name: `dk.fust.docgen.csv.format.table.CSVTableFormatter` + +Formats tables as CSV files. + +| Setting | Type | Description | Default | +|-----------|--------|---------------------------------------|---------| +| delimeter | String | Which delimeter to use between fields | ; | diff --git a/generators/documentation-generator-data-dictionary/README.md b/generators/documentation-generator-data-dictionary/README.md index 628e613..917ed2d 100644 --- a/generators/documentation-generator-data-dictionary/README.md +++ b/generators/documentation-generator-data-dictionary/README.md @@ -6,13 +6,14 @@ This generator can generate [data dictionaries](https://atlan.com/what-is-a-data To use the data dictionary generator, you must configure it with `dk.fust.docgen.datadict.DataDictionaryConfiguration` -| Setting | Type | Description | Default | +| Setting | Type | Description | Default | |-----------------------|----------------|--------------------------------------------------------------|--------------------------| | documentationFile | File | Location of the documentation file | | | destination | Destination | Where to send the generated documentation | | | key | String | Key to identify it on the destination | | | tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | | addDescriptionForFile | boolean | Will add a header row containing the description of the file | false | +| filter | String | If specified, only tables with this filter is used | | ## Examples diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java index 63f7e73..ac6d17e 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java @@ -22,6 +22,8 @@ public class DataDictionaryConfiguration implements GeneratorConfiguration { private String key; + private String filter; + private TableFormatter tableFormatter = new MarkdownTableFormatter(); private boolean addDescriptionForFile = false; diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java index c38ffe4..54d4652 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java @@ -35,7 +35,8 @@ public void generate(Documentation documentation, GeneratorConfiguration generat private FormatTable generateTable(Documentation documentation, DataDictionaryConfiguration dataDictionaryConfiguration) { FormatTable formatTable = new FormatTable(); formatTable.getRows().add(createHeaderRow()); - for (DataDictionaryFile dataDictionaryFile : documentation.getDataDictionary().getDataDictionaryFiles()) { + List dataDictionaryFiles = documentation.filterDataDictionaryFiles(dataDictionaryConfiguration.getFilter()); + for (DataDictionaryFile dataDictionaryFile : dataDictionaryFiles) { if (dataDictionaryConfiguration.isAddDescriptionForFile()) { Row row = new Row(); List cells = row.getCells(); From 8bb8a198b05174d525b71708c955f028ae7b0c88 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Mon, 11 Nov 2024 13:11:11 +0100 Subject: [PATCH 08/12] Versioning schema file in URL --- .github/workflows/github-pages.yml | 4 +--- README.md | 6 +++--- demos/demo-data-dictionary/documentation.yaml | 2 +- demos/demo-data-lineage/documentation.yaml | 2 +- .../demo-data-lineage/external-documentation.yaml | 2 +- demos/demo-erdiagram/documentation.yaml | 2 +- demos/demo-sqlscript/documentation.yaml | 2 +- .../src/test/resources/documentation-test.json | 2 +- site/index.html | 14 ++++++++++---- 9 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index 389bea0..be42081 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -4,8 +4,6 @@ name: Publish Documentation Generator to GitHub Pages on: push: branches: [main] - pull_request: - branches: [main] # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: @@ -24,7 +22,7 @@ jobs: distribution: 'temurin' - name: Copy Documentation Generator Schema to site - run: cp -Rv documentation-generator-api/src/main/resources/documentation-schema.json site/documentation-schema.json + run: cp -Rv documentation-generator-api/src/main/resources/documentation-schema.json site/v1/documentation-schema.json - name: Check GitHub Pages status uses: crazy-max/ghaction-github-status@v3 diff --git a/README.md b/README.md index 00e0b35..ed9b0ba 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,13 @@ The documentation file or files can be yaml- or json-files. If you in the beginning of your file add a reference to the schema, so you IDE can validate and have code completion. ```yaml -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json ``` Example of a file ```yaml -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json documentationTitle: My database schemaName: theSchema tables: @@ -121,7 +121,7 @@ tables: or as JSON: ```json { - "$schema": "https://patrickfust.github.io/documentation-generator/documentation-schema.json", + "$schema": "https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json", "documentationTitle": "My database", "schemaName": "theSchema", "tables": [ diff --git a/demos/demo-data-dictionary/documentation.yaml b/demos/demo-data-dictionary/documentation.yaml index 0f49e09..fd2bdf3 100644 --- a/demos/demo-data-dictionary/documentation.yaml +++ b/demos/demo-data-dictionary/documentation.yaml @@ -1,4 +1,4 @@ -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json documentationTitle: My database schemaName: theSchema dataDictionary: diff --git a/demos/demo-data-lineage/documentation.yaml b/demos/demo-data-lineage/documentation.yaml index ba7ee42..da1f106 100644 --- a/demos/demo-data-lineage/documentation.yaml +++ b/demos/demo-data-lineage/documentation.yaml @@ -1,4 +1,4 @@ -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json databaseName: This DB tables: - name: table_a diff --git a/demos/demo-data-lineage/external-documentation.yaml b/demos/demo-data-lineage/external-documentation.yaml index 4174ea2..10df19e 100644 --- a/demos/demo-data-lineage/external-documentation.yaml +++ b/demos/demo-data-lineage/external-documentation.yaml @@ -1,4 +1,4 @@ -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json documentationTitle: My external database databaseName: External DB name tables: diff --git a/demos/demo-erdiagram/documentation.yaml b/demos/demo-erdiagram/documentation.yaml index ec71137..8816d96 100644 --- a/demos/demo-erdiagram/documentation.yaml +++ b/demos/demo-erdiagram/documentation.yaml @@ -1,4 +1,4 @@ -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json documentationTitle: My database schemaName: theSchema tables: diff --git a/demos/demo-sqlscript/documentation.yaml b/demos/demo-sqlscript/documentation.yaml index 4814310..167d13d 100644 --- a/demos/demo-sqlscript/documentation.yaml +++ b/demos/demo-sqlscript/documentation.yaml @@ -1,4 +1,4 @@ -$schema: https://patrickfust.github.io/documentation-generator/documentation-schema.json +$schema: https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json documentationTitle: My database schemaName: theSchema tables: diff --git a/documentation-generator-core/src/test/resources/documentation-test.json b/documentation-generator-core/src/test/resources/documentation-test.json index f9d86b8..988549b 100644 --- a/documentation-generator-core/src/test/resources/documentation-test.json +++ b/documentation-generator-core/src/test/resources/documentation-test.json @@ -1,5 +1,5 @@ { - "$schema": "https://patrickfust.github.io/documentation-generator/documentation-schema.json", + "$schema": "https://patrickfust.github.io/documentation-generator/v1/documentation-schema.json", "documentationTitle": "The documentation title in a JSON file", "databaseName": "The database name", "schemaName": "xxx", diff --git a/site/index.html b/site/index.html index 8fd735f..c616d79 100755 --- a/site/index.html +++ b/site/index.html @@ -34,10 +34,16 @@

Documentation Generator

For usage and more information, go to:
GitHub Documentation Generator

-

- The documentation files must adhere to this JSON schema:
- Documentation Generator JSON schema -

+

The documentation files must adhere to the JSON schema

+
+ + + + + + + +
VersionLink to schema
1Documentation Generator JSON schema
From 859da0ec2fcbdf5408305cfdad4b869a5d9ba6e8 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Mon, 11 Nov 2024 14:03:23 +0100 Subject: [PATCH 09/12] Documentation --- .../java/dk/fust/docgen/model/ForeignKey.java | 3 ++ .../main/java/dk/fust/docgen/model/Index.java | 4 ++ .../main/java/dk/fust/docgen/model/View.java | 3 ++ .../main/resources/documentation-schema.json | 38 ++++++++++++---- .../README.md | 26 +++++++++++ .../README.md | 29 ++++++++++++ .../README.md | 32 +++++++++++++ .../README.md | 45 +++++++++++++++++++ 8 files changed, 171 insertions(+), 9 deletions(-) diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/ForeignKey.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/ForeignKey.java index e33bf0e..ef5bdd8 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/ForeignKey.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/ForeignKey.java @@ -8,7 +8,10 @@ @Data public class ForeignKey { + @Description(value = "Table name the foreign key points to. Is together with columnName", required = true) private String tableName; + + @Description(value = "Column name the foreign key points to. Is together with tableName", required = true) private String columnName; @Description(value = "Should the database enforce the reference?", hasDefaultBoolean = true, defaultBoolean = false) diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Index.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Index.java index 5b1e53c..ff97716 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Index.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Index.java @@ -10,8 +10,12 @@ @Data public class Index { + @Description("Name of the index") private String name; + private boolean unique = false; + + @Description(value = "List of fields for the index", required = true) private List fields; } diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/View.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/View.java index dfdb418..9acb05e 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/View.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/View.java @@ -8,7 +8,10 @@ @Data public class View { + @Description(value = "Name of the view") private String name; + + @Description(value = "SQL that defines the view", required = true) private String sql; } diff --git a/documentation-generator-api/src/main/resources/documentation-schema.json b/documentation-generator-api/src/main/resources/documentation-schema.json index 4a4a8c3..b7fba5a 100644 --- a/documentation-generator-api/src/main/resources/documentation-schema.json +++ b/documentation-generator-api/src/main/resources/documentation-schema.json @@ -101,6 +101,14 @@ "type" : "string", "description" : "Filename for the data dictionary" }, + "tags" : { + "description" : "List of tags that may be used in a filter", + "type" : "array", + "items" : { + "type" : "string", + "description" : "List of tags that may be used in a filter" + } + }, "version" : { "type" : "string", "description" : "Data dictionary version" @@ -168,7 +176,8 @@ "type" : "object", "properties" : { "columnName" : { - "type" : "string" + "type" : "string", + "description" : "Column name the foreign key points to. Is together with tableName" }, "enforceReference" : { "type" : "boolean", @@ -176,9 +185,11 @@ "default" : false }, "tableName" : { - "type" : "string" + "type" : "string", + "description" : "Table name the foreign key points to. Is together with columnName" } }, + "required" : [ "columnName", "tableName" ], "description" : "Is this a foreign key to another table" }, "name" : { @@ -218,27 +229,33 @@ "type" : "object", "properties" : { "fields" : { + "description" : "List of fields for the index", "type" : "array", "items" : { - "type" : "string" + "type" : "string", + "description" : "List of fields for the index" } }, "name" : { - "type" : "string" + "type" : "string", + "description" : "Name of the index" }, "unique" : { "type" : "boolean" } - } + }, + "required" : [ "fields" ] } }, "name" : { "type" : "string" }, "tags" : { + "description" : "List of tags that may be used in a filter", "type" : "array", "items" : { - "type" : "string" + "type" : "string", + "description" : "List of tags that may be used in a filter" } }, "views" : { @@ -247,12 +264,15 @@ "type" : "object", "properties" : { "name" : { - "type" : "string" + "type" : "string", + "description" : "Name of the view" }, "sql" : { - "type" : "string" + "type" : "string", + "description" : "SQL that defines the view" } - } + }, + "required" : [ "sql" ] } } } diff --git a/generators/documentation-generator-data-dictionary/README.md b/generators/documentation-generator-data-dictionary/README.md index 917ed2d..fd59cf0 100644 --- a/generators/documentation-generator-data-dictionary/README.md +++ b/generators/documentation-generator-data-dictionary/README.md @@ -15,6 +15,32 @@ To use the data dictionary generator, you must configure it with `dk.fust.docgen | addDescriptionForFile | boolean | Will add a header row containing the description of the file | false | | filter | String | If specified, only tables with this filter is used | | +## Model usage + +This generator uses these fields in [Documentation](../../documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java) + +``` +Documentation +└── dataDictionary + └── dataDictionaryFiles + ├── fileName + ├── fileDescription + ├── version + ├── columns + │ ├── columnName + │ ├── columnDescription + │ ├── dataType + │ ├── regex + │ ├── example + │ ├── mandatory + │ ├── minimumValue + │ ├── maximumValue + │ └── keys + └── tags +``` + +--- + ## Examples ### gradle.build diff --git a/generators/documentation-generator-datalineage/README.md b/generators/documentation-generator-datalineage/README.md index b5c7f13..543e9d3 100644 --- a/generators/documentation-generator-datalineage/README.md +++ b/generators/documentation-generator-datalineage/README.md @@ -15,6 +15,35 @@ To use the entity-relation diagram generator, you must configure it with `dk.fus | tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | | filter | String | If specified, only tables with this filter is used | | +## Model usage + +This generator uses these fields in [Documentation](../../documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java) + +``` +Documentation +├── databaseName +├── generation +│ ├── generateIdDataType +│ ├── generateId +│ ├── addCreatedAt +│ ├── columnNameCreatedAt +│ ├── addUpdatedAt +│ ├── columnNameUpdatedAt +│ └── triggerForUpdates +└── tables + ├── generation (same sub fields as above) + ├── name + ├── tags + └── fields + ├── name + ├── comment + ├── transformation + ├── source + └── dataType +``` + +--- + ## Examples ### gradle.build diff --git a/generators/documentation-generator-erdiagram/README.md b/generators/documentation-generator-erdiagram/README.md index 56a7682..4ef9216 100644 --- a/generators/documentation-generator-erdiagram/README.md +++ b/generators/documentation-generator-erdiagram/README.md @@ -28,6 +28,38 @@ so you can have several ER-diagram with a sub-set of the tables. Read more about `MarkdownDestination` [here](../../README.md#dkfustdocgendestinationmarkdowndestination-) +## Model usage + +This generator uses these fields in [Documentation](../../documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java) + +``` +Documentation +├── documentationTitle +├── schemaName +├── generation +│ ├── generateIdDataType +│ ├── generateId +│ ├── addCreatedAt +│ ├── columnNameCreatedAt +│ ├── addUpdatedAt +│ ├── columnNameUpdatedAt +│ └── triggerForUpdates +└── tables + ├── generation (same sub fields as above) + ├── name + ├── comment + ├── tags + └── fields + ├── name + ├── dataType + ├── foreignKey + │ ├── tableName + │ └── columnName + └── primaryKey +``` + +--- + ## Examples ### gradle.build diff --git a/generators/documentation-generator-sqlscript/README.md b/generators/documentation-generator-sqlscript/README.md index 62adc49..ed44907 100644 --- a/generators/documentation-generator-sqlscript/README.md +++ b/generators/documentation-generator-sqlscript/README.md @@ -12,6 +12,51 @@ To use the sql generator, you must configure it with `dk.fust.docgen.sqlscript.S | destination | Destination | Where to send the generated documentation | | | filter | String | If specified, only tables with this filter is used | | +## Model usage + +This generator uses these fields in [Documentation](../../documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java) + +``` +Documentation +├── schemaName +├── generation +│ ├── generateIdDataType +│ ├── generateId +│ ├── addCreatedAt +│ ├── columnNameCreatedAt +│ ├── addUpdatedAt +│ ├── columnNameUpdatedAt +│ └── triggerForUpdates +└── tables + ├── generation (same sub fields as above) + ├── name + ├── comment + ├── tags + ├── createTableScript + ├── fields + │ ├── name + │ ├── comment + │ ├── dataType + │ ├── foreignKey + │ │ ├── tableName + │ │ ├── columnName + │ │ └── enforceReference + │ ├── primaryKey + │ ├── unique + │ ├── nullable + │ ├── defaultValue + │ └── check + ├── views + │ ├── name + │ └── sql + └── indexes + ├── name + ├── unique + └── fields +``` + +--- + ## Examples ### gradle.build From 30dc69ba6b7a038f574e0d62a57f3679cad4a189 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Mon, 11 Nov 2024 14:40:41 +0100 Subject: [PATCH 10/12] Explicit that it's currently only generating sql scripts for Postgres --- .../README.md | 11 +- .../sqlscript/SqlScriptConfiguration.java | 6 + .../docgen/sqlscript/SqlScriptGenerator.java | 198 +--------------- .../generators/PostgresGenerator.java | 213 ++++++++++++++++++ .../sqlscript/generators/SqlDialect.java | 13 ++ .../sqlscript/generators/SqlGenerator.java | 12 + .../generators/SqlGeneratorFactory.java | 14 ++ 7 files changed, 268 insertions(+), 199 deletions(-) create mode 100644 generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java create mode 100644 generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlDialect.java create mode 100644 generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGenerator.java create mode 100644 generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGeneratorFactory.java diff --git a/generators/documentation-generator-sqlscript/README.md b/generators/documentation-generator-sqlscript/README.md index ed44907..21b67d7 100644 --- a/generators/documentation-generator-sqlscript/README.md +++ b/generators/documentation-generator-sqlscript/README.md @@ -6,11 +6,12 @@ This generator can generate sql files containing create tables, indexes and view To use the sql generator, you must configure it with `dk.fust.docgen.sqlscript.SqlScriptConfiguration` -| Setting | Type | Description | Default | -|-------------------|-------------|----------------------------------------------------|-----------| -| documentationFile | File | Location of the documentation file | | -| destination | Destination | Where to send the generated documentation | | -| filter | String | If specified, only tables with this filter is used | | +| Setting | Type | Description | Default | +|-------------------|-------------|----------------------------------------------------|----------| +| documentationFile | File | Location of the documentation file | | +| destination | Destination | Where to send the generated documentation | | +| filter | String | If specified, only tables with this filter is used | | +| sqlDialect | SqlDialect | SQL Dialect that the sql scripts are generated in | POSTGRES | ## Model usage diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java index 14e78ca..f79e769 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java @@ -3,6 +3,7 @@ import dk.fust.docgen.Generator; import dk.fust.docgen.GeneratorConfiguration; import dk.fust.docgen.destination.Destination; +import dk.fust.docgen.sqlscript.generators.SqlDialect; import dk.fust.docgen.util.Assert; import lombok.Data; @@ -20,6 +21,11 @@ public class SqlScriptConfiguration implements GeneratorConfiguration { private String filter; + /** + * Which dialect must the sql be? Default: POSTGRES + */ + private SqlDialect sqlDialect = SqlDialect.POSTGRES; + @Override public void validate() { Assert.isNotNull(documentationFile, "documentationFile must not be null"); diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java index be214f6..4578083 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptGenerator.java @@ -2,21 +2,14 @@ import dk.fust.docgen.Generator; import dk.fust.docgen.GeneratorConfiguration; -import dk.fust.docgen.model.DataType; import dk.fust.docgen.model.Documentation; -import dk.fust.docgen.model.Field; -import dk.fust.docgen.model.Generation; -import dk.fust.docgen.model.Index; -import dk.fust.docgen.model.Table; -import dk.fust.docgen.model.View; +import dk.fust.docgen.sqlscript.generators.SqlGenerator; +import dk.fust.docgen.sqlscript.generators.SqlGeneratorFactory; import dk.fust.docgen.util.Assert; import lombok.Data; import lombok.extern.slf4j.Slf4j; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; /** * Generator for SQL files @@ -30,191 +23,8 @@ public void generate(Documentation documentation, GeneratorConfiguration generat log.info("Generating SQL-scripts..."); Assert.isTrue(generatorConfiguration instanceof SqlScriptConfiguration, "configuration must be an instance of SqlScriptConfiguration"); SqlScriptConfiguration sqlScriptConfiguration = (SqlScriptConfiguration) generatorConfiguration; - String schemaName = documentation.getSchemaName(); - for (Table table : documentation.filterTables(sqlScriptConfiguration.getFilter())) { - Generation generationForTable = documentation.getGenerationForTable(table); - if (table.getCreateTableScript() != null) { - StringBuilder sqlScript = new StringBuilder(256); - - sqlScript.append(generateTable(schemaName, table, generationForTable)); - sqlScript.append(describeTable(schemaName, table, generationForTable)); - sqlScript.append(generateViews(schemaName, table)); - sqlScript.append(generateIndexes(schemaName, table)); - sqlScript.append(generateTriggers(schemaName, table, generationForTable)); - - generatorConfiguration.getDestination() - .sendDocumentToDestination(sqlScript.toString(), table.getCreateTableScript()); - } - } - } - - private String generateTable(String schemaName, Table table, Generation generationForTable) { - boolean hasColumn = false; - StringBuilder sql = new StringBuilder(128); - sql.append("create table "); - appendName(schemaName, table.getName(), sql).append(" (\n"); - - if (generationForTable.hasGenerateId() && generationForTable.getGenerateId()) { - sql.append(" %s_id %s primary key generated always as identity".formatted(table.getName(), generationForTable.getGenerateIdDataType().toLowerCase())); - hasColumn = true; - } - if (generationForTable.getAddCreatedAt() != null && generationForTable.getAddCreatedAt()) { - if (hasColumn) { - sql.append(",\n"); - } else { - hasColumn = true; - } - sql.append(" ").append(generationForTable.getColumnNameCreatedAt()).append(" timestamp with time zone not null default now()"); - } - if (generationForTable.getAddUpdatedAt() != null && generationForTable.getAddUpdatedAt()) { - if (hasColumn) { - sql.append(",\n"); - } else { - hasColumn = true; - } - sql.append(" ").append(generationForTable.getColumnNameUpdatedAt()).append(" timestamp with time zone not null default now()"); - } - generateTableFields(table, hasColumn, sql); - sql.append("\n);\n"); - return sql.toString(); - } - - private void generateTableFields(Table table, boolean hasColumn, StringBuilder sql) { - List primaryKeys = new ArrayList<>(); - for (Field field : table.getFields()) { - if (hasColumn) { - sql.append(",\n"); - } else { - hasColumn = true; - } - sql.append(" %s %s".formatted(field.getName(), field.getDataType().toLowerCase())); - if (field.isUnique()) { - sql.append(" unique"); - } - if (!field.isNullable()) { - sql.append(" not null"); - } - if (field.isPrimaryKey()) { - primaryKeys.add(field); - } - if (field.getForeignKey() != null && field.getForeignKey().isEnforceReference()) { - sql.append(" references %s(%s)".formatted( - field.getForeignKey().getTableName(), field.getForeignKey().getColumnName())); - } - if (field.getDefaultValue() != null && !field.getDefaultValue().isEmpty()) { - sql.append(" default '%s'".formatted(field.getDefaultValue())); - } - sql.append(generateCheckConstraint(field)); - } - if (!primaryKeys.isEmpty()) { - sql.append(",\n primary key("); - String keys = primaryKeys.stream().map(Field::getName).collect(Collectors.joining(", ")); - sql.append(keys).append(")"); - } - } - - private String generateCheckConstraint(Field field) { - if (field.getCheck() == null || field.getCheck().isEmpty()) { - return ""; - } - StringBuilder sql = new StringBuilder(" check (%s in (".formatted(field.getName())); - if (field.getDataType().equals(DataType.TEXT)) { - sql.append(field.getCheck().stream().collect(Collectors.joining("', '", "'", "'"))); - } else { - sql.append(String.join(", ", field.getCheck())); - } - sql.append("))"); - return sql.toString(); + SqlGenerator sqlGenerator = SqlGeneratorFactory.getSqlGenerator(sqlScriptConfiguration.getSqlDialect()); + sqlGenerator.generate(documentation, sqlScriptConfiguration); } - private String describeTable(String schemaName, Table table, Generation generationForTable) { - StringBuilder sql = new StringBuilder(128).append("\n"); - if (table.getComment() != null) { - sql.append("comment on table "); - appendName(schemaName, table.getName(), sql); - sql.append(" is '%s';\n".formatted(table.getComment())); - } - if (generationForTable.getAddCreatedAt() != null && generationForTable.getAddCreatedAt()) { - sql.append("comment on column "); - appendName(schemaName, table.getName(), sql).append(".%s".formatted(generationForTable.getColumnNameCreatedAt())).append(" is 'timestamp when row is inserted';\n"); - } - if (generationForTable.getAddUpdatedAt() != null && generationForTable.getAddUpdatedAt()) { - sql.append("comment on column "); - appendName(schemaName, table.getName(), sql).append(".%s".formatted(generationForTable.getColumnNameUpdatedAt())).append(" is 'timestamp when row is last updated';\n"); - } - table.getFields().forEach(field -> { - if (field.getComment() != null) { - sql.append("comment on column "); - appendName(schemaName, table.getName(), sql).append(".%s is '".formatted(field.getName())); - sql.append(field.getComment().replaceAll("'", "''")); - sql.append("';\n"); - } - }); - if (sql.length() > 3) { - return sql.toString(); - } - // Don't have anything - return ""; - } - - private String generateViews(String schemaName, Table table) { - StringBuilder sql = new StringBuilder(128); - if (table.getViews() != null && !table.getViews().isEmpty()) { - for (View views : table.getViews()) { - sql.append("\ndrop view if exists "); - appendName(schemaName, views.getName(), sql).append(";\n"); - sql.append("create view "); - appendName(schemaName, views.getName(), sql).append(" as \n "); - sql.append(views.getSql()); - sql.append(";\n"); - } - } - return sql.toString(); - } - - private String generateIndexes(String schemaName, Table table) { - StringBuilder sql = new StringBuilder(128); - if (table.getIndexes() != null && !table.getIndexes().isEmpty()) { - for (Index index : table.getIndexes()) { - sql.append("\ncreate"); - if (index.isUnique()) { - sql.append(" unique"); - } - sql.append(" index %s on ".formatted(index.getName())); - appendName(schemaName, table.getName(), sql); - sql.append("("); - boolean first = true; - for (String field : index.getFields()) { - if (!first) { - sql.append(", "); - } else { - first = false; - } - sql.append(field); - } - sql.append(");\n"); - } - } - return sql.toString(); - } - - private String generateTriggers(String schemaName, Table table, Generation generationForTable) { - StringBuilder sql = new StringBuilder(128); - if (generationForTable.getTriggerForUpdates() != null && !generationForTable.getTriggerForUpdates().isEmpty()) { - sql.append(("\n" + - "create trigger %s_update_time\n before update\n on ").formatted(table.getName())); - appendName(schemaName, table.getName(), sql).append("\n"); - sql.append(" for each row execute function "); - appendName(schemaName, generationForTable.getTriggerForUpdates(), sql).append("();\n"); - } - return sql.toString(); - } - - private StringBuilder appendName(String schemaName, String fieldName, StringBuilder stringBuilder) { - if (schemaName != null) { - stringBuilder.append(schemaName).append("."); - } - stringBuilder.append(fieldName); - return stringBuilder; - } } diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java new file mode 100644 index 0000000..8ceab28 --- /dev/null +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java @@ -0,0 +1,213 @@ +package dk.fust.docgen.sqlscript.generators; + +import dk.fust.docgen.model.DataType; +import dk.fust.docgen.model.Documentation; +import dk.fust.docgen.model.Field; +import dk.fust.docgen.model.Generation; +import dk.fust.docgen.model.Index; +import dk.fust.docgen.model.Table; +import dk.fust.docgen.model.View; +import dk.fust.docgen.sqlscript.SqlScriptConfiguration; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class PostgresGenerator implements SqlGenerator { + + @Override + public void generate(Documentation documentation, SqlScriptConfiguration sqlScriptConfiguration) throws IOException { + log.debug("Generating Postgres SQL-scripts..."); + String schemaName = documentation.getSchemaName(); + for (Table table : documentation.filterTables(sqlScriptConfiguration.getFilter())) { + Generation generationForTable = documentation.getGenerationForTable(table); + if (table.getCreateTableScript() != null) { + StringBuilder sqlScript = new StringBuilder(256); + + sqlScript.append(generateTable(schemaName, table, generationForTable)); + sqlScript.append(describeTable(schemaName, table, generationForTable)); + sqlScript.append(generateViews(schemaName, table)); + sqlScript.append(generateIndexes(schemaName, table)); + sqlScript.append(generateTriggers(schemaName, table, generationForTable)); + + sqlScriptConfiguration.getDestination() + .sendDocumentToDestination(sqlScript.toString(), table.getCreateTableScript()); + } + } + } + + @SuppressWarnings("ReassignedVariable") + private String generateTable(String schemaName, Table table, Generation generationForTable) { + boolean hasColumn = false; + StringBuilder sql = new StringBuilder(128); + sql.append("create table "); + appendName(schemaName, table.getName(), sql).append(" (\n"); + + if (generationForTable.hasGenerateId() && generationForTable.getGenerateId()) { + sql.append(" %s_id %s primary key generated always as identity".formatted(table.getName(), generationForTable.getGenerateIdDataType().toLowerCase())); + hasColumn = true; + } + if (generationForTable.getAddCreatedAt() != null && generationForTable.getAddCreatedAt()) { + if (hasColumn) { + sql.append(",\n"); + } else { + hasColumn = true; + } + sql.append(" ").append(generationForTable.getColumnNameCreatedAt()).append(" timestamp with time zone not null default now()"); + } + if (generationForTable.getAddUpdatedAt() != null && generationForTable.getAddUpdatedAt()) { + if (hasColumn) { + sql.append(",\n"); + } else { + hasColumn = true; + } + sql.append(" ").append(generationForTable.getColumnNameUpdatedAt()).append(" timestamp with time zone not null default now()"); + } + generateTableFields(table, hasColumn, sql); + sql.append("\n);\n"); + return sql.toString(); + } + + private void generateTableFields(Table table, boolean hasColumn, StringBuilder sql) { + List primaryKeys = new ArrayList<>(); + for (Field field : table.getFields()) { + if (hasColumn) { + sql.append(",\n"); + } else { + hasColumn = true; + } + sql.append(" %s %s".formatted(field.getName(), field.getDataType().toLowerCase())); + if (field.isUnique()) { + sql.append(" unique"); + } + if (!field.isNullable()) { + sql.append(" not null"); + } + if (field.isPrimaryKey()) { + primaryKeys.add(field); + } + if (field.getForeignKey() != null && field.getForeignKey().isEnforceReference()) { + sql.append(" references %s(%s)".formatted( + field.getForeignKey().getTableName(), field.getForeignKey().getColumnName())); + } + if (field.getDefaultValue() != null && !field.getDefaultValue().isEmpty()) { + sql.append(" default '%s'".formatted(field.getDefaultValue())); + } + sql.append(generateCheckConstraint(field)); + } + if (!primaryKeys.isEmpty()) { + sql.append(",\n primary key("); + String keys = primaryKeys.stream().map(Field::getName).collect(Collectors.joining(", ")); + sql.append(keys).append(")"); + } + } + + private String generateCheckConstraint(Field field) { + if (field.getCheck() == null || field.getCheck().isEmpty()) { + return ""; + } + StringBuilder sql = new StringBuilder(" check (%s in (".formatted(field.getName())); + if (field.getDataType().equals(DataType.TEXT)) { + sql.append(field.getCheck().stream().collect(Collectors.joining("', '", "'", "'"))); + } else { + sql.append(String.join(", ", field.getCheck())); + } + sql.append("))"); + return sql.toString(); + } + + private String describeTable(String schemaName, Table table, Generation generationForTable) { + StringBuilder sql = new StringBuilder(128).append("\n"); + if (table.getComment() != null) { + sql.append("comment on table "); + appendName(schemaName, table.getName(), sql); + sql.append(" is '%s';\n".formatted(table.getComment())); + } + if (generationForTable.getAddCreatedAt() != null && generationForTable.getAddCreatedAt()) { + sql.append("comment on column "); + appendName(schemaName, table.getName(), sql).append(".%s".formatted(generationForTable.getColumnNameCreatedAt())).append(" is 'timestamp when row is inserted';\n"); + } + if (generationForTable.getAddUpdatedAt() != null && generationForTable.getAddUpdatedAt()) { + sql.append("comment on column "); + appendName(schemaName, table.getName(), sql).append(".%s".formatted(generationForTable.getColumnNameUpdatedAt())).append(" is 'timestamp when row is last updated';\n"); + } + table.getFields().forEach(field -> { + if (field.getComment() != null) { + sql.append("comment on column "); + appendName(schemaName, table.getName(), sql).append(".%s is '".formatted(field.getName())); + sql.append(field.getComment().replaceAll("'", "''")); + sql.append("';\n"); + } + }); + if (sql.length() > 3) { + return sql.toString(); + } + // Don't have anything + return ""; + } + + private String generateViews(String schemaName, Table table) { + StringBuilder sql = new StringBuilder(128); + if (table.getViews() != null && !table.getViews().isEmpty()) { + for (View views : table.getViews()) { + sql.append("\ndrop view if exists "); + appendName(schemaName, views.getName(), sql).append(";\n"); + sql.append("create view "); + appendName(schemaName, views.getName(), sql).append(" as \n "); + sql.append(views.getSql()); + sql.append(";\n"); + } + } + return sql.toString(); + } + + private String generateIndexes(String schemaName, Table table) { + StringBuilder sql = new StringBuilder(128); + if (table.getIndexes() != null && !table.getIndexes().isEmpty()) { + for (Index index : table.getIndexes()) { + sql.append("\ncreate"); + if (index.isUnique()) { + sql.append(" unique"); + } + sql.append(" index %s on ".formatted(index.getName())); + appendName(schemaName, table.getName(), sql); + sql.append("("); + boolean first = true; + for (String field : index.getFields()) { + if (!first) { + sql.append(", "); + } else { + first = false; + } + sql.append(field); + } + sql.append(");\n"); + } + } + return sql.toString(); + } + + private String generateTriggers(String schemaName, Table table, Generation generationForTable) { + StringBuilder sql = new StringBuilder(128); + if (generationForTable.getTriggerForUpdates() != null && !generationForTable.getTriggerForUpdates().isEmpty()) { + sql.append(("\n" + + "create trigger %s_update_time\n before update\n on ").formatted(table.getName())); + appendName(schemaName, table.getName(), sql).append("\n"); + sql.append(" for each row execute function "); + appendName(schemaName, generationForTable.getTriggerForUpdates(), sql).append("();\n"); + } + return sql.toString(); + } + + private StringBuilder appendName(String schemaName, String fieldName, StringBuilder stringBuilder) { + if (schemaName != null) { + stringBuilder.append(schemaName).append("."); + } + stringBuilder.append(fieldName); + return stringBuilder; + } + +} diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlDialect.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlDialect.java new file mode 100644 index 0000000..9386f43 --- /dev/null +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlDialect.java @@ -0,0 +1,13 @@ +package dk.fust.docgen.sqlscript.generators; + +/** + * SQL Dialect that the sql scripts are generated in + */ +public enum SqlDialect { + + /** + * Postgres dialect + */ + POSTGRES + +} diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGenerator.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGenerator.java new file mode 100644 index 0000000..7d349b4 --- /dev/null +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGenerator.java @@ -0,0 +1,12 @@ +package dk.fust.docgen.sqlscript.generators; + +import dk.fust.docgen.model.Documentation; +import dk.fust.docgen.sqlscript.SqlScriptConfiguration; + +import java.io.IOException; + +public interface SqlGenerator { + + void generate(Documentation documentation, SqlScriptConfiguration sqlScriptConfiguration) throws IOException; + +} diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGeneratorFactory.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGeneratorFactory.java new file mode 100644 index 0000000..e77b1b0 --- /dev/null +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/SqlGeneratorFactory.java @@ -0,0 +1,14 @@ +package dk.fust.docgen.sqlscript.generators; + +/** + * Factory to choose the correct generator from + */ +public class SqlGeneratorFactory { + + public static SqlGenerator getSqlGenerator(SqlDialect sqlDialect) { + return switch (sqlDialect) { + case POSTGRES -> new PostgresGenerator(); + }; + + } +} From 0d7c1122ca1cbf1903e240c4b81a31b71797cd38 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Thu, 14 Nov 2024 13:42:15 +0100 Subject: [PATCH 11/12] Enum for choosing csv delimiter --- .../documentation-generator-csv/README.md | 6 +++--- .../docgen/csv/format/table/CSVDelimiter.java | 21 +++++++++++++++++++ .../csv/format/table/CSVTableFormatter.java | 4 ++-- 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVDelimiter.java diff --git a/extensions/documentation-generator-csv/README.md b/extensions/documentation-generator-csv/README.md index 1fec879..6422257 100644 --- a/extensions/documentation-generator-csv/README.md +++ b/extensions/documentation-generator-csv/README.md @@ -8,6 +8,6 @@ Class name: `dk.fust.docgen.csv.format.table.CSVTableFormatter` Formats tables as CSV files. -| Setting | Type | Description | Default | -|-----------|--------|---------------------------------------|---------| -| delimeter | String | Which delimeter to use between fields | ; | +| Setting | Type | Description | Default | +|-----------|--------------|---------------------------------------|-----------| +| delimiter | CSVDelimiter | Which delimiter to use between fields | SEMICOLON | diff --git a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVDelimiter.java b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVDelimiter.java new file mode 100644 index 0000000..f4bcb4c --- /dev/null +++ b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVDelimiter.java @@ -0,0 +1,21 @@ +package dk.fust.docgen.csv.format.table; + +public enum CSVDelimiter { + + COMMA(","), + SEMICOLON(";"), + PIPE("|"), + SPACE(" "), + TAB("\t"); + + CSVDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + private final String delimiter; + + public String getDelimiter() { + return delimiter; + } + +} diff --git a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java index 6db6b52..23a47c6 100644 --- a/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java +++ b/extensions/documentation-generator-csv/src/main/java/dk/fust/docgen/csv/format/table/CSVTableFormatter.java @@ -18,14 +18,14 @@ @Data public class CSVTableFormatter implements TableFormatter { - private String delimiter = ";"; + private CSVDelimiter delimiter = CSVDelimiter.SEMICOLON; @Override public String formatTable(FormatTable formatTable) { StringWriter sw = new StringWriter(); CSVFormat csvFormat = CSVFormat.DEFAULT.builder() - .setDelimiter(delimiter) + .setDelimiter(delimiter.getDelimiter()) .build(); try (final CSVPrinter printer = new CSVPrinter(sw, csvFormat)) { From 7c7bb6982c31fd56eef7d5db16c8dd8d8537ad50 Mon Sep 17 00:00:00 2001 From: Patrick Fust Date: Thu, 14 Nov 2024 14:06:12 +0100 Subject: [PATCH 12/12] Renamed filter to filterTags for clarification --- README.md | 2 +- demos/demo-erdiagram/build.gradle | 4 ++-- demos/demo-erdiagram/generator-configuration.yml | 2 +- .../java/dk/fust/docgen/model/Documentation.java | 16 ++++++++-------- .../README.md | 2 +- .../datadict/DataDictionaryConfiguration.java | 2 +- .../docgen/datadict/DataDictionaryGenerator.java | 2 +- .../README.md | 2 +- .../datalineage/DataLineageConfiguration.java | 2 +- .../docgen/datalineage/DataLineageGenerator.java | 2 +- .../generator-configuration-with-htmltable.yaml | 2 +- .../documentation-generator-erdiagram/README.md | 6 +++--- .../docgen/erdiagram/ERDiagramGenerator.java | 2 +- .../dk/fust/docgen/erdiagram/GenerateKey.java | 2 +- .../docgen/erdiagram/generators/ERGenerator.java | 4 ++-- .../erdiagram/generators/MermaidGenerator.java | 8 ++++---- .../erdiagram/generators/PlantUMLGenerator.java | 4 ++-- .../erdiagram/ERDiagramGeneratorSpec.groovy | 2 +- .../documentation-generator-sqlscript/README.md | 2 +- .../docgen/sqlscript/SqlScriptConfiguration.java | 2 +- .../sqlscript/generators/PostgresGenerator.java | 2 +- 21 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index ed9b0ba..22fdda4 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ You can have nested objects, you just have to specify which class name it is. destinationKey: MODEL_MERMAID_PLACEHOLDER - className: dk.fust.docgen.erdiagram.GenerateKey destinationKey: MODEL_MERMAID_GROUP_PLACEHOLDER - filter: my_group + filterTags: my_group destination: className: dk.fust.docgen.destination.MarkdownDestination file: README.md diff --git a/demos/demo-erdiagram/build.gradle b/demos/demo-erdiagram/build.gradle index 2d4769e..21fe07b 100644 --- a/demos/demo-erdiagram/build.gradle +++ b/demos/demo-erdiagram/build.gradle @@ -34,9 +34,9 @@ documentationGenerator { // documentationFile: new File(projectDir, 'documentation.yaml'), // umlGenerator : UMLGenerator.MERMAID, // Can be omitted because it's default // generateKeys: [ -// // Empty filter means all groups +// // Empty filterTags means all groups // new GenerateKey(destinationKey: 'MODEL_MERMAID_PLACEHOLDER'), -// new GenerateKey(destinationKey: 'MODEL_MERMAID_GROUP_PLACEHOLDER', filter: 'my_group') +// new GenerateKey(destinationKey: 'MODEL_MERMAID_GROUP_PLACEHOLDER', filterTags: 'my_group') // ], // destination: new MarkdownDestination( // file: new File('README.md'), diff --git a/demos/demo-erdiagram/generator-configuration.yml b/demos/demo-erdiagram/generator-configuration.yml index 3e7757a..e88ee04 100644 --- a/demos/demo-erdiagram/generator-configuration.yml +++ b/demos/demo-erdiagram/generator-configuration.yml @@ -6,7 +6,7 @@ destinationKey: MODEL_MERMAID_PLACEHOLDER - className: dk.fust.docgen.erdiagram.GenerateKey destinationKey: MODEL_MERMAID_GROUP_PLACEHOLDER - filter: my_group + filterTags: my_group destination: className: dk.fust.docgen.destination.MarkdownDestination file: README.md diff --git a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java index a3b5d9d..592aa69 100644 --- a/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java +++ b/documentation-generator-api/src/main/java/dk/fust/docgen/model/Documentation.java @@ -99,26 +99,26 @@ private void mergeGenerations(Generation from, Generation to) { /** * Returns those tables that have a tag that is equal to the filter - * @param filter filter to search for + * @param filterTags filter to search for * @return only matching tables */ - public List filterTables(String filter) { - if (filter != null && !filter.isEmpty()) { + public List
filterTables(String filterTags) { + if (filterTags != null && !filterTags.isEmpty()) { // Only those with the filer - return tables.stream().filter(t -> t.getTags() != null && t.getTags().contains(filter)).toList(); + return tables.stream().filter(t -> t.getTags() != null && t.getTags().contains(filterTags)).toList(); } return tables; } /** * Returns those data dictionaries that have a tag that i equal to the filter - * @param filter filter to search for + * @param filterTags filter to search for * @return only matching data dictionaries */ - public List filterDataDictionaryFiles(String filter) { - if (filter != null && !filter.isEmpty() && dataDictionary.getDataDictionaryFiles() != null) { + public List filterDataDictionaryFiles(String filterTags) { + if (filterTags != null && !filterTags.isEmpty() && dataDictionary.getDataDictionaryFiles() != null) { // Only those with the filer - return dataDictionary.getDataDictionaryFiles().stream().filter(t -> t.getTags() != null && t.getTags().contains(filter)).toList(); + return dataDictionary.getDataDictionaryFiles().stream().filter(t -> t.getTags() != null && t.getTags().contains(filterTags)).toList(); } return dataDictionary.getDataDictionaryFiles(); } diff --git a/generators/documentation-generator-data-dictionary/README.md b/generators/documentation-generator-data-dictionary/README.md index fd59cf0..0c138b3 100644 --- a/generators/documentation-generator-data-dictionary/README.md +++ b/generators/documentation-generator-data-dictionary/README.md @@ -13,7 +13,7 @@ To use the data dictionary generator, you must configure it with `dk.fust.docgen | key | String | Key to identify it on the destination | | | tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | | addDescriptionForFile | boolean | Will add a header row containing the description of the file | false | -| filter | String | If specified, only tables with this filter is used | | +| filterTags | String | If specified, only tables with this filter is used | | ## Model usage diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java index ac6d17e..d10bc87 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryConfiguration.java @@ -22,7 +22,7 @@ public class DataDictionaryConfiguration implements GeneratorConfiguration { private String key; - private String filter; + private String filterTags; private TableFormatter tableFormatter = new MarkdownTableFormatter(); diff --git a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java index 54d4652..dd504c2 100644 --- a/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java +++ b/generators/documentation-generator-data-dictionary/src/main/java/dk/fust/docgen/datadict/DataDictionaryGenerator.java @@ -35,7 +35,7 @@ public void generate(Documentation documentation, GeneratorConfiguration generat private FormatTable generateTable(Documentation documentation, DataDictionaryConfiguration dataDictionaryConfiguration) { FormatTable formatTable = new FormatTable(); formatTable.getRows().add(createHeaderRow()); - List dataDictionaryFiles = documentation.filterDataDictionaryFiles(dataDictionaryConfiguration.getFilter()); + List dataDictionaryFiles = documentation.filterDataDictionaryFiles(dataDictionaryConfiguration.getFilterTags()); for (DataDictionaryFile dataDictionaryFile : dataDictionaryFiles) { if (dataDictionaryConfiguration.isAddDescriptionForFile()) { Row row = new Row(); diff --git a/generators/documentation-generator-datalineage/README.md b/generators/documentation-generator-datalineage/README.md index 543e9d3..c5b7642 100644 --- a/generators/documentation-generator-datalineage/README.md +++ b/generators/documentation-generator-datalineage/README.md @@ -13,7 +13,7 @@ To use the entity-relation diagram generator, you must configure it with `dk.fus | key | String | Key to identify it on the destination | | | sourceDocumentationFiles | Map | External references to other documentationfiles. Is used when specifying `source` | | | tableFormatter | TableFormatter | Formatter that can create the table | `MarkdownTableFormatter` | -| filter | String | If specified, only tables with this filter is used | | +| filterTags | String | If specified, only tables with this filter is used | | ## Model usage diff --git a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java index 8d5607a..3103643 100644 --- a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java +++ b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageConfiguration.java @@ -31,7 +31,7 @@ public class DataLineageConfiguration implements GeneratorConfiguration { private TableFormatter tableFormatter = new MarkdownTableFormatter(); - private String filter; + private String filterTags; @Override public void validate() { diff --git a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageGenerator.java b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageGenerator.java index 40071d8..07ef905 100644 --- a/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageGenerator.java +++ b/generators/documentation-generator-datalineage/src/main/java/dk/fust/docgen/datalineage/DataLineageGenerator.java @@ -44,7 +44,7 @@ public void generate(Documentation documentation, GeneratorConfiguration generat rows.add(getHeaderRow()); rows.add(getSubHeaderRow()); - rows.addAll(getRows(documentation.filterTables(conf.getFilter()), documentation, conf)); + rows.addAll(getRows(documentation.filterTables(conf.getFilterTags()), documentation, conf)); String document = conf.getTableFormatter().formatTable(formatTable); generatorConfiguration.getDestination().sendDocumentToDestination(document, conf.getKey()); diff --git a/generators/documentation-generator-datalineage/src/test/resources/generator-configuration-with-htmltable.yaml b/generators/documentation-generator-datalineage/src/test/resources/generator-configuration-with-htmltable.yaml index 40769ba..d869bcb 100644 --- a/generators/documentation-generator-datalineage/src/test/resources/generator-configuration-with-htmltable.yaml +++ b/generators/documentation-generator-datalineage/src/test/resources/generator-configuration-with-htmltable.yaml @@ -1,7 +1,7 @@ - className: dk.fust.docgen.datalineage.DataLineageConfiguration documentationFile: documentation.yml key: data-lineage - filter: data-lineage + filterTags: data-lineage tableFormatter: className: dk.fust.docgen.format.table.HTMLTableFormatter dataFields: diff --git a/generators/documentation-generator-erdiagram/README.md b/generators/documentation-generator-erdiagram/README.md index 4ef9216..025a4e0 100644 --- a/generators/documentation-generator-erdiagram/README.md +++ b/generators/documentation-generator-erdiagram/README.md @@ -19,7 +19,7 @@ To use the entity-relation diagram generator, you must configure it with `dk.fus You can group them for instance into domain-model and external-reference and so forth, by adding a filter. Only tables with a tag matching the filter will be used. -If the `filter` is `null`, then all tables are selected. +If the `filterTags` is `null`, then all tables are selected. The `destinationKey` tells which part of the destination that needs replacing. @@ -76,7 +76,7 @@ new ERDiagramConfiguration( generateKeys: [ // Empty filter means all groups new GenerateKey(destinationKey: 'MODEL_MERMAID_PLACEHOLDER'), - new GenerateKey(destinationKey: 'MODEL_MERMAID_GROUP_PLACEHOLDER', filter: 'my_group') + new GenerateKey(destinationKey: 'MODEL_MERMAID_GROUP_PLACEHOLDER', filterTags: 'my_group') ], destination: new MarkdownDestination( file: new File('README.md'), @@ -95,7 +95,7 @@ new ERDiagramConfiguration( destinationKey: MODEL_MERMAID_PLACEHOLDER - className: dk.fust.docgen.erdiagram.GenerateKey destinationKey: MODEL_MERMAID_GROUP_PLACEHOLDER - filter: my_group + filterTags: my_group destination: className: dk.fust.docgen.destination.MarkdownDestination file: README.md diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramGenerator.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramGenerator.java index 9e15c1a..1292c8c 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramGenerator.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/ERDiagramGenerator.java @@ -24,7 +24,7 @@ public void generate(Documentation documentation, GeneratorConfiguration generat erDiagramConfiguration.getDestination().validate(); ERGenerator generator = ERGeneratorFactory.getGenerator(erDiagramConfiguration.getUmlGenerator()); for (GenerateKey generateKey : erDiagramConfiguration.getGenerateKeys()) { - String uml = generator.generateUML(generateKey.getFilter(), documentation, generatorConfiguration); + String uml = generator.generateUML(generateKey.getFilterTags(), documentation, generatorConfiguration); String document = """ ```%s %s diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/GenerateKey.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/GenerateKey.java index 8864192..a90b5f6 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/GenerateKey.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/GenerateKey.java @@ -8,7 +8,7 @@ @Data public class GenerateKey { - private String filter; + private String filterTags; private String destinationKey; } diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/ERGenerator.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/ERGenerator.java index aa545b9..a5b09e9 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/ERGenerator.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/ERGenerator.java @@ -16,11 +16,11 @@ public interface ERGenerator { /** * Go generate er-diagram please - * @param filter filter for which groups to render. It finds the tables that have a matching tag. If null or empty, all groups are rendered. + * @param filterTags filter for which groups to render. It finds the tables that have a matching tag. If null or empty, all groups are rendered. * @param documentation model containing the tables to render * @param generatorConfiguration configuration for how to render * @return markdown markup */ - String generateUML(String filter, Documentation documentation, GeneratorConfiguration generatorConfiguration); + String generateUML(String filterTags, Documentation documentation, GeneratorConfiguration generatorConfiguration); } diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/MermaidGenerator.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/MermaidGenerator.java index 4c0dced..fc460f8 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/MermaidGenerator.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/MermaidGenerator.java @@ -22,7 +22,7 @@ public String getMarkdownType() { } @Override - public String generateUML(String filter, Documentation documentation, GeneratorConfiguration generatorConfiguration) { + public String generateUML(String filterTags, Documentation documentation, GeneratorConfiguration generatorConfiguration) { StringBuilder uml = new StringBuilder(INITIAL_CAPACITY); if (documentation.getDocumentationTitle() != null && !documentation.getDocumentationTitle().isEmpty()) { uml.append(""" @@ -32,15 +32,15 @@ public String generateUML(String filter, Documentation documentation, GeneratorC """.formatted(documentation.getDocumentationTitle())); } uml.append("erDiagram\n"); - String tables = generateTables(filter, documentation); + String tables = generateTables(filterTags, documentation); uml.append(tables); return uml.toString(); } - private String generateTables(String filter, Documentation documentation) { + private String generateTables(String filterTags, Documentation documentation) { StringBuilder uml = new StringBuilder(INITIAL_CAPACITY); - List
tables = documentation.filterTables(filter); + List
tables = documentation.filterTables(filterTags); tables.forEach(table -> { uml.append(generateTableForeignKeys(table)); diff --git a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/PlantUMLGenerator.java b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/PlantUMLGenerator.java index bc66cd0..481f964 100644 --- a/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/PlantUMLGenerator.java +++ b/generators/documentation-generator-erdiagram/src/main/java/dk/fust/docgen/erdiagram/generators/PlantUMLGenerator.java @@ -38,9 +38,9 @@ public String getMarkdownType() { } @Override - public String generateUML(String filter, Documentation documentation, GeneratorConfiguration generatorConfiguration) { + public String generateUML(String filterTags, Documentation documentation, GeneratorConfiguration generatorConfiguration) { StringBuilder stringBuilder = new StringBuilder(1024); - List
tables = documentation.filterTables(filter); + List
tables = documentation.filterTables(filterTags); stringBuilder.append(""" @startuml diff --git a/generators/documentation-generator-erdiagram/src/test/groovy/dk/fust/docgen/erdiagram/ERDiagramGeneratorSpec.groovy b/generators/documentation-generator-erdiagram/src/test/groovy/dk/fust/docgen/erdiagram/ERDiagramGeneratorSpec.groovy index b3341d7..6e403ce 100644 --- a/generators/documentation-generator-erdiagram/src/test/groovy/dk/fust/docgen/erdiagram/ERDiagramGeneratorSpec.groovy +++ b/generators/documentation-generator-erdiagram/src/test/groovy/dk/fust/docgen/erdiagram/ERDiagramGeneratorSpec.groovy @@ -21,7 +21,7 @@ Trailing stuff ERDiagramConfiguration erDiagramConfiguration = new ERDiagramConfiguration( destination: new MarkdownDestination(file: markdownFile), generateKeys: [ - new GenerateKey(filter: 'domain-model', destinationKey: 'SOME_PLACEHOLDER') + new GenerateKey(filterTags: 'domain-model', destinationKey: 'SOME_PLACEHOLDER') ] ) diff --git a/generators/documentation-generator-sqlscript/README.md b/generators/documentation-generator-sqlscript/README.md index 21b67d7..60ac085 100644 --- a/generators/documentation-generator-sqlscript/README.md +++ b/generators/documentation-generator-sqlscript/README.md @@ -10,7 +10,7 @@ To use the sql generator, you must configure it with `dk.fust.docgen.sqlscript.S |-------------------|-------------|----------------------------------------------------|----------| | documentationFile | File | Location of the documentation file | | | destination | Destination | Where to send the generated documentation | | -| filter | String | If specified, only tables with this filter is used | | +| filterTags | String | If specified, only tables with this filterTags is used | | | sqlDialect | SqlDialect | SQL Dialect that the sql scripts are generated in | POSTGRES | ## Model usage diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java index f79e769..bc3728f 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/SqlScriptConfiguration.java @@ -19,7 +19,7 @@ public class SqlScriptConfiguration implements GeneratorConfiguration { private Destination destination; - private String filter; + private String filterTags; /** * Which dialect must the sql be? Default: POSTGRES diff --git a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java index 8ceab28..cf800fb 100644 --- a/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java +++ b/generators/documentation-generator-sqlscript/src/main/java/dk/fust/docgen/sqlscript/generators/PostgresGenerator.java @@ -22,7 +22,7 @@ public class PostgresGenerator implements SqlGenerator { public void generate(Documentation documentation, SqlScriptConfiguration sqlScriptConfiguration) throws IOException { log.debug("Generating Postgres SQL-scripts..."); String schemaName = documentation.getSchemaName(); - for (Table table : documentation.filterTables(sqlScriptConfiguration.getFilter())) { + for (Table table : documentation.filterTables(sqlScriptConfiguration.getFilterTags())) { Generation generationForTable = documentation.getGenerationForTable(table); if (table.getCreateTableScript() != null) { StringBuilder sqlScript = new StringBuilder(256);