diff --git a/config/gui_default.cfg b/config/gui_default.cfg index c4f5f3ca..32570f71 100644 --- a/config/gui_default.cfg +++ b/config/gui_default.cfg @@ -19,7 +19,7 @@ "VisualizerPanel": { "Width": 1500, "Height": 800, - "InnerWidth": 1600, + "InnerWidth": 2000, "InnerHeight": 1000 }, "Buttons": { @@ -36,7 +36,7 @@ "Name": "Control & Statistics", "Width": 330, "Height": 550, - "DateFormat": "hh:mm:ss:SS", + "DateFormat": "HH:mm:ss", "ShowTimePanel": true, "TimeSliderButtons": { @@ -146,12 +146,12 @@ "LinesPointSize": 5, "PaintFill": false, "VerticalBarSize": 5, - "xAxisTitle": "Timestamp", + "xAxisTitle": "", "xAxisType": "date", - "xAxisFormat": "hh:mm:ss:SS", + "xAxisFormat": "HH:mm:ss", "xAxisOffset": 0.2, - "y1AxisTitle": "y1", - "y2AxisTitle": "y2" + "y1AxisTitle": "", + "y2AxisTitle": "" }, "Legend": { "Width": 190, @@ -179,7 +179,9 @@ "MenuBar": { "Width": 635, "Height": 50, + "visible": true, "showIntervalPanel": true, + "connected": true, "showXOptionsPanel": true, "showYOptionsPanel": true, "showCoordsPanel": true, @@ -221,10 +223,10 @@ "PaintFill": false, "VerticalBarSize": 5, "xAxisOffset": 0.2, - "x1AxisTitle": "x1", - "x2AxisTitle": "x2", - "y1AxisTitle": "y1", - "y2AxisTitle": "y2" + "x1AxisTitle": "", + "x2AxisTitle": "", + "y1AxisTitle": "", + "y2AxisTitle": "" }, "Legend": { "Width": 190, @@ -252,6 +254,7 @@ "MenuBar": { "Width": 635, "Height": 50, + "visible": true, "showIntervalPanel": true, "showXOptionsPanel": true, "showYOptionsPanel": true, @@ -286,6 +289,71 @@ } } } - } + }, + "LabelVisualizerConfigs": { + "LabelVisualizer1": { + "Name": "Label Visualizer 1", + "TraceLength": 1000, + "TraceModeLtd": true, + + "AdditionPolicy": "all", + "AdditionList": ["labeler0.type0", "labeler0.type1"], + + "position": { + "x": 2, + "y": 0, + "rowspan": 1, + "colspan": 1 + }, + "Chart": { + "Width": 450, + "Height": 140, + "BarThickness": 0.7, + "xAxisTitle": "Timestamp", + "xAxisType": "date", + "xAxisFormat": "HH:mm:ss", + "y1AxisTitle": "Labels", + }, + "Legend": { + "Width": 190, + "Height": 150, + + "LegendItem": { + "Width": 165, + "Height": 40, + "Button_Width": 20, + "Button_Height": 20, + "ButtonPanel_Width": 100, + "ButtonPanel_Height": 20, + "NameLabel_Width": 160, + "NameLabel_Height": 16, + "ValueLabel_Width": 60, + "ValueLabel_Height": 20, + "ValueFont": { + "Name": "Dialog", + "Style": "PLAIN", + "Size": 9, + "Color": "BLACK" + } + } + }, + "MenuBar": { + "Width": 635, + "Height": 50, + "visible": false, + "showIntervalPanel": true, + "connected": true, + "showXOptionsPanel": true, + "showYOptionsPanel": false, + "showCoordsPanel": true, + "CoordsFont": { + "Name": "Dialog", + "Style": "PLAIN", + "Size": 11, + "Color": "BLACK" + } + }, + } + }, } } \ No newline at end of file diff --git a/config/gui_min_lab.cfg b/config/gui_min_lab.cfg new file mode 100644 index 00000000..bc90c2e0 --- /dev/null +++ b/config/gui_min_lab.cfg @@ -0,0 +1,61 @@ +{ + "MainDisplayConfig": { + "Name": "DNA - Dynamic Network Analyzer", + "LiveDisplayMode": false, + "DefaultDir": "data/test123/run.0/", + + "Width": 1680, + "Height": 800, + + "VisualizerPanel": { + "Width": 1000, + "Height": 800, + "InnerWidth": 900, + "InnerHeight": 1600 + }, + + + "StatsDisplayConfig": { + }, + + "MetricVisualizerConfigs": { + "MetricVisualizer1": { + "Name": "Metric Visualizer 1", + "position": { + "x": 0, + "y": 0, + }, + + "VisualizerConfig": { + "SingleConfigs": { + "inDegreeHosts": { + "Type": "MetricVisualizerItem", + "Name": "DegreeDistributionR-HOST.InDegreeDistribution_95p", + "DisplayMode": "linespoint", + "yAxis": "y1", + "visible": true, + } + } + } + } + }, + "MultiScalarVisualizerConfigs": { + "MultiScalarVisualizer1": { + "Name": "MultiScalar Visualizer 1", + "position": { + "x": 0, + "y": 2, + } + } + }, + "LabelVisualizerConfigs": { + "LabelVisualizer1": { + "Name": "Label Visualizer 2", + "position": { + "x": 0, + "y": 1, + } + } + } + } +} diff --git a/config/gvis/gids.cfg b/config/gvis/gids.cfg new file mode 100644 index 00000000..2ba7001f --- /dev/null +++ b/config/gvis/gids.cfg @@ -0,0 +1,25 @@ +{ + "GraphPanelConfig": { + "Width": 1024, + "Height": 768, + + "TimestampFormat": "HH:mm:ss", + "TimestampOffset": -7200, + "TimestampInSeconds": true, + + "StatPanelEnabled": true, + "TextPanelEnabled": true, + + "RulesConfig": { + "dna.visualization.graph.rules.nodes.NetworkNodeStyles": {}, + "dna.visualization.graph.rules.nodes.NodeSizeByDegree": { + "Name": "defaultNodeSizeByDegree", + "Growth": 0.3, + "Enabled": true, + }, + "dna.visualization.graph.rules.edges.EdgeWeightLabel": { + "label": "w=$weight$", + }, + } + } +} \ No newline at end of file diff --git a/config/vis/util.properties b/config/vis/util.properties index 1550a978..39964dfb 100644 --- a/config/vis/util.properties +++ b/config/vis/util.properties @@ -2,6 +2,13 @@ ## VISUALIZATION UTILITY OPTIONS ######################################## + +## TIMESTAMP INTERPRETATION SETTINGS ### + +## todo: put these in actual vis config +VISUALIZATION_TIMESTAMP_AS_SECOND = true +VISUALIZATION_TIMESTAMP_OFFSET = -7200 + ## SCREENSHOT SETTINGS ################# VIS_SCREENSHOT_DIR = images/ VIS_SCREENSHOT_FORMAT = png diff --git a/src/dna/graph/generators/network/NetflowBatch.java b/src/dna/graph/generators/network/NetflowBatch.java index e97de852..dada2337 100644 --- a/src/dna/graph/generators/network/NetflowBatch.java +++ b/src/dna/graph/generators/network/NetflowBatch.java @@ -726,4 +726,12 @@ public String getKey(Integer mapping) { return "unknown"; } + + public void setMap(HashMap map) { + this.map = map; + } + + public HashMap getMap() { + return this.map; + } } diff --git a/src/dna/graph/generators/network/NetflowGraph.java b/src/dna/graph/generators/network/NetflowGraph.java new file mode 100644 index 00000000..f9b9c9e7 --- /dev/null +++ b/src/dna/graph/generators/network/NetflowGraph.java @@ -0,0 +1,122 @@ +package dna.graph.generators.network; + +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; + +import org.joda.time.DateTime; + +import dna.graph.Graph; +import dna.graph.datastructures.GraphDataStructure; +import dna.graph.edges.network.UpdateEvent; +import dna.io.network.NetworkEvent; +import dna.io.network.netflow.NetflowEvent.NetflowDirection; +import dna.io.network.netflow.NetflowEvent.NetflowEventField; +import dna.io.network.netflow.NetflowEventReader; +import dna.io.network.netflow.darpa.DarpaNetflowReader; +import dna.updates.batch.Batch; +import dna.util.network.NetflowAnalysis.EdgeWeightValue; +import dna.util.network.NetflowAnalysis.NodeWeightValue; +import dna.visualization.graph.rules.nodes.NetworkNodeStyles; + +public class NetflowGraph extends NetworkGraph { + + private NetflowEventReader reader; + private NetflowBatch batchGenerator; + + private int batchLengthSeconds; + private int edgeLifeTimeSeconds; + + private NetflowEventField[][] edges; + private NetflowDirection[] edgeDirections; + private EdgeWeightValue[] edgeWeights; + private NodeWeightValue[] nodeWeights; + + private DateTime initDateTime; + private DateTime firstDateTime; + + private Graph graph; + + public NetflowGraph(GraphDataStructure gds, long timestampInit, long maximumTimestamp, String dir, String filename, + int batchLengthSeconds, int edgeLifeTimeSeconds, NetflowEventField[][] edges, + NetflowDirection[] edgeDirections, EdgeWeightValue[] edgeWeights, NodeWeightValue[] nodeWeights) + throws IOException, ParseException { + this(gds, timestampInit, maximumTimestamp, dir, filename, batchLengthSeconds, edgeLifeTimeSeconds, true, true, + 0, edges, edgeDirections, edgeWeights, nodeWeights); + } + + public NetflowGraph(GraphDataStructure gds, long timestampInit, long maximumTimestamp, String dir, String filename, + int batchLengthSeconds, int edgeLifeTimeSeconds, boolean removeZeroDegreeNodes, + boolean removeZeroWeightEdges, int dataOffsetSeconds, NetflowEventField[][] edges, + NetflowDirection[] edgeDirections, EdgeWeightValue[] edgeWeights, NodeWeightValue[] nodeWeights) + throws IOException, ParseException { + super("NetflowGraph", gds, timestampInit); + + this.batchLengthSeconds = batchLengthSeconds; + this.edgeLifeTimeSeconds = edgeLifeTimeSeconds; + this.edges = edges; + this.edgeDirections = edgeDirections; + this.edgeWeights = edgeWeights; + this.nodeWeights = nodeWeights; + + this.initDateTime = new DateTime(TimeUnit.SECONDS.toMillis(this.timestampInit)); + this.firstDateTime = new DateTime(TimeUnit.SECONDS.toMillis(this.timestampInit - edgeLifeTimeSeconds)); + + this.reader = new DarpaNetflowReader(dir, filename); + this.reader.setBatchIntervalSeconds(batchLengthSeconds); + this.reader.setEdgeLifeTimeSeconds(edgeLifeTimeSeconds); + this.reader.setRemoveZeroDegreeNodes(removeZeroDegreeNodes); + this.reader.setRemoveZeroWeightEdges(removeZeroWeightEdges); + this.reader.setDataOffset(this.reader.getDataOffset() + dataOffsetSeconds); + this.reader.setMinimumTimestamp(this.firstDateTime); + this.reader.setMaximumTimestamp(new DateTime(TimeUnit.SECONDS.toMillis(maximumTimestamp))); + + this.batchGenerator = new NetflowBatch("temp", reader, edges, edgeDirections, edgeWeights, nodeWeights); + + // set batch-generator for mapping retrieval + NetworkNodeStyles.netflowBatchGenerator = (NetflowBatch) this.batchGenerator; + + this.graph = generateGraph(); + } + + private Graph generateGraph() { + Graph g = this.newGraphInstance(); + g.setTimestamp(TimeUnit.MILLISECONDS.toSeconds(this.firstDateTime.getMillis())); + + DateTime simTime = this.firstDateTime; + + while (simTime.isBefore(this.initDateTime)) { + ArrayList events = this.reader.getEventsUntil(simTime); + if (events.size() > 0) { + Batch b = this.batchGenerator.craftBatch(g, simTime, events, new ArrayList(0), null); + b.apply(g); + } + simTime = simTime.plusSeconds(this.batchLengthSeconds); + } + + ArrayList events = this.reader.getEventsUntil(this.initDateTime); + if (events.size() > 0) { + Batch b = this.batchGenerator.craftBatch(g, this.initDateTime, events, new ArrayList(0), null); + b.apply(g); + } + + g.setTimestamp(TimeUnit.MILLISECONDS.toSeconds(this.initDateTime.getMillis())); + + // return graph + return g; + } + + @Override + public Graph generate() { + return this.graph; + } + + public NetflowEventReader getReader() { + return this.reader; + } + + public NetflowBatch getBatchGenerator() { + return this.batchGenerator; + } +} diff --git a/src/dna/graph/weights/network/NetworkNodeWeight.java b/src/dna/graph/weights/network/NetworkNodeWeight.java index 280be762..6c6c24c2 100644 --- a/src/dna/graph/weights/network/NetworkNodeWeight.java +++ b/src/dna/graph/weights/network/NetworkNodeWeight.java @@ -35,6 +35,11 @@ public String toString() { return buff; } + @Override + public String asString() { + return toString(); + } + public NetworkNodeWeight(WeightSelection ws) { this("unknown", new double[0]); } diff --git a/src/dna/labels/labeler/attacks/GroundTruthLabelerAttacks.java b/src/dna/labels/labeler/attacks/GroundTruthLabelerAttacks.java new file mode 100644 index 00000000..89a032f2 --- /dev/null +++ b/src/dna/labels/labeler/attacks/GroundTruthLabelerAttacks.java @@ -0,0 +1,37 @@ +package dna.labels.labeler.attacks; + +import java.io.IOException; + +import dna.labels.labeler.list.ListLabeler; +import dna.labels.labeler.util.IntervalLabeler; + +public class GroundTruthLabelerAttacks extends ListLabeler { + + protected int edgeLifeTimeSeconds; + protected boolean init = false; + + public GroundTruthLabelerAttacks(String name, String dir, String filename, int edgeLifeTimeSeconds) { + super(name, dir, filename); + this.edgeLifeTimeSeconds = edgeLifeTimeSeconds; + this.init = true; + + try { + this.labeler = parseLabelerFromList(name, dir, filename); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + protected IntervalLabeler parseLabelerFromString(String name, String line) { + if (this.init) { + String[] splits = line.split("\t"); + long timestamp = Long.parseLong(splits[0]); + String value = splits[3]; + return new IntervalLabeler(this.getName(), "0-1", value, timestamp, timestamp + this.edgeLifeTimeSeconds); + } else { + return new IntervalLabeler("", 0, 0); + } + } + +} diff --git a/src/dna/labels/labeler/models/ModelLabeler.java b/src/dna/labels/labeler/models/ModelLabeler.java new file mode 100644 index 00000000..27e5bb63 --- /dev/null +++ b/src/dna/labels/labeler/models/ModelLabeler.java @@ -0,0 +1,55 @@ +package dna.labels.labeler.models; + +import java.io.IOException; +import java.util.ArrayList; + +import dna.graph.Graph; +import dna.graph.generators.GraphGenerator; +import dna.labels.Label; +import dna.labels.labeler.Labeler; +import dna.metrics.IMetric; +import dna.series.data.BatchData; +import dna.updates.batch.Batch; +import dna.updates.generators.BatchGenerator; +import dna.util.machineLearning.ModelWrapper; + +/** + * The ModelLabeler labels a batch based on a trained machine-learning model. + * + * @author Rwilmes + * + */ +public class ModelLabeler extends Labeler { + + protected ModelWrapper model; + + public ModelLabeler(String name, String scriptPath, String featureListPath, int numberOfFeatures, + String modelPath) { + super(name); + + try { + this.model = new ModelWrapper(scriptPath, featureListPath, numberOfFeatures, modelPath); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + public boolean isApplicable(GraphGenerator gg, BatchGenerator bg, IMetric[] metrics) { + return true; + } + + @Override + public ArrayList