1.06-SNAPSHOT

Author: anders-wartoft

README.md

@@ -16,9 +16,11 @@ java -jar target/LogGenerator-with-dependencies.jar -i kafka -icn test2 -itn tes
 When running the last command, press Ctrl-C to see the gaps in the received data. Since we started the counter on 100, there should at least be one gap: 1-99.
 
 ### Latest Release notes
-#### 1.05
-- SelectFilter added. Use a regular expression to specify what to keep in each event
-- GapDetection will now print gaps each time the statistics is printed. Events that appear out of order will now be reflected without need to shut down the receiver.
+#### 1.06-SNAPSHOT
+- DropFilter added. Use a regular expression to specify events to drop
+- DateSubstitute now supports epoch format
+- Moved headers to last in event chain. Headers will now be created after all other filters have been executed
+- The eps limiter can now use floating points, so to send one event every 4 seconds, use -e 0.25
 
 ### Input modules:
 There are input module for the following tasks:
@@ -274,6 +276,8 @@ In another command window, start the client (same jar file):
 ### Filter modules:
 - Add a header
 - Replace by regex
+- Select by regex
+- Drop by regex
 - Replace variables
 - Remove transaction messages
 - Detect gaps
@@ -302,6 +306,22 @@ and for example want to extract the first word after the timestamp, you can use
 
 `-se '\.\d{3} (\S+) '`
 
+### Drop by regex
+If you want to drop an event depending on a regular expression, use the DropFilter.
+Use a regex to specify the content to search for. If the DropFilter finds a match, the event will be dropped.
+
+Parameters: `-df {regex}`
+
+Example: `-df 'INFO'`
+
+If you have events that are formatted like this:
+
+`[Sat Dec 03 00:35:57.399 Usb Host Notification Apple80211Set: seqNum 5460 Total 1 chg 0 en0]`
+
+and for example want to discard all events that contain `chg 0`, you can use the following parameters:
+
+`-df 'chg 0'`
+
 #### Replace by regex
 A use case is if you have a lot of nice logs, but the date is not possible to use. You can load the file and add a regex to find the date, then replace the date with a {date:...} variable or a static string.
 There must be a capture group in the regex. The text matched by the capture group will be replaced by the value.
@@ -386,13 +406,14 @@ So, the gap detection can detect events that should have been delivered earlier,
 ### Variables
 #### Date
 A Date variable will take the current time and format according to a Java 
-date format string. 
+date format string. Epoch timestamps are supported but only the version with 13 digits.
 
-Syntax: `{date:(?<datepattern>[yYmMHhsz+-dD:\d'T. ]+)(/(?<locale>[^}]+))?}`
+Syntax: `{date:(?<datepattern>[yYmMHhsz+-dD:\d'T. ]+|epoch)(/(?<locale>[^}]+))?}`
 
 Example (at new year 2023)
 - `{date:yyyyMMdd}` might be resolved to: `20230101`
 - `{date:MMM dd HH:mm:ss/en:US}` might be resolved to `Jan 01 00:00:00`
+- `{date:epoch}` might be resolved to e.g., `0946681200000` (for Jan 01, 2000)
 
 Date can also have an offset with the `-to` (`--time-offset`) parameter. Time offset can be used to add or remove a number of milliseconds from the date variable. 
 

pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>nu.sitia.LogGenerator</groupId>
   <artifactId>LogGenerator</artifactId>
-  <version>1.05-SNAPSHOT</version>
+  <version>1.06-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <name>LogGenerator</name>

src/main/java/nu/sitia/loggenerator/App.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */

src/main/java/nu/sitia/loggenerator/Configuration.java

@@ -112,6 +112,7 @@ public class Configuration {
                 keys.add(new Item("-jf", "--json-filter", "Get a specific node in the JSON input. Use . to traverse the JSON input. If the returned object is an array, the array items are sent in the event chain as new events"));
                 keys.add(new Item("-jfp", "--json-file-path", "Get a specific node in the JSON file input. Use . to traverse the JSON input. If the returned object is an array, the array items are sent in the event chain as new events"));
                 keys.add(new Item("-se", "--select", "Remove everything except what matches the next argument's first group. Example -se 'userid:(\\S+)'"));
+                keys.add(new Item("-df", "--drop-filter", "Drop an event if the supplied regex is found in an event. Example -df 'INFO|DEBUG'"));
         }
         static final Map<String, String> standardVariables = new HashMap<>();
         static {

src/main/java/nu/sitia/loggenerator/ItemProxy.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
@@ -48,7 +48,7 @@ public class ItemProxy {
     private final List<ProcessFilter> filterList;
 
     /** Preferred eps */
-    private final long eps;
+    private final double eps;
 
     /** Limit the number of events to send */
     private final long limit;
@@ -89,7 +89,7 @@ public ItemProxy(InputItem input, OutputItem output, List<ProcessFilter> filterL
 
         String epsString = config.getValue("-e");
         if (null != epsString) {
-            this.eps = Long.parseLong(epsString);
+            this.eps = Double.parseDouble(epsString);
         } else {
             this.eps = 0;
         }
@@ -119,8 +119,8 @@ public ItemProxy(InputItem input, OutputItem output, List<ProcessFilter> filterL
             shutdownHandlers.add((ShutdownHandler) output);
         }
 
-        String isContinousGapDetectionString = config.getValue("-cgd");
-        if (isContinousGapDetectionString != null && isContinousGapDetectionString.equalsIgnoreCase("true")) {
+        String isContinuousGapDetectionString = config.getValue("-cgd");
+        if (isContinuousGapDetectionString != null && isContinuousGapDetectionString.equalsIgnoreCase("true")) {
             gapDetector = getGapDetector(filterList);
             if (null == gapDetector) {
                 throw new RuntimeException("The flag -cgd cannot be used without a GapDetector (-gd)");
@@ -150,7 +150,7 @@ public ItemProxy(InputItem input, OutputItem output, List<ProcessFilter> filterL
 
 
     /**
-     * Search the list of filters and, if present, return a GapDetecttionFilter
+     * Search the list of filters and, if present, return a GapDetectionFilter
      * @param filterList a List<filter> to search
      * @return GapDetectionFilter or null
      */
@@ -238,7 +238,7 @@ private void throttle(LogStatistics statistics) {
             long sentMessages = statistics.getTransactionMessages();
             long now = new Date().getTime();
             // how long time should we spend on sending these messages?
-            long estimatedTime = 1000 * sentMessages / eps;
+            long estimatedTime = (long)(1000 * sentMessages / eps);
             long waitTime = transactionStart + estimatedTime - now;
             if (waitTime > 10) {
                 try {

src/main/java/nu/sitia/loggenerator/ShutdownHandler.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */

src/main/java/nu/sitia/loggenerator/filter/DropFilter.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2022 sitia.nu https://github.com/anders-wartoft/LogGenerator
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package nu.sitia.loggenerator.filter;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DropFilter implements ProcessFilter {
+    /** Cached regex pattern */
+    private final Pattern pattern;
+
+    /** for toString() */
+    private final String regex;
+
+    /**
+     * Create a RegexFilter and set all parameters
+     * @param regex The value to search for
+     */
+    public DropFilter(String regex) {
+        this.regex = regex;
+        // What to look for
+        if (null == regex) {
+            throw new RuntimeException("regex is null");
+        }
+        pattern = Pattern.compile(regex);
+    }
+
+    /**
+     * Filter one string
+     *
+     * @param toFilter The string to change
+     * @return toFilter with a header added before the string.
+     */
+    private String filter(String toFilter) {
+        Matcher matcher = pattern.matcher(toFilter);
+        if (matcher.find()) {
+            // Drop this one
+            return null;
+        }
+        return toFilter;
+    }
+
+    @Override
+    public List<String> filter(List<String> toFilter) {
+        List<String> filtered = new ArrayList<>();
+        toFilter.forEach(s -> {
+                    String result = filter(s);
+                    if (result != null) {
+                        filtered.add(result);
+                    }
+                });
+
+        return filtered;
+    }
+
+    /**
+     * The current configuration
+     * @return A printout of the current configuration
+     */
+    @Override
+    public String toString() {
+        return "DropFilter" + System.lineSeparator() + regex + System.lineSeparator();
+    }
+}

src/main/java/nu/sitia/loggenerator/filter/FilterListFactory.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
@@ -43,7 +43,8 @@ public List<ProcessFilter> create(Configuration config) {
         String jsonFilter = config.getValue("-jf");
         String jsonReport = config.getValue("-gdjr");
         String select = config.getValue("-se");
-
+        String drop = config.getValue("-df");
+        boolean df = "true".equalsIgnoreCase(drop);
         boolean dd = "true".equalsIgnoreCase(doubleDetection);
 
         List<ProcessFilter> filterList = new LinkedList<>();
@@ -52,12 +53,12 @@ public List<ProcessFilter> create(Configuration config) {
             filterList.add(new GuardFilter(config));
         }
 
-        if (regex != null) {
-            filterList.add(new RegexFilter(regex, value));
+        if (df) {
+            filterList.add(new DropFilter(regex));
         }
 
-        if (header != null) {
-            filterList.add(new HeaderFilter(header));
+        if (regex != null) {
+            filterList.add(new RegexFilter(regex, value));
         }
 
         Template template1 = new NoneTemplate();
@@ -83,6 +84,11 @@ public List<ProcessFilter> create(Configuration config) {
         if (select != null) {
             filterList.add(new SelectFilter(select));
         }
+
+        if (header != null) {
+            filterList.add(new HeaderFilter(header));
+        }
+
         return filterList;
     }
 }

src/main/java/nu/sitia/loggenerator/filter/GapDetectionFilter.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */

src/main/java/nu/sitia/loggenerator/filter/GuardFilter.java

@@ -10,7 +10,7 @@
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
- *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ *  WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */