add scripts to dump and repeat http traffic

README.md

@@ -43,7 +43,7 @@ as root:
 ## Requirements
 
 - Host for testing framework: `python3`, `wrk`, `ab`, `nghttp2`, `h2spec`, 
-`curl`, `h2load`, `tls-perf`, `netstat`, `lxc`, `nginx`, web content 
+`curl`, `h2load`, `tls-perf`, `netstat`, `lxc`, `nginx`, `tshark`, web content 
 directory accessible by nginx, nginx should not be running before the tests start.
 See Python libraries in `requirements.txt`
 - All hosts except previous one: `sftp-server`

framework/deproxy_client.py

@@ -217,7 +217,7 @@ def send_request(self, request, expected_status_code: Optional[str] = None, time
             )
 
     def send_bytes(self, data: bytes, expect_response=False):
-        self._add_to_request_buffers(data=data, end_stream=None)
+        self._add_to_request_buffers(data=data, end_stream=None, is_body=False)
         self.nrreq += 1
         if expect_response:
             self.valid_req_num += 1
@@ -438,7 +438,7 @@ def make_requests(self, requests, huffman=True, *args, **kwargs):
 
     def make_request(
         self,
-        request: Union[tuple, list, str, deproxy.H2Request],
+        request: Union[tuple, list, str, bytes, deproxy.H2Request],
         end_stream=True,
         priority_weight=None,
         priority_depends_on=None,
@@ -449,7 +449,7 @@ def make_request(
         Add request to buffers and change counters.
         Args:
             request:
-                str - send data frame;
+                str or bytes - send data frame;
                 list - send headers frame;
                 tuple - send headers and data frame in one TCP-packet;
             end_stream (bool) - set END_STREAM flag for frame;
@@ -470,6 +470,7 @@ def make_request(
             priority_weight=priority_weight,
             priority_depends_on=priority_depends_on,
             priority_exclusive=priority_exclusive,
+            is_body=True if isinstance(request, bytes) else False,
         )
 
         self.nrreq += 1
@@ -503,7 +504,7 @@ def update_initial_settings(
         self,
         header_table_size: int = None,
         enable_push: int = None,
-        max_concurrent_stream: int = None,
+        max_concurrent_streams: int = None,
         initial_window_size: int = None,
         max_frame_size: int = None,
         max_header_list_size: int = None,
@@ -515,7 +516,7 @@ def update_initial_settings(
         new_settings = self.__generate_new_settings(
             header_table_size,
             enable_push,
-            max_concurrent_stream,
+            max_concurrent_streams,
             initial_window_size,
             max_frame_size,
             max_header_list_size,
@@ -532,7 +533,7 @@ def send_settings_frame(
         self,
         header_table_size: int = None,
         enable_push: int = None,
-        max_concurrent_stream: int = None,
+        max_concurrent_streams: int = None,
         initial_window_size: int = None,
         max_frame_size: int = None,
         max_header_list_size: int = None,
@@ -542,7 +543,7 @@ def send_settings_frame(
         new_settings = self.__generate_new_settings(
             header_table_size,
             enable_push,
-            max_concurrent_stream,
+            max_concurrent_streams,
             initial_window_size,
             max_frame_size,
             max_header_list_size,
@@ -733,7 +734,7 @@ def __binary_headers_to_string(headers):
     def __generate_new_settings(
         header_table_size: int = None,
         enable_push: int = None,
-        max_concurrent_stream: int = None,
+        max_concurrent_streams: int = None,
         initial_window_size: int = None,
         max_frame_size: int = None,
         max_header_list_size: int = None,
@@ -742,9 +743,9 @@ def __generate_new_settings(
         if header_table_size is not None:
             new_settings[SettingCodes.HEADER_TABLE_SIZE] = header_table_size
         if enable_push is not None:
-            new_settings[SettingCodes.ENABLE_PUSH] = header_table_size
-        if max_concurrent_stream is not None:
-            new_settings[SettingCodes.MAX_CONCURRENT_STREAMS] = max_concurrent_stream
+            new_settings[SettingCodes.ENABLE_PUSH] = enable_push
+        if max_concurrent_streams is not None:
+            new_settings[SettingCodes.MAX_CONCURRENT_STREAMS] = max_concurrent_streams
         if initial_window_size is not None:
             new_settings[SettingCodes.INITIAL_WINDOW_SIZE] = initial_window_size
         if max_frame_size is not None:
@@ -792,11 +793,16 @@ def _add_to_request_buffers(
         priority_weight=None,
         priority_depends_on=None,
         priority_exclusive=None,
+        is_body: bool,
     ) -> None:
-        if isinstance(data, bytes):
+        if isinstance(data, bytes) and not is_body:
             # in case when you use `send_bytes` method
             self._request_buffers.append(data)
             self._add_to_body_buffers(body=None, stream_id=None, end_stream=None)
+        elif isinstance(data, bytes) and is_body:
+            # in case when you use `mak_request` method
+            self._request_buffers.append(b"")
+            self._add_to_body_buffers(body=data, stream_id=self.stream_id, end_stream=end_stream)
         elif isinstance(data, str):
             # in case when you use `make_request` to sending body
             self._request_buffers.append(b"")

helpers/deproxy.py

@@ -965,6 +965,8 @@ def apply_proto_settings(self):
             # RFC 9113 Section 9.2.1: A deployment of HTTP/2 over TLS 1.2 MUST disable
             # compression.
             self.context.options |= ssl.OP_NO_COMPRESSION
+        elif self.proto == "http/1.1":
+            self.context.set_alpn_protocols(["http/1.1"])
 
 
 class Client(TlsClient, stateful.Stateful):

scripts/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "test_replay",
+]
+
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

scripts/tcpdump.py

@@ -0,0 +1,225 @@
+import argparse
+import datetime
+import os
+import shutil
+import signal
+import subprocess
+import time
+
+
+class Logger:
+    def __add_ports(self, ports, args):
+        if ports:
+            args += ["tcp port"]
+            args += [f"{ports[0]}"]
+
+            for i in range(1, len(ports)):
+                args += ["or"]
+                args += ["tcp port"]
+                args += [f"{ports[i]}"]
+
+    def __add_ip(self, ip, direction, args):
+        if ip:
+            args += [f"ip {direction}"]
+            args += [f"{ip[0]}"]
+
+            for i in range(1, len(ip)):
+                args += ["or"]
+                args += [f"ip {direction}"]
+                args += [f"{ip[i]}"]
+
+    def __run_tcpdump(
+        self, ethname, file_size, file_count, file_name, src_ports, dst_ports, src, dst, direction
+    ) -> None:
+        """
+        Save result in a <file_name>.pcap file.
+        """
+        path = f"/var/tcpdump/{datetime.date.today()}"
+
+        if not os.path.isdir(path):
+            os.makedirs(path)
+
+        args = [
+            "tcpdump",
+            "-U",
+            "-i",
+            f"{ethname}",
+            "-C",
+            f"{file_size}",
+            "-W",
+            f"{file_count}",
+            "-w",
+            f"{path}/{file_name}-{direction}.pcap",
+            "-Q",
+            f"{direction}",
+            "-Z",
+            "root",
+        ]
+
+        combine = src_ports and dst_ports
+        self.__add_ports(src_ports, args)
+        if combine:
+            args += ["or"]
+        self.__add_ports(dst_ports, args)
+
+        if (src_ports or dst_ports) and (src or dst):
+            args += ["and"]
+
+        combine = src and dst
+        self.__add_ip(src, "src", args)
+        if combine:
+            args += ["or"]
+        self.__add_ip(dst, "dst", args)
+
+        if direction == "in":
+            self.__tcpdump_in = subprocess.Popen(
+                args=args,
+                shell=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        elif direction == "out":
+            self.__tcpdump_out = subprocess.Popen(
+                args=args,
+                shell=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        else:
+            print("Fail to start tcpdump - invalid direction")
+            return
+
+    def __stop_tcpdump(self, direction) -> None:
+        """
+        Stop tcpdump.
+        `wait()` should never causes `TimeoutExpired` error because `tcpdump` can
+        be successfully terminate by SIGINT. But it requires a timeout to flush
+        data from buffer.
+        """
+
+        __tcpdump = None
+        if direction == "in":
+            __tcpdump = self.__tcpdump_in
+        elif direction == "out":
+            __tcpdump = self.__tcpdump_out
+        else:
+            print("Fail to stop tcpdump - invalid direction")
+            return
+
+        try:
+            __tcpdump.send_signal(signal.SIGINT)
+            __tcpdump.wait(timeout=3)
+        except subprocess.TimeoutExpired:
+            __tcpdump.kill()
+            __tcpdump.wait()
+
+        if direction == "in":
+            self.__tcpdump_in = None
+        elif direction == "out":
+            self.__tcpdump_out = None
+
+    def run(
+        self,
+        ethname=None,
+        exec_time=None,
+        file_size=None,
+        file_count=None,
+        file_name=None,
+        src_ports=None,
+        dst_ports=None,
+        src=None,
+        dst=None,
+        direction=None,
+    ) -> None:
+        ethname = ethname if ethname else "lo"
+        exec_time = exec_time if exec_time else 60
+        file_size = file_size if file_size else 50
+        file_count = file_count if file_count else 10
+        file_name = file_name if file_name else f"{datetime.datetime.now().strftime('%H:%M:%S')}"
+        direction = direction if direction else ["in", "out"]
+
+        if "in" in direction:
+            self.__run_tcpdump(
+                ethname, file_size, file_count, file_name, src_ports, dst_ports, src, dst, "in"
+            )
+        if "out" in direction:
+            self.__run_tcpdump(
+                ethname, file_size, file_count, file_name, src_ports, dst_ports, src, dst, "out"
+            )
+        if (not "in" in direction) and (not "out" in direction):
+            print("Invalid direction, (in|out) supported")
+            return
+
+        time.sleep(exec_time * 60)
+        self.__stop_tcpdump("in")
+        self.__stop_tcpdump("out")
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-e", "--ethname", type=str, help="Device name to capture packets.")
+parser.add_argument(
+    "-t", "--exec-time", type=int, help="Execution time in minutes (60 by default)."
+)
+parser.add_argument(
+    "-s",
+    "--file-size",
+    type=int,
+    help="Dump file size in megabytes (50 by default). When size is exceeded new file will be created.",
+)
+parser.add_argument(
+    "-c",
+    "--file-count",
+    type=int,
+    help="Count of dump files (10 by default). When count is exceeded new file overwrite old file.",
+)
+parser.add_argument(
+    "-n", "--file-name", type=str, help="Dump file name (current time in H:M:S by default)."
+)
+parser.add_argument(
+    "-ps",
+    "--src-port",
+    action="append",
+    type=int,
+    help="Source ports, used in tcpdump filtration (empty by default, dump for all ports).",
+)
+parser.add_argument(
+    "-pd",
+    "--dst-port",
+    action="append",
+    type=int,
+    help="Destination ports, used in tcpdump filtration (empty by default, dump for all ports).",
+)
+parser.add_argument(
+    "--src",
+    type=str,
+    help="Source ip addresses, used in tcpdump filtration (empty by default, dump for all source ip).",
+    action="append",
+)
+parser.add_argument(
+    "--dst",
+    type=str,
+    help="Destination ip addresses, used in tcpdump filtration (empty by default, dump for all destination ip).",
+    action="append",
+)
+parser.add_argument(
+    "--direction",
+    type=str,
+    help="Type of collected traffic(in|put)",
+    action="append",
+)
+
+args = parser.parse_args()
+
+Log = Logger()
+Log.run(
+    args.ethname,
+    exec_time=args.exec_time,
+    file_size=args.file_size,
+    file_count=args.file_count,
+    file_name=args.file_name,
+    src_ports=args.src_port,
+    dst_ports=args.dst_port,
+    src=args.src,
+    dst=args.dst,
+    direction=args.direction,
+)