Rework tls_filter

test/modules/tls/env.py

@@ -107,23 +107,12 @@ def __init__(self, pytestconfig=None):
             '    AddHandler cgi-script .py',
             '    Options +ExecCGI',
             '</Directory>',
-            f'<VirtualHost *:{self.http_port}>',
-            '    ServerName localhost',
-            '    DocumentRoot "htdocs"',
-            '</VirtualHost>',
-            f'<VirtualHost *:{self.http_port}>',
-            f'    ServerName {self.domain_a}',
-            '    DocumentRoot "htdocs/a.mod-tls.test"',
-            '</VirtualHost>',
-            f'<VirtualHost *:{self.http_port}>',
-            f'    ServerName {self.domain_b}',
-            '    DocumentRoot "htdocs/b.mod-tls.test"',
-            '</VirtualHost>',
         ])
         self.add_cert_specs([
             CertificateSpec(domains=[self.domain_a]),
             CertificateSpec(domains=[self.domain_b], key_type='secp256r1', single_file=True),
             CertificateSpec(domains=[self.domain_b], key_type='rsa4096'),
+            CertificateSpec(domains=['localhost'], key_type='rsa4096'),
             CertificateSpec(name="clientsX", sub_specs=[
                 CertificateSpec(name="user1", client=True, single_file=True),
                 CertificateSpec(name="user2", client=True, single_file=True),

test/modules/tls/test_09_timeout.py

@@ -24,7 +24,8 @@ def _function_scope(self, env):
     def test_tls_09_timeout_handshake(self, env):
         # in domain_b root, the StdEnvVars is switch on
         s = socket.create_connection(('localhost', env.https_port))
-        s.send(b'1234')
+        # something that looks like a ClientHello
+        s.send(bytes.fromhex('3c37121e'))
         s.settimeout(0.0)
         try:
             s.recv(1024)

test/modules/tls/test_13_proxy.py

@@ -18,6 +18,8 @@ def _class_scope(self, env):
             ]
         })
         # add vhosts a+b and a ssl proxy from a to b
+        conf.add_vhost('localhost', port=env.http_port)
+        conf.add_vhost(env.domain_b, port=env.http_port, doc_root=f"htdocs/{env.domain_b}")
         conf.add_tls_vhosts(domains=[env.domain_a, env.domain_b])
         conf.install()
         assert env.apache_restart() == 0

test/modules/tls/test_18_ws.py

@@ -0,0 +1,212 @@
+import ssl
+from datetime import datetime, timedelta
+import inspect
+import os
+import shutil
+import subprocess
+import time
+
+import pytest
+import websockets
+from websockets.sync.client import connect
+
+from .conf import TlsTestConf
+
+
+class WsServer:
+
+    def __init__(self, name, env, port, creds=None):
+        self.name = name
+        self.env = env
+        self.process = None
+        self.cerr = None
+        self.port = port
+        self.creds = creds
+        self.run_dir = os.path.join(env.gen_dir, self.name)
+        self.err_file = os.path.join(self.run_dir, 'stderr')
+        self._rmrf(self.run_dir)
+        self._mkpath(self.run_dir)
+
+    def start(self):
+        if not self.process:
+            self.cerr = open(self.err_file, 'w')
+            cmd = os.path.join(os.path.dirname(inspect.getfile(TestWebSockets)),
+                               'ws_server.py')
+            args = ['python3', cmd, '--port', str(self.port)]
+            if self.creds:
+                args.extend([
+                    '--cert', self.creds[0].cert_file,
+                    '--key', self.creds[0].pkey_file,
+                ])
+            self.process = subprocess.Popen(args=args, cwd=self.run_dir,
+                                            stderr=self.cerr, stdout=self.cerr)
+            if not self.check_alive():
+                self.stop()
+                pytest.fail(f'ws_server did not start. stderr={open(self.err_file).readlines()}')
+
+    def stop(self):
+        if self.process:
+            self.process.kill()
+            self.process.wait()
+            self.process = None
+        if self.cerr:
+            self.cerr.close()
+            self.cerr = None
+
+    def check_alive(self, timeout=5):
+        if self.creds:
+            url = f'https://localhost:{self.port}/'
+        else:
+            url = f'http://localhost:{self.port}/'
+        end = datetime.now() + timedelta(seconds=timeout)
+        while datetime.now() < end:
+            r = self.env.curl_get(url, 5)
+            if r.exit_code == 0:
+                return True
+            time.sleep(.1)
+        return False
+
+    def _mkpath(self, path):
+        if not os.path.exists(path):
+            return os.makedirs(path)
+
+    def _rmrf(self, path):
+        if os.path.exists(path):
+            return shutil.rmtree(path)
+
+
+
+class TestWebSockets:
+
+    @staticmethod
+    def mk_text_file(fpath: str, lines: int):
+        t110 = 11 * "0123456789"
+        with open(fpath, "w") as fd:
+            for i in range(lines):
+                fd.write("{0:015d}: ".format(i))  # total 128 bytes per line
+                fd.write(t110)
+                fd.write("\n")
+
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        # Apache config that CONNECT proxies a WebSocket server for paths starting
+        # with '/ws/'
+        # The WebSocket server is started in pytest fixture 'ws_server' below.
+        conf = TlsTestConf(env, extras={
+            'base': [
+                'Timeout 1',
+                f'<Proxy https://localhost:{env.wss_port}/>',
+                '    TLSProxyEngine on',
+                f'    TLSProxyCA {env.ca.cert_file}',
+                '    ProxyPreserveHost on',
+                '</Proxy>',
+            ],
+            'localhost': [
+                f'ProxyPass /ws/ http://127.0.0.1:{env.ws_port}/ upgrade=websocket \\',
+                f'timeout=2 flushpackets=on',
+                f'ProxyPass /wss/ https://localhost:{env.wss_port}/ upgrade=websocket \\',
+                f'timeout=2 flushpackets=on',
+            ],
+        })
+        conf.add_vhost('localhost', port=env.http_port)
+        conf.add_tls_vhosts(['localhost'], port=env.https_port)
+        conf.install()
+        TestWebSockets.mk_text_file(os.path.join(env.gen_dir, "1k.txt"), 8)
+        TestWebSockets.mk_text_file(os.path.join(env.gen_dir, "10k.txt"), 80)
+        TestWebSockets.mk_text_file(os.path.join(env.gen_dir, "100k.txt"), 800)
+        TestWebSockets.mk_text_file(os.path.join(env.gen_dir, "1m.txt"), 8000)
+        assert env.apache_restart() == 0
+
+    @pytest.fixture(autouse=True, scope='class')
+    def ws_server(self, env):
+        # Run our python websockets server that has some special behaviour
+        # for the different path to CONNECT to.
+        ws_server = WsServer('ws-server', env, port=env.ws_port)
+        ws_server.start()
+        yield ws_server
+        ws_server.stop()
+
+    @pytest.fixture(autouse=True, scope='class')
+    def wss_server(self, env):
+        # Run our python websockets server that has some special behaviour
+        # for the different path to CONNECT to.
+        creds = env.get_credentials_for_name('localhost')
+        assert creds
+        ws_server = WsServer('wss-server', env, port=env.wss_port, creds=creds)
+        ws_server.start()
+        yield ws_server
+        ws_server.stop()
+
+    def ssl_ctx(self, env):
+        ssl_ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_CLIENT)
+        ssl_ctx.load_verify_locations(cafile=env.ca.cert_file)
+        return ssl_ctx
+
+    def ws_recv_text(self, ws):
+        msg = ""
+        while True:
+            try:
+                msg += ws.recv()
+            except websockets.exceptions.ConnectionClosedOK:
+                return msg
+
+    def ws_recv_bytes(self, ws):
+        msg = b''
+        while True:
+            try:
+                msg += ws.recv()
+            except websockets.exceptions.ConnectionClosedOK:
+                return msg
+
+    # verify the our plain websocket server works
+    def test_tls_18_01_ws_direct(self, env, ws_server):
+        with connect(f"ws://127.0.0.1:{env.ws_port}/echo") as ws:
+            message = "Hello world!"
+            ws.send(message)
+            response = self.ws_recv_text(ws)
+            assert response == message
+
+    # verify that our secure websocket server works
+    def test_tls_18_02_wss_direct(self, env, wss_server):
+        pytest.skip(reason='For unknown reasons, this is flaky in CI')
+        with connect(f"wss://localhost:{env.wss_port}/echo",
+                     ssl_context=self.ssl_ctx(env)) as ws:
+            message = "Hello world!"
+            ws.send(message)
+            response = self.ws_recv_text(ws)
+            assert response == message
+
+    # verify to send plain websocket message pingpong through apache
+    def test_tls_18_03_http_ws(self, env, ws_server):
+        with connect(f"ws://localhost:{env.http_port}/ws/echo/") as ws:
+            message = "Hello world!"
+            ws.send(message)
+            response = self.ws_recv_text(ws)
+            assert response == message
+
+    # verify to send secure websocket message pingpong through apache
+    def test_tls_18_04_http_wss(self, env, wss_server):
+        pytest.skip(reason='This fails, needing a fix like PR #9')
+        with connect(f"ws://localhost:{env.http_port}/wss/echo/") as ws:
+            message = "Hello world!"
+            ws.send(message)
+            response = self.ws_recv_text(ws)
+            assert response == message
+
+    # verify that getting a large file works without any TLS involved
+    @pytest.mark.parametrize("fname", ["1m.txt"])
+    def test_tls_18_05_http_ws_file(self, env, fname, ws_server):
+        expected = open(os.path.join(env.gen_dir, fname), 'rb').read()
+        with connect(f"ws://localhost:{env.http_port}/ws/file/{fname}") as ws:
+            response = self.ws_recv_bytes(ws)
+            assert response == expected
+
+    # verify getting secure websocket from the http: server
+    # this is "backend" mod_tls work
+    @pytest.mark.parametrize("fname", ["1k.txt", "10k.txt", "100k.txt", "1m.txt"])
+    def test_tls_18_06_http_wss_file(self, env, fname, ws_server):
+        expected = open(os.path.join(env.gen_dir, fname), 'rb').read()
+        with connect(f"ws://localhost:{env.http_port}/wss/file/{fname}") as ws:
+            response = self.ws_recv_bytes(ws)
+            assert response == expected

test/modules/tls/ws_server.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import logging
+import os
+import ssl
+import sys
+import time
+
+import websockets.server as ws_server
+from websockets.exceptions import ConnectionClosedError
+
+log = logging.getLogger(__name__)
+
+logging.basicConfig(
+    format="[%(asctime)s] %(message)s",
+    level=logging.DEBUG,
+)
+
+
+async def echo(websocket):
+    try:
+        async for message in websocket:
+            try:
+                log.info(f'got request {message}')
+            except Exception as e:
+                log.error(f'error {e} getting path from {message}')
+            await websocket.send(message)
+    except ConnectionClosedError:
+        pass
+
+
+async def on_async_conn(conn):
+    rpath = str(conn.path)
+    pcomps = rpath[1:].split('/')
+    if len(pcomps) == 0:
+        pcomps = ['echo']  # default handler
+    log.info(f'connection for {pcomps}')
+    if pcomps[0] == 'echo':
+        log.info(f'/echo endpoint')
+        for message in await conn.recv():
+            await conn.send(message)
+    elif pcomps[0] == 'text':
+        await conn.send('hello!')
+    elif pcomps[0] == 'file':
+        if len(pcomps) < 2:
+            conn.close(code=4999, reason='unknown file')
+            return
+        fpath = os.path.join('../', pcomps[1])
+        if not os.path.exists(fpath):
+            conn.close(code=4999, reason='file not found')
+            return
+        bufsize = 0
+        if len(pcomps) > 2:
+            bufsize = int(pcomps[2])
+        if bufsize <= 0:
+            bufsize = 16*1024
+        delay_ms = 0
+        if len(pcomps) > 3:
+            delay_ms = int(pcomps[3])
+        n = 1
+        if len(pcomps) > 4:
+            n = int(pcomps[4])
+        for _ in range(n):
+            with open(fpath, 'r+b') as fd:
+                while True:
+                    buf = fd.read(bufsize)
+                    if buf is None or len(buf) == 0:
+                        break
+                    await conn.send(buf)
+                    if delay_ms > 0:
+                        time.sleep(delay_ms/1000)
+    else:
+        log.info(f'unknown endpoint: {rpath}')
+        await conn.close(code=4999, reason='path unknown')
+    await conn.close(code=1000, reason='')
+
+
+async def run_server(port, cert, pkey):
+    log.info(f'starting server on port {port}, cert {cert}, key {pkey}')
+    ssl_ctx = None
+    if cert and pkey:
+        ssl_ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_SERVER)
+        ssl_ctx.load_cert_chain(certfile=cert, keyfile=pkey)
+    async with ws_server.serve(ws_handler=on_async_conn,
+                               host="localhost", port=port, ssl=ssl_ctx):
+        await asyncio.Future()
+
+
+async def main():
+    parser = argparse.ArgumentParser(prog='scorecard',
+                                     description="Run a websocket echo server.")
+    parser.add_argument("--port", type=int,
+                        default=0, help="port to listen on")
+    parser.add_argument("--cert", type=str,
+                        default=None, help="TLS certificate")
+    parser.add_argument("--key", type=str,
+                        default=None, help="TLS private key")
+    args = parser.parse_args()
+
+    if args.port == 0:
+        sys.stderr.write('need --port\n')
+        sys.exit(1)
+
+    logging.basicConfig(
+        format="%(asctime)s %(message)s",
+        level=logging.DEBUG,
+    )
+    await run_server(args.port, args.cert, args.key)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

test/pyhttpd/config.ini.in

@@ -29,6 +29,7 @@ https_port = 5001
 proxy_port = 5003
 http_port2 = 5004
 ws_port = 5100
+wss_port = 5101
 http_tld = tests.httpd.apache.org
 test_dir = @abs_srcdir@
 test_src_dir = @abs_srcdir@