Merged with the default branch.1.23.0-1

14 files changed, 192 insertions, 36 deletions

diff --git a/test/conftest.py b/test/conftest.py
index b36aabad..20ac6e81 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,9 +1,12 @@
 import fcntl
+import inspect
+import json
 import os
 import platform
 import re
 import shutil
 import signal
+import socket
 import stat
 import subprocess
 import sys
@@ -16,6 +19,8 @@ from unit.check.go import check_go
 from unit.check.isolation import check_isolation
 from unit.check.node import check_node
 from unit.check.tls import check_openssl
+from unit.check.regex import check_regex
+from unit.http import TestHTTP
 from unit.option import option
 from unit.utils import public_dir
 from unit.utils import waitforfiles
@@ -51,10 +56,18 @@ def pytest_addoption(parser):
         type=str,
         help="Default user for non-privileged processes of unitd",
     )
+    parser.addoption(
+        "--restart",
+        default=False,
+        action="store_true",
+        help="Force Unit to restart after every test",
+    )
 
 
 unit_instance = {}
+unit_log_copy = "unit.log.copy"
 _processes = []
+http = TestHTTP()
 
 def pytest_configure(config):
     option.config = config.option
@@ -64,6 +77,7 @@ def pytest_configure(config):
     option.save_log = config.option.save_log
     option.unsafe = config.option.unsafe
     option.user = config.option.user
+    option.restart = config.option.restart
 
     option.generated_tests = {}
     option.current_dir = os.path.abspath(
@@ -163,6 +177,7 @@ def pytest_sessionstart(session):
         option.current_dir, unit['temp_dir'], option.test_dir
     )
     option.available['modules']['node'] = check_node(option.current_dir)
+    option.available['modules']['regex'] = check_regex(unit['unitd'])
 
     # remove None values
 
@@ -172,12 +187,17 @@ def pytest_sessionstart(session):
 
     check_isolation()
 
+    _clear_conf(unit['temp_dir'] + '/control.unit.sock')
+
     unit_stop()
 
     _check_alerts()
 
-    shutil.rmtree(unit_instance['temp_dir'])
+    if option.restart:
+        shutil.rmtree(unit_instance['temp_dir'])
 
+    elif option.save_log:
+        open(unit_instance['temp_dir'] + '/' + unit_log_copy, 'w').close()
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item, call):
@@ -241,38 +261,70 @@ def run(request):
 
     # stop unit
 
-    error = unit_stop()
+    error_stop_unit = unit_stop()
+    error_stop_processes = stop_processes()
 
-    if error:
-        _print_log()
+    # prepare log
 
-    assert error is None, 'stop unit'
+    with open(
+        unit_instance['log'], 'r', encoding='utf-8', errors='ignore'
+    ) as f:
+        log = f.read()
 
-    # stop all processes
+    if not option.restart and option.save_log:
+        with open(unit_instance['temp_dir'] + '/' + unit_log_copy, 'a') as f:
+            f.write(log)
 
-    error = stop_processes()
+    # remove unit.log
 
-    if error:
-        _print_log()
+    if not option.save_log and option.restart:
+        shutil.rmtree(unit['temp_dir'])
 
-    assert error is None, 'stop unit'
+    # clean temp_dir before the next test
 
-    # check unit.log for alerts
+    if not option.restart:
+        _clear_conf(unit['temp_dir'] + '/control.unit.sock', log)
 
-    _check_alerts()
+        open(unit['log'], 'w').close()
+
+        for item in os.listdir(unit['temp_dir']):
+            if item not in [
+                'control.unit.sock',
+                'state',
+                'unit.pid',
+                'unit.log',
+                unit_log_copy,
+            ]:
+                path = os.path.join(unit['temp_dir'], item)
+
+                public_dir(path)
+
+                if os.path.isfile(path) or stat.S_ISSOCK(os.stat(path).st_mode):
+                    os.remove(path)
+                else:
+                    shutil.rmtree(path)
 
     # print unit.log in case of error
 
     if hasattr(request.node, 'rep_call') and request.node.rep_call.failed:
-        _print_log()
+        _print_log(log)
 
-    # remove unit.log
+    if error_stop_unit or error_stop_processes:
+        _print_log(log)
 
-    if not option.save_log:
-        shutil.rmtree(unit['temp_dir'])
+    # check unit.log for errors
+
+    assert error_stop_unit is None, 'stop unit'
+    assert error_stop_processes is None, 'stop processes'
+
+    _check_alerts(log=log)
 
 def unit_run():
     global unit_instance
+
+    if not option.restart and 'unitd' in unit_instance:
+        return unit_instance
+
     build_dir = option.current_dir + '/build'
     unitd = build_dir + '/unitd'
 
@@ -323,6 +375,12 @@ def unit_run():
 
 
 def unit_stop():
+    if not option.restart:
+        if inspect.stack()[1].function.startswith('test_'):
+            pytest.skip('no restart mode')
+
+        return
+
     p = unit_instance['process']
 
     if p.poll() is not None:
@@ -345,12 +403,13 @@ def unit_stop():
 
 
 
-def _check_alerts(path=None):
+def _check_alerts(path=None, log=None):
     if path is None:
         path = unit_instance['log']
 
-    with open(path, 'r', encoding='utf-8', errors='ignore') as f:
-        log = f.read()
+    if log is None:
+        with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+            log = f.read()
 
     found = False
 
@@ -396,6 +455,43 @@ def _print_log(data=None):
             sys.stdout.write(data)
 
 
+def _clear_conf(sock, log=None):
+    def check_success(resp):
+        if 'success' not in resp:
+            _print_log(log)
+            assert 'success' in resp
+
+    resp = http.put(
+        url='/config',
+        sock_type='unix',
+        addr=sock,
+        body=json.dumps({"listeners": {}, "applications": {}}),
+    )['body']
+
+    check_success(resp)
+
+    if 'openssl' not in option.available['modules']:
+        return
+
+    try:
+        certs = json.loads(http.get(
+            url='/certificates',
+            sock_type='unix',
+            addr=sock,
+        )['body']).keys()
+
+    except json.JSONDecodeError:
+        pytest.fail('Can\'t parse certificates list.')
+
+    for cert in certs:
+        resp = http.delete(
+            url='/certificates/' + cert,
+            sock_type='unix',
+            addr=sock,
+        )['body']
+
+        check_success(resp)
+
 def run_process(target, *args):
     global _processes
 
@@ -446,5 +542,10 @@ def unit_pid(request):
     return unit_instance['process'].pid
 
 def pytest_sessionfinish(session):
+    if not option.restart and option.save_log:
+        print('Path to unit.log:\n' + unit_instance['log'] + '\n')
+
+    option.restart = True
+
     unit_stop()
     shutil.rmtree(option.cache_dir)
diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py
index 5770265d..886a160b 100644
--- a/test/test_asgi_application.py
+++ b/test/test_asgi_application.py
@@ -377,7 +377,7 @@ Connection: close
             self.get(no_recv=True)
 
         assert (
-            self.wait_for_record(r'\(5\) Thread: 100') is not None
+            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
         ), 'last thread finished'
 
     def test_asgi_application_threads(self):
diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py
index 6121fcc5..7218d526 100644
--- a/test/test_asgi_websockets.py
+++ b/test/test_asgi_websockets.py
@@ -30,8 +30,9 @@ class TestASGIWebsockets(TestApplicationPython):
 
         self.check_close(sock)
 
-    def check_close(self, sock, code=1000, no_close=False):
-        frame = self.ws.frame_read(sock)
+    def check_close(self, sock, code=1000, no_close=False, frame=None):
+        if frame == None:
+            frame = self.ws.frame_read(sock)
 
         assert frame['fin'] == True, 'close fin'
         assert frame['opcode'] == self.ws.OP_CLOSE, 'close opcode'
@@ -930,7 +931,14 @@ class TestASGIWebsockets(TestApplicationPython):
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment2', fin=True)
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment3', fin=False)
         self.ws.frame_write(sock, self.ws.OP_TEXT, 'fragment4', fin=True)
-        self.check_close(sock, 1002)
+
+        frame = self.ws.frame_read(sock)
+
+        if frame['opcode'] == self.ws.OP_TEXT:
+            self.check_frame(frame, True, self.ws.OP_TEXT, 'fragment1fragment2')
+            frame = None
+
+        self.check_close(sock, 1002, frame=frame)
 
         # 5_16
 
diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py
index 315c496d..df9e0885 100644
--- a/test/test_java_websockets.py
+++ b/test/test_java_websockets.py
@@ -27,8 +27,9 @@ class TestJavaWebsockets(TestApplicationJava):
 
         self.check_close(sock)
 
-    def check_close(self, sock, code=1000, no_close=False):
-        frame = self.ws.frame_read(sock)
+    def check_close(self, sock, code=1000, no_close=False, frame=None):
+        if frame == None:
+            frame = self.ws.frame_read(sock)
 
         assert frame['fin'] == True, 'close fin'
         assert frame['opcode'] == self.ws.OP_CLOSE, 'close opcode'
@@ -862,7 +863,14 @@ class TestJavaWebsockets(TestApplicationJava):
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment2', fin=True)
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment3', fin=False)
         self.ws.frame_write(sock, self.ws.OP_TEXT, 'fragment4', fin=True)
-        self.check_close(sock, 1002)
+
+        frame = self.ws.frame_read(sock)
+
+        if frame['opcode'] == self.ws.OP_TEXT:
+            self.check_frame(frame, True, self.ws.OP_TEXT, 'fragment1fragment2')
+            frame = None
+
+        self.check_close(sock, 1002, frame=frame)
 
         # 5_16
 
diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py
index 4f1e5906..d5f79811 100644
--- a/test/test_node_websockets.py
+++ b/test/test_node_websockets.py
@@ -27,8 +27,9 @@ class TestNodeWebsockets(TestApplicationNode):
 
         self.check_close(sock)
 
-    def check_close(self, sock, code=1000, no_close=False):
-        frame = self.ws.frame_read(sock)
+    def check_close(self, sock, code=1000, no_close=False, frame=None):
+        if frame == None:
+            frame = self.ws.frame_read(sock)
 
         assert frame['fin'] == True, 'close fin'
         assert frame['opcode'] == self.ws.OP_CLOSE, 'close opcode'
@@ -881,7 +882,14 @@ class TestNodeWebsockets(TestApplicationNode):
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment2', fin=True)
         self.ws.frame_write(sock, self.ws.OP_CONT, 'fragment3', fin=False)
         self.ws.frame_write(sock, self.ws.OP_TEXT, 'fragment4', fin=True)
-        self.check_close(sock, 1002)
+
+        frame = self.ws.frame_read(sock)
+
+        if frame['opcode'] == self.ws.OP_TEXT:
+            self.check_frame(frame, True, self.ws.OP_TEXT, 'fragment1fragment2')
+            frame = None
+
+        self.check_close(sock, 1002, frame=frame)
 
         # 5_16
 
diff --git a/test/test_proxy.py b/test/test_proxy.py
index 2d305e98..7e7c7246 100644
--- a/test/test_proxy.py
+++ b/test/test_proxy.py
@@ -482,6 +482,7 @@ Content-Length: 10
         check_proxy('http://[:]:7080')
         check_proxy('http://[::7080')
 
+    @pytest.mark.skip('not yet')
     def test_proxy_loop(self, skip_alert):
         skip_alert(
             r'socket.*failed',
diff --git a/test/test_python_application.py b/test/test_python_application.py
index 709df3ff..41f6f538 100644
--- a/test/test_python_application.py
+++ b/test/test_python_application.py
@@ -569,7 +569,7 @@ last line: 987654321
             self.get(no_recv=True)
 
         assert (
-            self.wait_for_record(r'\(5\) Thread: 100') is not None
+            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
         ), 'last thread finished'
 
     def test_python_application_iter_exception(self):
diff --git a/test/test_routing.py b/test/test_routing.py
index 4d27cb61..be9a1faf 100644
--- a/test/test_routing.py
+++ b/test/test_routing.py
@@ -229,6 +229,9 @@ class TestRouting(TestApplicationProto):
         assert self.get(url='/ABc')['status'] == 404
 
     def test_routes_empty_regex(self):
+        if not option.available['modules']['regex']:
+            pytest.skip('requires regex')
+
         self.route_match({"uri":"~"})
         assert self.get(url='/')['status'] == 200, 'empty regexp'
         assert self.get(url='/anything')['status'] == 200, '/anything'
@@ -238,6 +241,9 @@ class TestRouting(TestApplicationProto):
         assert self.get(url='/nothing')['status'] == 404, '/nothing'
 
     def test_routes_bad_regex(self):
+        if not option.available['modules']['regex']:
+            pytest.skip('requires regex')
+
         assert 'error' in self.route(
             {"match": {"uri": "~/bl[ah"}, "action": {"return": 200}}
         ), 'bad regex'
@@ -255,6 +261,9 @@ class TestRouting(TestApplicationProto):
             assert self.get(url='/nothing_z')['status'] == 500, '/nothing_z'
 
     def test_routes_match_regex_case_sensitive(self):
+        if not option.available['modules']['regex']:
+            pytest.skip('requires regex')
+
         self.route_match({"uri": "~/bl[ah]"})
 
         assert self.get(url='/rlah')['status'] == 404, '/rlah'
@@ -263,6 +272,9 @@ class TestRouting(TestApplicationProto):
         assert self.get(url='/BLAH')['status'] == 404, '/BLAH'
 
     def test_routes_match_regex_negative_case_sensitive(self):
+        if not option.available['modules']['regex']:
+            pytest.skip('requires regex')
+
         self.route_match({"uri": "!~/bl[ah]"})
 
         assert self.get(url='/rlah')['status'] == 200, '/rlah'
diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py
index a02cb1a3..46464670 100644
--- a/test/test_share_fallback.py
+++ b/test/test_share_fallback.py
@@ -1,5 +1,6 @@
 import os
 
+import pytest
 from unit.applications.proto import TestApplicationProto
 from unit.option import option
 
@@ -27,7 +28,10 @@ class TestStatic(TestApplicationProto):
         )
 
     def teardown_method(self):
-        os.chmod(option.temp_dir + '/assets/403', 0o777)
+        try:
+            os.chmod(option.temp_dir + '/assets/403', 0o777)
+        except FileNotFoundError:
+            pass
 
     def action_update(self, conf):
         assert 'success' in self.conf(conf, 'routes/0/action')
@@ -116,6 +120,7 @@ class TestStatic(TestApplicationProto):
         assert resp['status'] == 200, 'fallback proxy status'
         assert resp['body'] == '', 'fallback proxy'
 
+    @pytest.mark.skip('not yet')
     def test_fallback_proxy_loop(self, skip_alert):
         skip_alert(
             r'open.*/blah/index.html.*failed',
diff --git a/test/test_tls.py b/test/test_tls.py
index 89c57d07..206ea828 100644
--- a/test/test_tls.py
+++ b/test/test_tls.py
@@ -199,7 +199,7 @@ class TestTLS(TestApplicationTLS):
                 self.sec_epoch()
                 - self.openssl_date_to_sec_epoch(cert['validity']['since'])
             )
-            < 5
+            < 60
         ), 'certificate validity since'
         assert (
             self.openssl_date_to_sec_epoch(cert['validity']['until'])
diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py
index b2e17f23..c9c2095e 100644
--- a/test/unit/applications/lang/java.py
+++ b/test/unit/applications/lang/java.py
@@ -52,7 +52,7 @@ class TestApplicationJava(TestApplicationProto):
                 os.makedirs(classes_path)
 
             classpath = (
-                option.current_dir + '/build/tomcat-servlet-api-9.0.39.jar'
+                option.current_dir + '/build/tomcat-servlet-api-9.0.44.jar'
             )
 
             ws_jars = glob.glob(
diff --git a/test/unit/applications/proto.py b/test/unit/applications/proto.py
index af05d071..5c400621 100644
--- a/test/unit/applications/proto.py
+++ b/test/unit/applications/proto.py
@@ -19,8 +19,8 @@ class TestApplicationProto(TestControl):
         with open(option.temp_dir + '/' + name, 'r', errors='ignore') as f:
             return re.search(pattern, f.read())
 
-    def wait_for_record(self, pattern, name='unit.log'):
-        for i in range(50):
+    def wait_for_record(self, pattern, name='unit.log', wait=150):
+        for i in range(wait):
             found = self.search_in_log(pattern, name)
 
             if found is not None:
diff --git a/test/unit/check/regex.py b/test/unit/check/regex.py
new file mode 100644
index 00000000..734c0150
--- /dev/null
+++ b/test/unit/check/regex.py
@@ -0,0 +1,13 @@
+import re
+import subprocess
+
+
+def check_regex(unitd):
+    output = subprocess.check_output(
+        [unitd, '--version'], stderr=subprocess.STDOUT
+    )
+
+    if re.search('--no-regex', output.decode()):
+        return False
+
+    return True
diff --git a/test/unit/utils.py b/test/unit/utils.py
index 7a0a3fe5..e80fc469 100644
--- a/test/unit/utils.py
+++ b/test/unit/utils.py
@@ -47,7 +47,7 @@ def waitforsocket(port):
             except KeyboardInterrupt:
                 raise
 
-    pytest.fail('Can\'t connect to the 127.0.0.1:' + port)
+    pytest.fail('Can\'t connect to the 127.0.0.1:' + str(port))
 
 
 def findmnt():