From 950a28f5eac81a35a23bc6b67b95615fcd046885 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Thu, 18 Mar 2021 22:48:13 +0100 Subject: [PATCH 01/11] chore(ci): :arrow_up: porting python 2.7 code to python 3.9 --- test/conftest.py | 32 ++++++++++--------- .../Dockerfile-nginx-proxy-tester | 2 +- .../test_restart_while_missing_cert.py | 4 +-- test/test_custom/test_location-per-vhost.py | 2 +- test/test_dockergen/test_dockergen_v2.py | 22 +++++++------ test/test_dockergen/test_dockergen_v3.py | 32 +++++++++++-------- test/test_events.py | 2 +- test/test_ssl/test_dhparam.py | 14 ++++---- test/test_ssl/test_dhparam_generation.py | 2 +- .../test_wildcard_cert_nohttps.py | 2 +- 10 files changed, 63 insertions(+), 51 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index a9bead2..0195712 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,3 @@ -from __future__ import print_function import contextlib import logging import os @@ -142,7 +141,7 @@ def container_ip(container): return net_info["bridge"]["IPAddress"] # not default bridge network, fallback on first network defined - network_name = net_info.keys()[0] + network_name = list(net_info.keys())[0] return net_info[network_name]["IPAddress"] @@ -155,7 +154,7 @@ def container_ipv6(container): return net_info["bridge"]["GlobalIPv6Address"] # not default bridge network, fallback on first network defined - network_name = net_info.keys()[0] + network_name = list(net_info.keys())[0] return net_info[network_name]["GlobalIPv6Address"] @@ -188,7 +187,7 @@ def docker_container_dns_resolver(domain_name): log = logging.getLogger('DNS') log.debug("docker_container_dns_resolver(%r)" % domain_name) - match = re.search('(^|.+\.)(?P[^.]+)\.container\.docker$', domain_name) + match = re.search(r'(^|.+\.)(?P[^.]+)\.container\.docker$', domain_name) if not match: log.debug("%r does not match" % domain_name) return @@ -253,9 +252,12 @@ def get_nginx_conf_from_container(container): return the nginx /etc/nginx/conf.d/default.conf file content from a container """ import tarfile - from cStringIO import StringIO - strm, stat = container.get_archive('/etc/nginx/conf.d/default.conf') - with tarfile.open(fileobj=StringIO(strm.read())) as tf: + from io import BytesIO + + strm_generator, stat = container.get_archive('/etc/nginx/conf.d/default.conf') + strm_fileobj = BytesIO(b"".join(strm_generator)) + + with tarfile.open(fileobj=strm_fileobj) as tf: conffile = tf.extractfile('default.conf') return conffile.read() @@ -264,7 +266,7 @@ def docker_compose_up(compose_file='docker-compose.yml'): logging.info('docker-compose -f %s up -d' % compose_file) try: subprocess.check_output(shlex.split('docker-compose -f %s up -d' % compose_file), stderr=subprocess.STDOUT) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: pytest.fail("Error while runninng 'docker-compose -f %s up -d':\n%s" % (compose_file, e.output), pytrace=False) @@ -272,7 +274,7 @@ def docker_compose_down(compose_file='docker-compose.yml'): logging.info('docker-compose -f %s down' % compose_file) try: subprocess.check_output(shlex.split('docker-compose -f %s down' % compose_file), stderr=subprocess.STDOUT) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: pytest.fail("Error while runninng 'docker-compose -f %s down':\n%s" % (compose_file, e.output), pytrace=False) @@ -286,7 +288,7 @@ def wait_for_nginxproxy_to_be_ready(): return container = containers[0] for line in container.logs(stream=True): - if "Watching docker events" in line: + if b"Watching docker events" in line: logging.debug("nginx-proxy ready") break @@ -337,7 +339,7 @@ def connect_to_network(network): return # figure out our container networks - my_networks = my_container.attrs["NetworkSettings"]["Networks"].keys() + my_networks = list(my_container.attrs["NetworkSettings"]["Networks"].keys()) # make sure our container is connected to the nginx-proxy's network if network not in my_networks: @@ -360,7 +362,7 @@ def disconnect_from_network(network=None): return # figure out our container networks - my_networks_names = my_container.attrs["NetworkSettings"]["Networks"].keys() + my_networks_names = list(my_container.attrs["NetworkSettings"]["Networks"].keys()) # disconnect our container from the given network if network.name in my_networks_names: @@ -378,7 +380,7 @@ def connect_to_all_networks(): return [] else: # find the list of docker networks - networks = filter(lambda network: len(network.containers) > 0 and network.name != 'bridge', docker_client.networks.list()) + networks = [network for network in docker_client.networks.list() if len(network.containers) > 0 and network.name != 'bridge'] return [connect_to_network(network) for network in networks] @@ -388,7 +390,7 @@ def connect_to_all_networks(): # ############################################################################### -@pytest.yield_fixture(scope="module") +@pytest.fixture(scope="module") def docker_compose(request): """ pytest fixture providing containers described in a docker compose file. After the tests, remove the created containers @@ -412,7 +414,7 @@ def docker_compose(request): restore_urllib_dns_resolver(original_dns_resolver) -@pytest.yield_fixture() +@pytest.fixture() def nginxproxy(): """ Provides the `nginxproxy` object that can be used in the same way the requests module is: diff --git a/test/requirements/Dockerfile-nginx-proxy-tester b/test/requirements/Dockerfile-nginx-proxy-tester index 27d0538..6c0f060 100644 --- a/test/requirements/Dockerfile-nginx-proxy-tester +++ b/test/requirements/Dockerfile-nginx-proxy-tester @@ -1,4 +1,4 @@ -FROM python:2.7-alpine +FROM python:3.9-alpine # Note: we're using alpine because it has openssl 1.0.2, which we need for testing RUN apk add --update bash openssl curl && rm -rf /var/cache/apk/* diff --git a/test/stress_tests/test_deleted_cert/test_restart_while_missing_cert.py b/test/stress_tests/test_deleted_cert/test_restart_while_missing_cert.py index 2b74acd..0ec36c7 100644 --- a/test/stress_tests/test_deleted_cert/test_restart_while_missing_cert.py +++ b/test/stress_tests/test_deleted_cert/test_restart_while_missing_cert.py @@ -12,7 +12,7 @@ script_dir = os.path.dirname(__file__) pytestmark = pytest.mark.xfail() # TODO delete this marker once those issues are fixed -@pytest.yield_fixture(scope="module", autouse=True) +@pytest.fixture(scope="module", autouse=True) def certs(): """ pytest fixture that provides cert and key files into the tmp_certs directory @@ -43,7 +43,7 @@ def test_http_web_is_301(docker_compose, nginxproxy): def test_https_web_is_200(docker_compose, nginxproxy): r = nginxproxy.get("https://web.nginx-proxy/port") assert r.status_code == 200 - assert 'answer from port 81\n' in r.text + assert "answer from port 81\n" in r.text @pytest.mark.incremental diff --git a/test/test_custom/test_location-per-vhost.py b/test/test_custom/test_location-per-vhost.py index b99996e..f67b501 100644 --- a/test/test_custom/test_location-per-vhost.py +++ b/test/test_custom/test_location-per-vhost.py @@ -19,4 +19,4 @@ def test_custom_conf_does_not_apply_to_web2(docker_compose, nginxproxy): assert "X-test" not in r.headers def test_custom_block_is_present_in_nginx_generated_conf(docker_compose, nginxproxy): - assert "include /etc/nginx/vhost.d/web1.nginx-proxy.local_location;" in nginxproxy.get_conf() \ No newline at end of file + assert b"include /etc/nginx/vhost.d/web1.nginx-proxy.local_location;" in nginxproxy.get_conf() \ No newline at end of file diff --git a/test/test_dockergen/test_dockergen_v2.py b/test/test_dockergen/test_dockergen_v2.py index af02649..a3f2484 100644 --- a/test/test_dockergen/test_dockergen_v2.py +++ b/test/test_dockergen/test_dockergen_v2.py @@ -4,7 +4,7 @@ import logging import pytest -@pytest.yield_fixture(scope="module") +@pytest.fixture(scope="module") def nginx_tmpl(): """ pytest fixture which extracts the the nginx config template from @@ -13,14 +13,18 @@ def nginx_tmpl(): script_dir = os.path.dirname(__file__) logging.info("extracting nginx.tmpl from nginxproxy/nginx-proxy:test") docker_client = docker.from_env() - print(docker_client.containers.run( - image='nginxproxy/nginx-proxy:test', - remove=True, - volumes=['{current_dir}:{current_dir}'.format(current_dir=script_dir)], - entrypoint='sh', - command='-xc "cp /app/nginx.tmpl {current_dir} && chmod 777 {current_dir}/nginx.tmpl"'.format( - current_dir=script_dir), - stderr=True)) + print( + docker_client.containers.run( + image="nginxproxy/nginx-proxy:test", + remove=True, + volumes=["{current_dir}:{current_dir}".format(current_dir=script_dir)], + entrypoint="sh", + command='-xc "cp /app/nginx.tmpl {current_dir} && chmod 777 {current_dir}/nginx.tmpl"'.format( + current_dir=script_dir + ), + stderr=True, + ) + ) yield logging.info("removing nginx.tmpl") os.remove(os.path.join(script_dir, "nginx.tmpl")) diff --git a/test/test_dockergen/test_dockergen_v3.py b/test/test_dockergen/test_dockergen_v3.py index 453889a..1beffeb 100644 --- a/test/test_dockergen/test_dockergen_v3.py +++ b/test/test_dockergen/test_dockergen_v3.py @@ -18,16 +18,18 @@ def versiontuple(v): >>> versiontuple("17.03.0-ce") < (1, 13) False """ - return tuple(map(int, (v.split('-')[0].split(".")))) + return tuple(map(int, (v.split("-")[0].split(".")))) -raw_version = docker.from_env().version()['Version'] +raw_version = docker.from_env().version()["Version"] pytestmark = pytest.mark.skipif( versiontuple(raw_version) < (1, 13), - reason="Docker compose syntax v3 requires docker engine v1.13 or later (got %s)" % raw_version) + reason="Docker compose syntax v3 requires docker engine v1.13 or later (got %s)" + % raw_version, +) -@pytest.yield_fixture(scope="module") +@pytest.fixture(scope="module") def nginx_tmpl(): """ pytest fixture which extracts the the nginx config template from @@ -36,14 +38,18 @@ def nginx_tmpl(): script_dir = os.path.dirname(__file__) logging.info("extracting nginx.tmpl from nginxproxy/nginx-proxy:test") docker_client = docker.from_env() - print(docker_client.containers.run( - image='nginxproxy/nginx-proxy:test', - remove=True, - volumes=['{current_dir}:{current_dir}'.format(current_dir=script_dir)], - entrypoint='sh', - command='-xc "cp /app/nginx.tmpl {current_dir} && chmod 777 {current_dir}/nginx.tmpl"'.format( - current_dir=script_dir), - stderr=True)) + print( + docker_client.containers.run( + image="nginxproxy/nginx-proxy:test", + remove=True, + volumes=["{current_dir}:{current_dir}".format(current_dir=script_dir)], + entrypoint="sh", + command='-xc "cp /app/nginx.tmpl {current_dir} && chmod 777 {current_dir}/nginx.tmpl"'.format( + current_dir=script_dir + ), + stderr=True, + ) + ) yield logging.info("removing nginx.tmpl") os.remove(os.path.join(script_dir, "nginx.tmpl")) @@ -61,6 +67,6 @@ def test_forwards_to_whoami(nginx_tmpl, docker_compose, nginxproxy): assert r.text == "I'm %s\n" % whoami_container.id[:12] -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/test/test_events.py b/test/test_events.py index fa97f84..201917f 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -7,7 +7,7 @@ import pytest from docker.errors import NotFound -@pytest.yield_fixture() +@pytest.fixture() def web1(docker_compose): """ pytest fixture creating a web container with `VIRTUAL_HOST=web1.nginx-proxy` listening on port 81. diff --git a/test/test_ssl/test_dhparam.py b/test/test_ssl/test_dhparam.py index 40339a1..8899c6a 100644 --- a/test/test_ssl/test_dhparam.py +++ b/test/test_ssl/test_dhparam.py @@ -26,7 +26,7 @@ def assert_log_contains(expected_log_line): """ sut_container = docker_client.containers.get("nginxproxy") docker_logs = sut_container.logs(stdout=True, stderr=True, stream=False, follow=False) - assert expected_log_line in docker_logs + assert bytes(expected_log_line, encoding="utf8") in docker_logs def require_openssl(required_version): @@ -42,7 +42,7 @@ def require_openssl(required_version): """ def versiontuple(v): - clean_v = re.sub("[^\d\.]", "", v) + clean_v = re.sub(r"[^\d\.]", "", v) return tuple(map(int, (clean_v.split(".")))) try: @@ -52,7 +52,7 @@ def require_openssl(required_version): else: if not command_output: raise Exception("Could not get openssl version") - openssl_version = command_output.split()[1] + openssl_version = str(command_output.split()[1]) return pytest.mark.skipif( versiontuple(openssl_version) < versiontuple(required_version), reason="openssl v%s is less than required version %s" % (openssl_version, required_version)) @@ -71,8 +71,8 @@ def test_dhparam_is_not_generated_if_present(docker_compose): assert_log_contains("Custom dhparam.pem file found, generation skipped") # Make sure the dhparam in use is not the default, pre-generated one - default_checksum = sut_container.exec_run("md5sum /app/dhparam.pem.default").split() - current_checksum = sut_container.exec_run("md5sum /etc/nginx/dhparam/dhparam.pem").split() + default_checksum = sut_container.exec_run("md5sum /app/dhparam.pem.default").output.split() + current_checksum = sut_container.exec_run("md5sum /etc/nginx/dhparam/dhparam.pem").output.split() assert default_checksum[0] != current_checksum[0] @@ -89,5 +89,5 @@ def test_web5_dhparam_is_used(docker_compose): host = "%s:443" % sut_container.attrs["NetworkSettings"]["IPAddress"] r = subprocess.check_output( - "echo '' | openssl s_client -connect %s -cipher 'EDH' | grep 'Server Temp Key'" % host, shell=True) - assert "Server Temp Key: X25519, 253 bits\n" == r + f"echo '' | openssl s_client -connect {host} -cipher 'EDH' | grep 'Server Temp Key'", shell=True) + assert b"Server Temp Key: X25519, 253 bits\n" == r diff --git a/test/test_ssl/test_dhparam_generation.py b/test/test_ssl/test_dhparam_generation.py index 0f5398b..4ba1c53 100644 --- a/test/test_ssl/test_dhparam_generation.py +++ b/test/test_ssl/test_dhparam_generation.py @@ -22,7 +22,7 @@ def assert_log_contains(expected_log_line): """ sut_container = docker_client.containers.get("nginxproxy") docker_logs = sut_container.logs(stdout=True, stderr=True, stream=False, follow=False) - assert expected_log_line in docker_logs + assert bytes(expected_log_line, encoding="utf8") in docker_logs ############################################################################### diff --git a/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py b/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py index 2808dee..1946cc0 100644 --- a/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py +++ b/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py @@ -1,5 +1,5 @@ import pytest -from backports.ssl_match_hostname import CertificateError +from ssl import CertificateError from requests.exceptions import SSLError From eba9ac42610290b93e2ab41dcf367d929bb30140 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Thu, 18 Mar 2021 22:48:49 +0100 Subject: [PATCH 02/11] chore(ci): :arrow_up: update python dependencies to latests release --- test/conftest.py | 4 ++-- test/requirements/python-requirements.txt | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 0195712..0d7a488 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -471,5 +471,5 @@ try: except docker.errors.ImageNotFound: pytest.exit("The docker image 'nginxproxy/nginx-proxy:test' is missing") -if docker.__version__ != "2.1.0": - pytest.exit("This test suite is meant to work with the python docker module v2.1.0") +if docker.__version__ != "4.4.4": + pytest.exit("This test suite is meant to work with the python docker module v4.4.4") diff --git a/test/requirements/python-requirements.txt b/test/requirements/python-requirements.txt index ba95455..11f8665 100644 --- a/test/requirements/python-requirements.txt +++ b/test/requirements/python-requirements.txt @@ -1,5 +1,5 @@ -backoff==1.3.2 -docker-compose==1.11.2 -docker==2.1.0 -pytest==3.0.5 -requests==2.11.1 +backoff==1.10.0 +docker-compose==1.28.5 +docker==4.4.4 +pytest==6.2.2 +requests==2.25.1 From 37e85e6e8d8aa413f3d242585c37e98abdbfb018 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 12:12:24 +0100 Subject: [PATCH 03/11] chore(ci): :recycle: convert Python old `%` string to f-strings --- test/conftest.py | 50 +++++++++---------- test/requirements/web/webserver.py | 6 +-- test/test_dockergen/test_dockergen_v2.py | 2 +- test/test_dockergen/test_dockergen_v3.py | 5 +- test/test_ssl/test_dhparam.py | 4 +- test/test_ssl/test_wildcard.py | 8 +-- .../test_wildcard_cert_nohttps.py | 10 ++-- test/test_wildcard_host.py | 6 +-- 8 files changed, 45 insertions(+), 46 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 0d7a488..aa398e6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -132,7 +132,7 @@ def container_ip(container): pytest.skip("This system does not support IPv6") ip = container_ipv6(container) if ip == '': - pytest.skip("Container %s has no IPv6 address" % container.name) + pytest.skip(f"Container {container.name} has no IPv6 address") else: return ip else: @@ -166,15 +166,15 @@ def nginx_proxy_dns_resolver(domain_name): :return: IP or None """ log = logging.getLogger('DNS') - log.debug("nginx_proxy_dns_resolver(%r)" % domain_name) + log.debug(f"nginx_proxy_dns_resolver({domain_name!r})") if 'nginx-proxy' in domain_name: nginxproxy_containers = docker_client.containers.list(filters={"status": "running", "ancestor": "nginxproxy/nginx-proxy:test"}) if len(nginxproxy_containers) == 0: - log.warn("no container found from image nginxproxy/nginx-proxy:test while resolving %r", domain_name) + log.warn(f"no container found from image nginxproxy/nginx-proxy:test while resolving {domain_name!r}") return nginxproxy_container = nginxproxy_containers[0] ip = container_ip(nginxproxy_container) - log.info("resolving domain name %r as IP address %s of nginx-proxy container %s" % (domain_name, ip, nginxproxy_container.name)) + log.info(f"resolving domain name {domain_name!r} as IP address {ip} of nginx-proxy container {nginxproxy_container.name}") return ip def docker_container_dns_resolver(domain_name): @@ -185,24 +185,24 @@ def docker_container_dns_resolver(domain_name): :return: IP or None """ log = logging.getLogger('DNS') - log.debug("docker_container_dns_resolver(%r)" % domain_name) + log.debug(f"docker_container_dns_resolver({domain_name!r})") match = re.search(r'(^|.+\.)(?P[^.]+)\.container\.docker$', domain_name) if not match: - log.debug("%r does not match" % domain_name) + log.debug(f"{domain_name!r} does not match") return container_name = match.group('container') - log.debug("looking for container %r" % container_name) + log.debug(f"looking for container {container_name!r}") try: container = docker_client.containers.get(container_name) except docker.errors.NotFound: - log.warn("container named %r not found while resolving %r" % (container_name, domain_name)) + log.warn(f"container named {container_name!r} not found while resolving {domain_name!r}") return - log.debug("container %r found (%s)" % (container.name, container.short_id)) + log.debug(f"container {container.name!r} found ({container.short_id})") ip = container_ip(container) - log.info("resolving domain name %r as IP address %s of container %s" % (domain_name, ip, container.name)) + log.info(f"resolving domain name {domain_name!r} as IP address {ip} of container {container.name}") return ip @@ -215,7 +215,7 @@ def monkey_patch_urllib_dns_resolver(): prv_getaddrinfo = socket.getaddrinfo dns_cache = {} def new_getaddrinfo(*args): - logging.getLogger('DNS').debug("resolving domain name %s" % repr(args)) + logging.getLogger('DNS').debug(f"resolving domain name {repr(args)}") _args = list(args) # custom DNS resolvers @@ -243,7 +243,7 @@ def remove_all_containers(): for container in docker_client.containers.list(all=True): if I_AM_RUNNING_INSIDE_A_DOCKER_CONTAINER and container.id.startswith(socket.gethostname()): continue # pytest is running within a Docker container, so we do not want to remove that particular container - logging.info("removing container %s" % container.name) + logging.info(f"removing container {container.name}") container.remove(v=True, force=True) @@ -263,19 +263,19 @@ def get_nginx_conf_from_container(container): def docker_compose_up(compose_file='docker-compose.yml'): - logging.info('docker-compose -f %s up -d' % compose_file) + logging.info(f'docker-compose -f {compose_file} up -d') try: - subprocess.check_output(shlex.split('docker-compose -f %s up -d' % compose_file), stderr=subprocess.STDOUT) + subprocess.check_output(shlex.split(f'docker-compose -f {compose_file} up -d'), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - pytest.fail("Error while runninng 'docker-compose -f %s up -d':\n%s" % (compose_file, e.output), pytrace=False) + pytest.fail(f"Error while runninng 'docker-compose -f {compose_file} up -d':\n{e.output}", pytrace=False) def docker_compose_down(compose_file='docker-compose.yml'): - logging.info('docker-compose -f %s down' % compose_file) + logging.info(f'docker-compose -f {compose_file} down') try: - subprocess.check_output(shlex.split('docker-compose -f %s down' % compose_file), stderr=subprocess.STDOUT) + subprocess.check_output(shlex.split(f'docker-compose -f {compose_file} down'), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - pytest.fail("Error while runninng 'docker-compose -f %s down':\n%s" % (compose_file, e.output), pytrace=False) + pytest.fail(f"Error while runninng 'docker-compose -f {compose_file} down':\n{e.output}", pytrace=False) def wait_for_nginxproxy_to_be_ready(): @@ -309,7 +309,7 @@ def find_docker_compose_file(request): if docker_compose_file_module_variable is not None: docker_compose_file = os.path.join( test_module_dir, docker_compose_file_module_variable) if not os.path.isfile(docker_compose_file): - raise ValueError("docker compose file %r could not be found. Check your test module `docker_compose_file` variable value." % docker_compose_file) + raise ValueError(f"docker compose file {docker_compose_file!r} could not be found. Check your test module `docker_compose_file` variable value.") else: if os.path.isfile(yml_file): docker_compose_file = yml_file @@ -321,7 +321,7 @@ def find_docker_compose_file(request): if not os.path.isfile(docker_compose_file): logging.error("Could not find any docker-compose file named either '{0}.yml', '{0}.yaml' or 'docker-compose.yml'".format(request.module.__name__)) - logging.debug("using docker compose file %s" % docker_compose_file) + logging.debug(f"using docker compose file {docker_compose_file}") return docker_compose_file @@ -335,7 +335,7 @@ def connect_to_network(network): try: my_container = docker_client.containers.get(socket.gethostname()) except docker.errors.NotFound: - logging.warn("container %r not found" % socket.gethostname()) + logging.warn(f"container {socket.gethostname()!r} not found") return # figure out our container networks @@ -343,7 +343,7 @@ def connect_to_network(network): # make sure our container is connected to the nginx-proxy's network if network not in my_networks: - logging.info("Connecting to docker network: %s" % network.name) + logging.info(f"Connecting to docker network: {network.name}") network.connect(my_container) return network @@ -358,7 +358,7 @@ def disconnect_from_network(network=None): try: my_container = docker_client.containers.get(socket.gethostname()) except docker.errors.NotFound: - logging.warn("container %r not found" % socket.gethostname()) + logging.warn(f"container {socket.gethostname()!r} not found") return # figure out our container networks @@ -366,7 +366,7 @@ def disconnect_from_network(network=None): # disconnect our container from the given network if network.name in my_networks_names: - logging.info("Disconnecting from network %s" % network.name) + logging.info(f"Disconnecting from network {network.name}") network.disconnect(my_container) @@ -458,7 +458,7 @@ def pytest_runtest_makereport(item, call): def pytest_runtest_setup(item): previousfailed = getattr(item.parent, "_previousfailed", None) if previousfailed is not None: - pytest.xfail("previous test failed (%s)" % previousfailed.name) + pytest.xfail(f"previous test failed ({previousfailed.name})") ############################################################################### # diff --git a/test/requirements/web/webserver.py b/test/requirements/web/webserver.py index 9334657..b8e81c0 100755 --- a/test/requirements/web/webserver.py +++ b/test/requirements/web/webserver.py @@ -13,13 +13,13 @@ class Handler(http.server.SimpleHTTPRequestHandler): if self.path == "/headers": response_body += self.headers.as_string() elif self.path == "/port": - response_body += "answer from port %s\n" % PORT + response_body += f"answer from port {PORT}\n" elif re.match("/status/(\d+)", self.path): result = re.match("/status/(\d+)", self.path) response_code = int(result.group(1)) - response_body += "answer with response code %s\n" % response_code + response_body += f"answer with response code {response_code}\n" elif self.path == "/": - response_body += "I'm %s\n" % os.environ['HOSTNAME'] + response_body += f"I'm {os.environ['HOSTNAME']}\n" else: response_body += "No route for this path!\n" response_code = 404 diff --git a/test/test_dockergen/test_dockergen_v2.py b/test/test_dockergen/test_dockergen_v2.py index a3f2484..43b1431 100644 --- a/test/test_dockergen/test_dockergen_v2.py +++ b/test/test_dockergen/test_dockergen_v2.py @@ -39,4 +39,4 @@ def test_forwards_to_whoami(nginx_tmpl, docker_compose, nginxproxy): r = nginxproxy.get("http://whoami.nginx.container.docker/") assert r.status_code == 200 whoami_container = docker_compose.containers.get("whoami") - assert r.text == "I'm %s\n" % whoami_container.id[:12] + assert r.text == f"I'm {whoami_container.id[:12]}\n" diff --git a/test/test_dockergen/test_dockergen_v3.py b/test/test_dockergen/test_dockergen_v3.py index 1beffeb..358f793 100644 --- a/test/test_dockergen/test_dockergen_v3.py +++ b/test/test_dockergen/test_dockergen_v3.py @@ -24,8 +24,7 @@ def versiontuple(v): raw_version = docker.from_env().version()["Version"] pytestmark = pytest.mark.skipif( versiontuple(raw_version) < (1, 13), - reason="Docker compose syntax v3 requires docker engine v1.13 or later (got %s)" - % raw_version, + reason="Docker compose syntax v3 requires docker engine v1.13 or later (got {raw_version})" ) @@ -64,7 +63,7 @@ def test_forwards_to_whoami(nginx_tmpl, docker_compose, nginxproxy): r = nginxproxy.get("http://whoami.nginx.container.docker/") assert r.status_code == 200 whoami_container = docker_compose.containers.get("whoami") - assert r.text == "I'm %s\n" % whoami_container.id[:12] + assert r.text == f"I'm {whoami_container.id[:12]}\n" if __name__ == "__main__": diff --git a/test/test_ssl/test_dhparam.py b/test/test_ssl/test_dhparam.py index 8899c6a..acb4269 100644 --- a/test/test_ssl/test_dhparam.py +++ b/test/test_ssl/test_dhparam.py @@ -55,7 +55,7 @@ def require_openssl(required_version): openssl_version = str(command_output.split()[1]) return pytest.mark.skipif( versiontuple(openssl_version) < versiontuple(required_version), - reason="openssl v%s is less than required version %s" % (openssl_version, required_version)) + reason=f"openssl v{openssl_version} is less than required version {required_version}") ############################################################################### @@ -87,7 +87,7 @@ def test_web5_dhparam_is_used(docker_compose): sut_container = docker_client.containers.get("nginxproxy") assert sut_container.status == "running" - host = "%s:443" % sut_container.attrs["NetworkSettings"]["IPAddress"] + host = f"{sut_container.attrs['NetworkSettings']['IPAddress']}:443" r = subprocess.check_output( f"echo '' | openssl s_client -connect {host} -cipher 'EDH' | grep 'Server Temp Key'", shell=True) assert b"Server Temp Key: X25519, 253 bits\n" == r diff --git a/test/test_ssl/test_wildcard.py b/test/test_ssl/test_wildcard.py index 9885d94..202ba24 100644 --- a/test/test_ssl/test_wildcard.py +++ b/test/test_ssl/test_wildcard.py @@ -3,21 +3,21 @@ import pytest @pytest.mark.parametrize("subdomain", ["foo", "bar"]) def test_web1_http_redirects_to_https(docker_compose, nginxproxy, subdomain): - r = nginxproxy.get("http://%s.nginx-proxy.tld/" % subdomain, allow_redirects=False) + r = nginxproxy.get(f"http://{subdomain}.nginx-proxy.tld/", allow_redirects=False) assert r.status_code == 301 assert "Location" in r.headers - assert "https://%s.nginx-proxy.tld/" % subdomain == r.headers['Location'] + assert f"https://{subdomain}.nginx-proxy.tld/" == r.headers['Location'] @pytest.mark.parametrize("subdomain", ["foo", "bar"]) def test_web1_https_is_forwarded(docker_compose, nginxproxy, subdomain): - r = nginxproxy.get("https://%s.nginx-proxy.tld/port" % subdomain, allow_redirects=False) + r = nginxproxy.get(f"https://{subdomain}.nginx-proxy.tld/port", allow_redirects=False) assert r.status_code == 200 assert "answer from port 81\n" in r.text @pytest.mark.parametrize("subdomain", ["foo", "bar"]) def test_web1_HSTS_policy_is_active(docker_compose, nginxproxy, subdomain): - r = nginxproxy.get("https://%s.nginx-proxy.tld/port" % subdomain, allow_redirects=False) + r = nginxproxy.get(f"https://{subdomain}.nginx-proxy.tld/port", allow_redirects=False) assert "answer from port 81\n" in r.text assert "Strict-Transport-Security" in r.headers diff --git a/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py b/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py index 1946cc0..03af625 100644 --- a/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py +++ b/test/test_ssl/wildcard_cert_and_nohttps/test_wildcard_cert_nohttps.py @@ -9,19 +9,19 @@ from requests.exceptions import SSLError (3, False), ]) def test_http_redirects_to_https(docker_compose, nginxproxy, subdomain, should_redirect_to_https): - r = nginxproxy.get("http://%s.web.nginx-proxy.tld/port" % subdomain) + r = nginxproxy.get(f"http://{subdomain}.web.nginx-proxy.tld/port") if should_redirect_to_https: assert len(r.history) > 0 assert r.history[0].is_redirect - assert r.history[0].headers.get("Location") == "https://%s.web.nginx-proxy.tld/port" % subdomain - assert "answer from port 8%s\n" % subdomain == r.text + assert r.history[0].headers.get("Location") == f"https://{subdomain}.web.nginx-proxy.tld/port" + assert f"answer from port 8{subdomain}\n" == r.text @pytest.mark.parametrize("subdomain", [1, 2]) def test_https_get_served(docker_compose, nginxproxy, subdomain): - r = nginxproxy.get("https://%s.web.nginx-proxy.tld/port" % subdomain, allow_redirects=False) + r = nginxproxy.get(f"https://{subdomain}.web.nginx-proxy.tld/port", allow_redirects=False) assert r.status_code == 200 - assert "answer from port 8%s\n" % subdomain == r.text + assert f"answer from port 8{subdomain}\n" == r.text def test_web3_https_is_500_and_SSL_validation_fails(docker_compose, nginxproxy): diff --git a/test/test_wildcard_host.py b/test/test_wildcard_host.py index eb8428e..a5b6633 100644 --- a/test/test_wildcard_host.py +++ b/test/test_wildcard_host.py @@ -18,9 +18,9 @@ import pytest ("web4.whatever.nginx-proxy.regexp", 84), ]) def test_wildcard_prefix(docker_compose, nginxproxy, host, expected_port): - r = nginxproxy.get("http://%s/port" % host) + r = nginxproxy.get(f"http://{host}/port") assert r.status_code == 200 - assert r.text == "answer from port %s\n" % expected_port + assert r.text == f"answer from port {expected_port}\n" @pytest.mark.parametrize("host", [ @@ -28,5 +28,5 @@ def test_wildcard_prefix(docker_compose, nginxproxy, host, expected_port): "web4.whatever.nginx-proxy.regexp-to-infinity-and-beyond" ]) def test_non_matching_host_is_503(docker_compose, nginxproxy, host): - r = nginxproxy.get("http://%s/port" % host) + r = nginxproxy.get(f"http://{host}/port") assert r.status_code == 503, r.text From 6fd3cfb38f134e4741136d5013c14f76c9e80783 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 12:32:35 +0100 Subject: [PATCH 04/11] fix(ci): :wrench: add markers on pytest.ini to fix warnings --- test/pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/pytest.ini b/test/pytest.ini index 30f3e19..9ca7667 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,5 @@ [pytest] # disable the creation of the `.cache` folders -addopts = -p no:cacheprovider --ignore=requirements --ignore=certs -r s -v \ No newline at end of file +addopts = -p no:cacheprovider --ignore=requirements --ignore=certs -r s -v +markers = + incremental: mark a test as incremental. \ No newline at end of file From 0c60d5703150be43abc1200da0644fbeeacd24b8 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 2 Apr 2021 01:03:19 +0200 Subject: [PATCH 05/11] fix(ci): fix test_dhparam_is_generated_if_missing --- test/test_ssl/test_dhparam_generation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_ssl/test_dhparam_generation.py b/test/test_ssl/test_dhparam_generation.py index 4ba1c53..ec1c90e 100644 --- a/test/test_ssl/test_dhparam_generation.py +++ b/test/test_ssl/test_dhparam_generation.py @@ -35,10 +35,10 @@ def test_dhparam_is_generated_if_missing(docker_compose): sut_container = docker_client.containers.get("nginxproxy") assert sut_container.status == "running" - assert_log_contains("Generating DH parameters") + assert_log_contains("Generating DSA parameters") assert_log_contains("dhparam generation complete, reloading nginx") # Make sure the dhparam in use is not the default, pre-generated one - default_checksum = sut_container.exec_run("md5sum /app/dhparam.pem.default").split() - generated_checksum = sut_container.exec_run("md5sum /etc/nginx/dhparam/dhparam.pem").split() + default_checksum = sut_container.exec_run("md5sum /app/dhparam.pem.default").output.split() + generated_checksum = sut_container.exec_run("md5sum /etc/nginx/dhparam/dhparam.pem").output.split() assert default_checksum[0] != generated_checksum[0] From dd7f7e842725b87f8ff2ba6f412ca3358a5d8f19 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Sat, 3 Apr 2021 21:38:49 +0200 Subject: [PATCH 06/11] fix(ci): wrong nginx-proxy image used on default_host test --- test/test_default-host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_default-host.yml b/test/test_default-host.yml index f195f58..47b8525 100644 --- a/test/test_default-host.yml +++ b/test/test_default-host.yml @@ -10,7 +10,7 @@ web1: # WHEN nginx-proxy runs with DEFAULT_HOST set to web1.tld sut: - image: jwilder/nginx-proxy:test + image: nginxproxy/nginx-proxy:test volumes: - /var/run/docker.sock:/tmp/docker.sock:ro - ./lib/ssl/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro From dd853b25726053cb92dc811900e0d4fe1254fc4f Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 13:46:38 +0100 Subject: [PATCH 07/11] chore(ci): :construction_worker: mv unit test from travis to ga --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b1930a5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: Test + +on: [push, pull_request] + +jobs: + unit: + name: Unit Test + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + docker_image: [alpine, debian] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r python-requirements.txt + working-directory: test/requirements + + - name: Build Docker web server image + run: make update-dependencies + + - name: Run tests + run: make test-${{ matrix.docker_image }} From 3b1163291b9bf859bd72f76bdb6488b48a1a9db3 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 14:08:54 +0100 Subject: [PATCH 08/11] fix(test): test_dockergen_v3 version comparison --- test/test_dockergen/test_dockergen_v3.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/test/test_dockergen/test_dockergen_v3.py b/test/test_dockergen/test_dockergen_v3.py index 358f793..67561bf 100644 --- a/test/test_dockergen/test_dockergen_v3.py +++ b/test/test_dockergen/test_dockergen_v3.py @@ -3,27 +3,12 @@ import docker import logging import pytest import re - -def versiontuple(v): - """ - >>> versiontuple("1.12.3") - (1, 12, 3) - - >>> versiontuple("1.13.0") - (1, 13, 0) - - >>> versiontuple("17.03.0-ce") - (17, 3, 0) - - >>> versiontuple("17.03.0-ce") < (1, 13) - False - """ - return tuple(map(int, (v.split("-")[0].split(".")))) +from distutils.version import LooseVersion raw_version = docker.from_env().version()["Version"] pytestmark = pytest.mark.skipif( - versiontuple(raw_version) < (1, 13), + LooseVersion(raw_version) < LooseVersion("1.13"), reason="Docker compose syntax v3 requires docker engine v1.13 or later (got {raw_version})" ) From 1591fd7968e9bc2904cc9c052d94addfd583f15f Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 15:12:16 +0100 Subject: [PATCH 09/11] chore(ci): :green_heart: use standard python for nginx-proxy-tester --- test/requirements/Dockerfile-nginx-proxy-tester | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/requirements/Dockerfile-nginx-proxy-tester b/test/requirements/Dockerfile-nginx-proxy-tester index 6c0f060..3c25c0c 100644 --- a/test/requirements/Dockerfile-nginx-proxy-tester +++ b/test/requirements/Dockerfile-nginx-proxy-tester @@ -1,7 +1,4 @@ -FROM python:3.9-alpine - -# Note: we're using alpine because it has openssl 1.0.2, which we need for testing -RUN apk add --update bash openssl curl && rm -rf /var/cache/apk/* +FROM python:3.9 COPY python-requirements.txt /requirements.txt RUN pip install -r /requirements.txt From 39f822dd8bfea95e6e91d2e9b4c46d33b639d13e Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Fri, 19 Mar 2021 16:44:50 +0100 Subject: [PATCH 10/11] refactor(ci): :recycle: refactor makefile and modify its usage on CI --- .github/workflows/test.yml | 10 +++++++--- Makefile | 14 +++++++++----- test/requirements/build.sh | 6 ------ 3 files changed, 16 insertions(+), 14 deletions(-) delete mode 100755 test/requirements/build.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1930a5..cde7ff3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: true matrix: - docker_image: [alpine, debian] + base_docker_image: [alpine, debian] steps: - uses: actions/checkout@v2 @@ -27,7 +27,11 @@ jobs: working-directory: test/requirements - name: Build Docker web server image - run: make update-dependencies + run: make build-webserver + + - name: Build Docker nginx proxy test image + run: make build-nginx-proxy-test-${{ matrix.base_docker_image }} - name: Run tests - run: make test-${{ matrix.docker_image }} + run: pytest + working-directory: test \ No newline at end of file diff --git a/Makefile b/Makefile index d7db2b8..18fcd33 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,19 @@ .PHONY : test-debian test-alpine test -update-dependencies: - test/requirements/build.sh +build-webserver: + docker build -t web test/requirements/web -test-debian: update-dependencies +build-nginx-proxy-test-debian: docker build -t nginxproxy/nginx-proxy:test . + +build-nginx-proxy-test-alpine: + docker build -f Dockerfile.alpine -t nginxproxy/nginx-proxy:test . + +test-debian: build-webserver build-nginx-proxy-test-debian test/pytest.sh -test-alpine: update-dependencies - docker build -f Dockerfile.alpine -t nginxproxy/nginx-proxy:test . +test-alpine: build-webserver build-nginx-proxy-test-alpine test/pytest.sh test: test-debian test-alpine diff --git a/test/requirements/build.sh b/test/requirements/build.sh deleted file mode 100755 index f29897a..0000000 --- a/test/requirements/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -docker build -t web $DIR/web \ No newline at end of file From 1518c39e1bdafb33b03e8e48781078a6bdf0c9c8 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Mon, 26 Apr 2021 23:15:08 +0200 Subject: [PATCH 11/11] docs: update "how to install/test" parts --- README.md | 21 +++++++-------------- test/README.md | 15 +++++---------- test/requirements/README.md | 2 +- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 813fdda..bc8c3db 100644 --- a/README.md +++ b/README.md @@ -415,22 +415,15 @@ Before submitting pull requests or issues, please check github to make sure an e #### Running Tests Locally -To run tests, you need to prepare the docker image to test which must be tagged `nginxproxy/nginx-proxy:test`: - - docker build -t nginxproxy/nginx-proxy:test . # build the Debian variant image - -and call the [test/pytest.sh](test/pytest.sh) script. - -Then build the Alpine variant of the image: - - docker build -f Dockerfile.alpine -t nginxproxy/nginx-proxy:test . # build the Alpline variant image - -and call the [test/pytest.sh](test/pytest.sh) script again. - - -If your system has the `make` command, you can automate those tasks by calling: +To run tests, you just need to run the command below: make test +This commands run tests on two variants of the nginx-proxy docker image: Debian and Alpine. + +You can run the tests for each of these images with their respective commands: + + make test-debian + make test-alpine You can learn more about how the test suite works and how to write new tests in the [test/README.md](test/README.md) file. diff --git a/test/README.md b/test/README.md index c62960a..dd9db44 100644 --- a/test/README.md +++ b/test/README.md @@ -4,9 +4,8 @@ Nginx proxy test suite Install requirements -------------------- -You need [python 2.7](https://www.python.org/) and [pip](https://pip.pypa.io/en/stable/installing/) installed. Then run the commands: +You need [python 3.9](https://www.python.org/) and [pip](https://pip.pypa.io/en/stable/installing/) installed. Then run the commands: - requirements/build.sh pip install -r requirements/python-requirements.txt If you can't install those requirements on your computer, you can alternatively use the _pytest.sh_ script which will run the tests from a Docker container which has those requirements. @@ -15,14 +14,11 @@ If you can't install those requirements on your computer, you can alternatively Prepare the nginx-proxy test image ---------------------------------- - docker build -t nginxproxy/nginx-proxy:test .. + make build-nginx-proxy-test-debian or if you want to test the alpine flavor: - docker build -t nginxproxy/nginx-proxy:test -f Dockerfile.alpine .. - -make sure to tag that test image exactly `nginxproxy/nginx-proxy:test` or the test suite won't work. - + make build-nginx-proxy-test-alpine Run the test suite ------------------ @@ -61,7 +57,7 @@ The fixture will run the _docker-compose_ command with the `-f` option to load t In the case you are running pytest from within a docker container, the `docker_compose` fixture will make sure the container running pytest is attached to all docker networks. That way, your test will be able to reach any of them. -In your tests, you can use the `docker_compose` variable to query and command the docker daemon as it provides you with a [client from the docker python module](https://docker-py.readthedocs.io/en/2.0.2/client.html#client-reference). +In your tests, you can use the `docker_compose` variable to query and command the docker daemon as it provides you with a [client from the docker python module](https://docker-py.readthedocs.io/en/4.4.4/client.html#client-reference). Also this fixture alters the way the python interpreter resolves domain names to IP addresses in the following ways: @@ -99,8 +95,7 @@ Furthermore, the nginxproxy methods accept an additional keyword parameter: `ipv ### The web docker image -When you ran the `requirements/build.sh` script earlier, you built a [`web`](requirements/README.md) docker image which is convenient for running a small web server in a container. This image can produce containers that listens on multiple ports at the same time. - +When you run the `make build-webserver` command, you built a [`web`](requirements/README.md) docker image which is convenient for running a small web server in a container. This image can produce containers that listens on multiple ports at the same time. ### Testing TLS diff --git a/test/requirements/README.md b/test/requirements/README.md index 3a0c389..394c9b1 100644 --- a/test/requirements/README.md +++ b/test/requirements/README.md @@ -2,7 +2,7 @@ This directory contains resources to build Docker images tests depend on # Build images - ./build.sh + make build-webserver # python-requirements.txt