From de9809a9e57bef9f2025a382ce7baa040b9cab30 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 14 Mar 2023 12:11:33 +0100 Subject: [PATCH 1/5] Add more control over log_format config directive Introduces 3 new environmental variables: - `LOG_FORMAT` - `LOG_FORMAT_ESCAPE` - `LOG_JSON` `LOG_FORMAT` and `LOG_FORMAT_ESCAPE` default to standard values. When `LOG_JSON` is set, defaults are changed to: `{"time_local":"$time_iso8601","client_ip":"$http_x_forwarded_for","remote_addr":"$remote_addr","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","request_time":"$request_time","upstream_response_time":"$upstream_response_time","upstream_addr":"$upstream_addr","http_referrer":"$http_referer","http_user_agent":"$http_user_agent","request_id":"$request_id"}`and `json`. See `nginx.tmpl` and https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format for details --- nginx.tmpl | 19 ++++++++++++++++++- test/test_log_json.py | 14 ++++++++++++++ test/test_log_json.yml | 15 +++++++++++++++ test/test_log_json_format.py | 16 ++++++++++++++++ test/test_log_json_format.yml | 15 +++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 test/test_log_json.py create mode 100644 test/test_log_json.yml create mode 100644 test/test_log_json_format.py create mode 100644 test/test_log_json_format.yml diff --git a/nginx.tmpl b/nginx.tmpl index 8b8b12e..5405e7e 100644 --- a/nginx.tmpl +++ b/nginx.tmpl @@ -402,7 +402,24 @@ map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl { gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; -log_format vhost '{{ or $globals.Env.LOG_FORMAT "$host $remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$upstream_addr\"" }}'; + +{{- /* See https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format for details and variables + * LOG_FORMAT_ESCAPE sets the escape part of the log format + * LOG_FORMAT sets the log format + */}} +{{- $logEscape := printf "escape=%s" ( or $globals.Env.LOG_FORMAT_ESCAPE `default`) }} +{{- $logFormat := (or $globals.Env.LOG_FORMAT `$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$upstream_addr"`) }} + +{{- if $globals.Env.LOG_JSON }} +{{- /* LOG_JSON is a shorthand + * that sets logging defaults to JSON format + */}} +# JSON Logging enabled (via LOG_JSON env variable) +{{- $logEscape = printf "escape=%s" ( or $globals.Env.LOG_FORMAT_ESCAPE `json`) }} +{{- $logFormat = (or $globals.Env.LOG_FORMAT `{"time_local":"$time_iso8601","client_ip":"$http_x_forwarded_for","remote_addr":"$remote_addr","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","request_time":"$request_time","upstream_response_time":"$upstream_response_time","upstream_addr":"$upstream_addr","http_referrer":"$http_referer","http_user_agent":"$http_user_agent","request_id":"$request_id"}`) }} +{{- end }} + +log_format vhost {{ $logEscape }} '{{ or $globals.Env.LOG_FORMAT $logFormat }}'; access_log off; diff --git a/test/test_log_json.py b/test/test_log_json.py new file mode 100644 index 0000000..1a04b22 --- /dev/null +++ b/test/test_log_json.py @@ -0,0 +1,14 @@ +import pytest + +def test_log_json(docker_compose, nginxproxy): + log_conf = [line for line in nginxproxy.get_conf().decode('ASCII').splitlines() if "log_format vhost escape=" in line] + assert "{\"time_local\":\"$time_iso8601\"," in log_conf[0] + + r = nginxproxy.get("http://nginx-proxy.test/port") + assert r.status_code == 200 + assert r.text == "answer from port 81\n" + sut_container = docker_compose.containers.get("sut") + docker_logs = sut_container.logs(stdout=True, stderr=True, stream=False, follow=False) + docker_logs = docker_logs.decode("utf-8").splitlines() + docker_logs = [line for line in docker_logs if "{\"time_local\":" in line] + assert "GET /port" in docker_logs[0] diff --git a/test/test_log_json.yml b/test/test_log_json.yml new file mode 100644 index 0000000..7740191 --- /dev/null +++ b/test/test_log_json.yml @@ -0,0 +1,15 @@ +web1: + image: web + expose: + - "81" + environment: + WEB_PORTS: 81 + VIRTUAL_HOST: nginx-proxy.test + +sut: + container_name: sut + image: nginxproxy/nginx-proxy:test + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + environment: + LOG_JSON: 1 diff --git a/test/test_log_json_format.py b/test/test_log_json_format.py new file mode 100644 index 0000000..56cfdea --- /dev/null +++ b/test/test_log_json_format.py @@ -0,0 +1,16 @@ +import pytest + +def test_log_json_format(docker_compose, nginxproxy): + log_conf = [line for line in nginxproxy.get_conf().decode('ASCII').splitlines() if "log_format vhost escape=" in line] + assert "{\"time_local\":\"$time_iso8601\"," in log_conf[0] + + r = nginxproxy.get("http://nginx-proxy.test/port") + assert r.status_code == 200 + assert r.text == "answer from port 81\n" + sut_container = docker_compose.containers.get("sut") + docker_logs = sut_container.logs(stdout=True, stderr=True, stream=False, follow=False) + docker_logs = docker_logs.decode("utf-8").splitlines() + docker_logs = [line for line in docker_logs if "{\"time_local\":" in line] + assert "GET /port" in docker_logs[0] + + diff --git a/test/test_log_json_format.yml b/test/test_log_json_format.yml new file mode 100644 index 0000000..caf20a7 --- /dev/null +++ b/test/test_log_json_format.yml @@ -0,0 +1,15 @@ +web1: + image: web + expose: + - "81" + environment: + WEB_PORTS: 81 + VIRTUAL_HOST: nginx-proxy.test + +sut: + container_name: sut + image: nginxproxy/nginx-proxy:test + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + environment: + LOG_FORMAT: '{"time_local":"$$time_iso8601","remote_addr":"$$remote_addr","request":"$$request","upstream_addr":"$$upstream_addr"}' From 8aefce916f2b5c6dcf8f6130fe73348ee49306aa Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Sat, 10 Feb 2024 18:02:42 +0100 Subject: [PATCH 2/5] tests: fix the json log test compose files --- test/test_log_json.yml | 31 +++++++++++++++++-------------- test/test_log_json_format.yml | 31 +++++++++++++++++-------------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/test/test_log_json.yml b/test/test_log_json.yml index 7740191..2e6fefd 100644 --- a/test/test_log_json.yml +++ b/test/test_log_json.yml @@ -1,15 +1,18 @@ -web1: - image: web - expose: - - "81" - environment: - WEB_PORTS: 81 - VIRTUAL_HOST: nginx-proxy.test +version: "2" -sut: - container_name: sut - image: nginxproxy/nginx-proxy:test - volumes: - - /var/run/docker.sock:/tmp/docker.sock:ro - environment: - LOG_JSON: 1 +services: + web1: + image: web + expose: + - "81" + environment: + WEB_PORTS: 81 + VIRTUAL_HOST: nginx-proxy.test + + sut: + container_name: sut + image: nginxproxy/nginx-proxy:test + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + environment: + LOG_JSON: 1 diff --git a/test/test_log_json_format.yml b/test/test_log_json_format.yml index caf20a7..5e01e92 100644 --- a/test/test_log_json_format.yml +++ b/test/test_log_json_format.yml @@ -1,15 +1,18 @@ -web1: - image: web - expose: - - "81" - environment: - WEB_PORTS: 81 - VIRTUAL_HOST: nginx-proxy.test +version: "2" -sut: - container_name: sut - image: nginxproxy/nginx-proxy:test - volumes: - - /var/run/docker.sock:/tmp/docker.sock:ro - environment: - LOG_FORMAT: '{"time_local":"$$time_iso8601","remote_addr":"$$remote_addr","request":"$$request","upstream_addr":"$$upstream_addr"}' +services: + web1: + image: web + expose: + - "81" + environment: + WEB_PORTS: 81 + VIRTUAL_HOST: nginx-proxy.test + + sut: + container_name: sut + image: nginxproxy/nginx-proxy:test + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + environment: + LOG_FORMAT: '{"time_local":"$$time_iso8601","remote_addr":"$$remote_addr","request":"$$request","upstream_addr":"$$upstream_addr"}' From 76778cebb17d24629a0aea3c13565d111f64d939 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Sat, 10 Feb 2024 17:54:14 +0100 Subject: [PATCH 3/5] refactor: parse LOG_JSON as boolean + avoid unnecessary backticks --- nginx.tmpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nginx.tmpl b/nginx.tmpl index 5405e7e..c72c1c7 100644 --- a/nginx.tmpl +++ b/nginx.tmpl @@ -407,16 +407,16 @@ gzip_types text/plain text/css application/javascript application/json applicati * LOG_FORMAT_ESCAPE sets the escape part of the log format * LOG_FORMAT sets the log format */}} -{{- $logEscape := printf "escape=%s" ( or $globals.Env.LOG_FORMAT_ESCAPE `default`) }} -{{- $logFormat := (or $globals.Env.LOG_FORMAT `$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$upstream_addr"`) }} +{{- $logEscape := printf "escape=%s" (or $globals.Env.LOG_FORMAT_ESCAPE "default") }} +{{- $logFormat := or $globals.Env.LOG_FORMAT `$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$upstream_addr"` }} -{{- if $globals.Env.LOG_JSON }} -{{- /* LOG_JSON is a shorthand - * that sets logging defaults to JSON format - */}} +{{- if parseBool (or $globals.Env.LOG_JSON "false") }} + {{- /* LOG_JSON is a shorthand + * that sets logging defaults to JSON format + */}} # JSON Logging enabled (via LOG_JSON env variable) -{{- $logEscape = printf "escape=%s" ( or $globals.Env.LOG_FORMAT_ESCAPE `json`) }} -{{- $logFormat = (or $globals.Env.LOG_FORMAT `{"time_local":"$time_iso8601","client_ip":"$http_x_forwarded_for","remote_addr":"$remote_addr","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","request_time":"$request_time","upstream_response_time":"$upstream_response_time","upstream_addr":"$upstream_addr","http_referrer":"$http_referer","http_user_agent":"$http_user_agent","request_id":"$request_id"}`) }} + {{- $logEscape = printf "escape=%s" (or $globals.Env.LOG_FORMAT_ESCAPE "json") }} + {{- $logFormat = or $globals.Env.LOG_FORMAT `{"time_local":"$time_iso8601","client_ip":"$http_x_forwarded_for","remote_addr":"$remote_addr","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","request_time":"$request_time","upstream_response_time":"$upstream_response_time","upstream_addr":"$upstream_addr","http_referrer":"$http_referer","http_user_agent":"$http_user_agent","request_id":"$request_id"}` }} {{- end }} log_format vhost {{ $logEscape }} '{{ or $globals.Env.LOG_FORMAT $logFormat }}'; From 62f55b4428a1133e5f44a75dde310271fef28c60 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Sat, 10 Feb 2024 19:03:13 +0100 Subject: [PATCH 4/5] docs: add logging documentation --- docs/README.md | 65 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/docs/README.md b/docs/README.md index 32d432c..e381b7f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -181,12 +181,61 @@ If you would like to connect to FastCGI backend, set `VIRTUAL_PROTO=fastcgi` on If you use fastcgi,you can set `VIRTUAL_ROOT=xxx` for your root directory -### Custom log format +### Logging -If you want to use a custom log format, you can set `LOG_FORMAT=xxx` on the proxy container. +The default nginx access log format is + +``` +$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$upstream_addr" +``` + +#### Custom log format + +If you want to use a custom access log format, you can set `LOG_FORMAT=xxx` on the proxy container. With docker compose take care to escape the `$` character with `$$` to avoid variable interpolation. Example: `$remote_addr` becomes `$$remote_addr`. +#### JSON log format + +If you want access logs in JSON format, you can set `LOG_JSON=true`. This will correctly set the escape character to `json` and the log format to : + +```json +{ + "time_local": "$time_iso8601", + "client_ip": "$http_x_forwarded_for", + "remote_addr": "$remote_addr", + "request": "$request", + "status": "$status", + "body_bytes_sent": "$body_bytes_sent", + "request_time": "$request_time", + "upstream_response_time": "$upstream_response_time", + "upstream_addr": "$upstream_addr", + "http_referrer": "$http_referer", + "http_user_agent": "$http_user_agent", + "request_id": "$request_id" +} +``` + +#### Log format escaping + +If you want to manually set nginx `log_format`'s `escape`, set the `LOG_FORMAT_ESCAPE` variable to [a value supported by nginx](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). + +#### Disable access logs + +To disable nginx access logs entirely, set the `DISABLE_ACCESS_LOGS` environment variable to any value. + +#### Disabling colors in the container log output + +To remove colors from the container log output, set the [`NO_COLOR` environment variable to any value other than an empty string](https://no-color.org/) on the nginx-proxy container. + +```console +docker run --detach \ + --publish 80:80 \ + --env NO_COLOR=1 \ + --volume /var/run/docker.sock:/tmp/docker.sock:ro \ + nginxproxy/nginx-proxy +``` + ### Default Host To set the default host for nginx use the env var `DEFAULT_HOST=foo.bar.com` for example @@ -679,18 +728,6 @@ By default the nginx configuration `upstream` blocks will use this block's corre Please note that using regular expressions in `VIRTUAL_HOST` will always result in a corresponding `upstream` block with an SHA1 name. -### Disabling colors in the log output - -To remove colors from the log output, set the [`NO_COLOR` environment variable to any value other than an empty string](https://no-color.org/) on the nginx-proxy container. - -```console -docker run --detach \ - --publish 80:80 \ - --env NO_COLOR=1 \ - --volume /var/run/docker.sock:/tmp/docker.sock:ro \ - nginxproxy/nginx-proxy -``` - ### Troubleshooting If you can't access your `VIRTUAL_HOST`, inspect the generated nginx configuration: From 25883ac05fd3ba2ca0efcad2d45200ac62fb5f6b Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Sat, 10 Feb 2024 19:04:56 +0100 Subject: [PATCH 5/5] style: remove extra blank lines --- test/test_log_json_format.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_log_json_format.py b/test/test_log_json_format.py index 56cfdea..2d158cb 100644 --- a/test/test_log_json_format.py +++ b/test/test_log_json_format.py @@ -12,5 +12,3 @@ def test_log_json_format(docker_compose, nginxproxy): docker_logs = docker_logs.decode("utf-8").splitlines() docker_logs = [line for line in docker_logs if "{\"time_local\":" in line] assert "GET /port" in docker_logs[0] - -