From da0cb6fd0eecbe24d8711ad56cda479af97da174 Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Mon, 14 Oct 2024 17:44:24 +0300 Subject: [PATCH] Sync v2.9.0 --- .gitignore | 5 +- .markdownlint.json | 25 + CHANGELOG.md | 96 +- HACKING.md | 2 + Makefile | 75 +- README.md | 79 +- config.dist.yaml | 10 +- doc/configuration.md | 873 +-- doc/debugdns.md | 63 +- doc/debughttp.md | 87 +- doc/development.md | 258 +- doc/environment.md | 167 +- doc/externalhttp.md | 121 +- doc/http.md | 114 +- doc/metrics.md | 2 +- doc/querylog.md | 226 +- go.mod | 50 +- go.sum | 101 +- go.work | 2 +- go.work.sum | 46 +- internal/agd/context.go | 38 +- internal/agd/devicefinder.go | 106 + internal/agd/devicetype.go | 1 + internal/agd/profile.go | 14 +- internal/agd/ratelimit.go | 147 + internal/agd/requestid.go | 1 + internal/agd/servergroup.go | 12 +- internal/agd/version.go | 33 - internal/agdcache/manager.go | 8 +- internal/agdhttp/agdhttp.go | 24 +- internal/agdhttp/url.go | 41 +- internal/agdhttp/url_test.go | 2 +- internal/agdservice/agdservice_test.go | 11 +- internal/agdservice/refresh.go | 59 +- internal/agdservice/refresh_test.go | 3 +- internal/agdtest/agdtest.go | 33 +- internal/agdtest/interface.go | 101 +- internal/backendpb/backendpb_internal_test.go | 4 + internal/backendpb/backendpb_test.go | 10 +- internal/backendpb/billstat.go | 16 +- internal/backendpb/billstat_test.go | 5 +- internal/backendpb/device.go | 7 +- internal/backendpb/dns.pb.go | 526 +- internal/backendpb/dns.proto | 7 + internal/backendpb/dns_grpc.pb.go | 130 +- internal/backendpb/error.go | 34 +- internal/backendpb/metrics.go | 36 + internal/backendpb/profile.go | 26 +- internal/backendpb/profile_internal_test.go | 143 +- internal/backendpb/profiledb.go | 78 +- internal/backendpb/profiledb_test.go | 43 +- internal/backendpb/stats.go | 27 +- internal/billstat/billstat.go | 3 +- internal/billstat/billstat_test.go | 13 +- internal/billstat/metrics.go | 26 + internal/billstat/runtime.go | 51 +- internal/billstat/runtime_test.go | 11 +- internal/bindtodevice/bindtodevice_test.go | 6 - .../bindtodevice/interfacelistener_linux.go | 73 +- internal/bindtodevice/manager.go | 9 +- internal/bindtodevice/manager_linux.go | 29 +- internal/bindtodevice/manager_linux_test.go | 29 +- internal/cmd/access.go | 12 +- internal/cmd/additional.go | 24 +- internal/cmd/backend.go | 245 +- internal/cmd/builder.go | 1212 +++++ internal/cmd/cache.go | 26 +- internal/cmd/check.go | 216 +- internal/cmd/cmd.go | 379 +- internal/cmd/config.go | 152 +- internal/cmd/conncheck.go | 35 +- internal/cmd/context.go | 29 + internal/cmd/ddr.go | 63 +- internal/cmd/dns.go | 26 +- internal/cmd/dnscrypt.go | 6 +- internal/cmd/dnsdb.go | 15 +- internal/cmd/env.go | 370 +- internal/cmd/error.go | 57 +- internal/cmd/filter.go | 100 +- internal/cmd/filteringgroup.go | 26 +- internal/cmd/geoip.go | 74 +- internal/cmd/ifacelistener.go | 61 +- internal/cmd/network.go | 39 +- internal/cmd/plugin/plugin.go | 46 + internal/cmd/querylog.go | 15 +- internal/cmd/ratelimit.go | 106 +- internal/cmd/safebrowsing.go | 102 +- internal/cmd/server.go | 34 +- internal/cmd/servergroup.go | 34 +- internal/cmd/tickrot.go | 119 + internal/cmd/tls.go | 184 +- internal/cmd/upstream.go | 77 +- internal/cmd/validation.go | 4 +- internal/cmd/websvc.go | 173 +- internal/connlimiter/conn.go | 24 +- internal/connlimiter/limiter.go | 9 + internal/connlimiter/limiter_test.go | 7 +- internal/connlimiter/listenconfig_test.go | 2 + internal/connlimiter/listener.go | 16 +- internal/consul/allowlist.go | 80 +- internal/consul/allowlist_test.go | 45 +- internal/debugsvc/cache.go | 86 + internal/debugsvc/debugsvc.go | 261 +- internal/debugsvc/debugsvc_test.go | 143 +- internal/debugsvc/handler.go | 24 - internal/debugsvc/middleware.go | 62 - internal/debugsvc/refresh.go | 93 +- internal/debugsvc/route.go | 90 + internal/dnscheck/error.go | 39 +- internal/dnscheck/httpkv_internal_test.go | 17 - internal/dnscheck/httpkv_test.go | 283 - internal/dnscheck/{consul.go => remotekv.go} | 164 +- .../{consul_test.go => remotekv_test.go} | 112 +- internal/dnsdb/dnsdb.go | 3 +- internal/dnsdb/http.go | 2 - internal/dnsdb/http_test.go | 4 +- internal/dnsmsg/cloner_test.go | 120 +- internal/dnsmsg/constructor.go | 62 +- internal/dnsmsg/constructor_test.go | 170 +- internal/dnsmsg/dnsmsg.go | 26 +- internal/dnsmsg/dnsmsg_test.go | 7 - internal/dnsmsg/error.go | 2 - internal/dnsmsg/optcloner.go | 14 +- internal/dnsmsg/svcbmsg_test.go | 19 +- internal/dnsserver/dnsserver.go | 2 +- internal/dnsserver/doc.go | 2 +- internal/dnsserver/forward/forward.go | 17 +- internal/dnsserver/forward/forward_test.go | 1 + internal/dnsserver/forward/healthcheck.go | 30 +- .../dnsserver/forward/healthcheck_test.go | 2 + internal/dnsserver/forward/metrics.go | 2 +- internal/dnsserver/forward/upstreamplain.go | 2 + .../dnsserver/forward/upstreamplain_test.go | 6 +- internal/dnsserver/go.mod | 39 +- internal/dnsserver/go.sum | 85 +- internal/dnsserver/msg.go | 12 +- internal/dnsserver/prometheus/cache.go | 82 +- internal/dnsserver/prometheus/cache_test.go | 2 +- internal/dnsserver/prometheus/dns.go | 23 +- internal/dnsserver/prometheus/forward.go | 120 +- internal/dnsserver/prometheus/forward_test.go | 4 +- internal/dnsserver/prometheus/initsyncmap.go | 51 - .../prometheus/initsyncmap_internal_test.go | 41 - internal/dnsserver/prometheus/prometheus.go | 5 +- .../dnsserver/prometheus/prometheus_test.go | 3 + internal/dnsserver/prometheus/ratelimit.go | 73 +- .../dnsserver/prometheus/ratelimit_test.go | 17 +- internal/dnsserver/prometheus/server.go | 270 +- internal/dnsserver/prometheus/server_test.go | 4 +- internal/dnsserver/ratelimit/backoff.go | 46 +- internal/dnsserver/ratelimit/counter.go | 47 + internal/dnsserver/ratelimit/metrics.go | 33 +- internal/dnsserver/ratelimit/ratelimit.go | 58 +- .../dnsserver/ratelimit/ratelimit_test.go | 8 +- internal/dnsserver/ratelimit/rps.go | 43 - internal/dnsserver/serverdns.go | 18 +- internal/dnsserver/serverdnstcp.go | 10 +- internal/dnsserver/serverquic.go | 5 +- internal/dnssvc/dnssvc.go | 157 +- internal/dnssvc/dnssvc_test.go | 21 +- internal/dnssvc/integration_test.go | 44 +- internal/dnssvc/internal/accessmw/access.go | 68 - .../dnssvc/internal/accessmw/access_test.go | 140 - .../dnssvc/internal/devicefinder/device.go | 302 ++ .../internal/devicefinder/device_test.go | 353 ++ .../devicedata.go | 102 +- .../devicedata_test.go | 333 +- .../internal/devicefinder/devicefinder.go | 104 + .../devicefinder_test.go} | 211 +- .../{devicesetter => devicefinder}/error.go | 11 +- .../{devicesetter => devicefinder}/humanid.go | 10 +- .../humanid_test.go | 59 +- .../dnssvc/internal/devicesetter/device.go | 288 - .../internal/devicesetter/device_test.go | 381 -- .../internal/devicesetter/devicesetter.go | 130 - .../dnssvc/internal/dnssvctest/dnssvctest.go | 5 - .../dnssvc/internal/dnssvctest/interface.go | 37 - internal/dnssvc/internal/initial/initial.go | 214 +- .../dnssvc/internal/initial/initial_test.go | 839 +-- internal/dnssvc/internal/initial/location.go | 60 - .../dnssvc/internal/initial/specialdomain.go | 67 +- .../internal/initial/specialdomain_test.go | 267 +- internal/dnssvc/internal/mainmw/debug.go | 7 +- .../internal/mainmw/debug_internal_test.go | 51 +- .../internal/mainmw/filter_internal_test.go | 8 +- internal/dnssvc/internal/mainmw/mainmw.go | 44 +- .../dnssvc/internal/mainmw/mainmw_test.go | 66 +- internal/dnssvc/internal/mainmw/metrics.go | 58 + internal/dnssvc/internal/mainmw/record.go | 13 +- .../internal/mainmw/record_internal_test.go | 8 +- .../internal/preservice/preservice_test.go | 16 +- .../internal/preupstream/preupstream.go | 3 +- .../internal/preupstream/preupstream_test.go | 3 +- .../dnssvc/internal/ratelimitmw/access.go | 58 + .../internal/ratelimitmw/access_test.go | 201 + internal/dnssvc/internal/ratelimitmw/limit.go | 140 + .../dnssvc/internal/ratelimitmw/metrics.go | 66 + .../internal/ratelimitmw/ratelimitmw.go | 199 + .../internal/ratelimitmw/requestinfo.go | 117 + internal/ecscache/cache.go | 4 +- internal/ecscache/ecsblocklist.go | 4755 ++++++++--------- internal/ecscache/ecscache.go | 30 +- internal/ecscache/ecscache_test.go | 14 +- internal/ecscache/msg.go | 49 +- internal/errcoll/errcoll.go | 11 + internal/errcoll/sentry.go | 11 +- internal/errcoll/sentry_test.go | 11 +- internal/filter/filter_test.go | 17 +- internal/filter/hashprefix/filter.go | 51 +- internal/filter/hashprefix/filter_test.go | 72 +- internal/filter/hashprefix/hashprefix_test.go | 7 - internal/filter/hashprefix/matcher.go | 6 +- internal/filter/hashprefix/storage.go | 3 - internal/filter/index.go | 16 +- .../filter/internal/composite/composite.go | 17 +- .../composite/composite_internal_test.go | 17 +- .../internal/composite/composite_test.go | 24 +- internal/filter/internal/custom/custom.go | 40 +- .../filter/internal/custom/custom_test.go | 32 +- .../filter/internal/filtertest/filtertest.go | 37 +- internal/filter/internal/internal_test.go | 11 - internal/filter/internal/refreshable.go | 125 +- internal/filter/internal/refreshable_test.go | 49 +- .../filter/internal/rulelist/dnsrewrite.go | 3 - .../filter/internal/rulelist/refreshable.go | 44 +- .../internal/rulelist/refreshable_test.go | 11 +- internal/filter/internal/rulelist/rulelist.go | 4 +- .../filter/internal/safesearch/safesearch.go | 11 +- .../internal/safesearch/safesearch_test.go | 11 +- .../filter/internal/serviceblock/index.go | 19 +- .../internal/serviceblock/serviceblock.go | 21 +- .../serviceblock/serviceblock_test.go | 20 +- internal/filter/storage.go | 338 +- internal/filter/storage_test.go | 111 +- internal/geoip/asntops.go | 1032 ++-- internal/geoip/asntops_generate.go | 15 +- internal/geoip/country.go | 14 +- internal/geoip/country_generate.go | 13 +- internal/geoip/file.go | 44 +- internal/geoip/file_test.go | 7 + internal/geoip/filescanner.go | 27 +- internal/geoip/geoip_test.go | 6 - internal/metrics/access.go | 27 - internal/metrics/backend.go | 310 +- internal/metrics/billstat.go | 140 +- internal/metrics/dnsmsg_test.go | 9 - internal/metrics/dnssvc.go | 71 - internal/metrics/mainmw.go | 196 + internal/metrics/metrics.go | 41 +- internal/metrics/metrics_test.go | 27 + internal/metrics/profiledb.go | 253 + internal/metrics/ratelimitmw.go | 324 ++ internal/metrics/rediskv.go | 77 + internal/metrics/tls.go | 26 +- internal/metrics/tls_test.go | 22 +- internal/metrics/usercount.go | 112 +- ...unt_internal_test.go => usercount_test.go} | 69 +- internal/optslog/optslog.go | 103 + .../internal/filecachepb/filecache.pb.go | 383 +- .../internal/filecachepb/filecache.proto | 7 + .../internal/filecachepb/filecachepb.go | 75 +- .../filecachepb/filecachepb_internal_test.go | 16 +- .../internal/filecachepb/filecachepb_test.go | 6 + .../profiledb/internal/filecachepb/storage.go | 32 +- .../internal/filecachepb/storage_test.go | 14 +- internal/profiledb/internal/internal.go | 11 +- .../internal/profiledbtest/profiledbtest.go | 9 + internal/profiledb/metrics.go | 66 + internal/profiledb/profiledb.go | 223 +- internal/profiledb/profiledb_test.go | 71 +- internal/querylog/entry.go | 25 +- internal/querylog/fs.go | 60 +- internal/querylog/fs_test.go | 15 +- internal/querylog/querylog.go | 2 - internal/querylog/querylog_test.go | 4 +- internal/remotekv/cachekv.go | 43 + internal/remotekv/cachekv_test.go | 37 + .../consulkv}/consulkv.go | 230 +- internal/remotekv/consulkv/consulkv_test.go | 225 + internal/remotekv/consulkv/error.go | 44 + internal/remotekv/keynamespace.go | 52 + internal/remotekv/keynamespace_test.go | 46 + internal/remotekv/rediskv/metrics.go | 21 + internal/remotekv/rediskv/rediskv.go | 188 + internal/remotekv/remotekv.go | 33 + internal/rulestat/http.go | 18 +- internal/rulestat/http_test.go | 11 +- internal/rulestat/rulestat_test.go | 11 +- internal/tools/doc.go | 5 - internal/tools/go.mod | 69 +- internal/tools/go.sum | 263 +- internal/tools/tools.go | 7 +- internal/version/norace.go | 6 + internal/version/race.go | 6 + internal/version/version.go | 40 + internal/websvc/blockpage.go | 107 +- internal/websvc/blockpage_test.go | 32 +- internal/websvc/handler.go | 58 +- internal/websvc/handler_test.go | 1 + internal/websvc/linkip.go | 3 +- internal/websvc/linkip_internal_test.go | 26 +- internal/websvc/static.go | 19 +- internal/websvc/static_test.go | 2 +- internal/websvc/testdata/block_page.html | 1 + internal/websvc/websvc.go | 102 +- internal/websvc/websvc_test.go | 14 +- main.go | 4 +- scripts/backend/main.go | 13 +- scripts/hooks/pre-commit | 84 +- scripts/make/go-bench.sh | 31 +- scripts/make/go-build.sh | 47 +- scripts/make/go-deps.sh | 8 +- scripts/make/go-fuzz.sh | 49 +- scripts/make/go-gen.sh | 24 +- scripts/make/go-lint.sh | 180 +- scripts/make/go-test.sh | 63 +- scripts/make/go-tools.sh | 94 +- scripts/make/go-upd-tools.sh | 8 +- scripts/make/helper.sh | 23 +- scripts/make/md-lint.sh | 21 + scripts/make/sh-lint.sh | 29 + scripts/make/txt-lint.sh | 73 +- staticcheck.conf | 1 + 323 files changed, 16294 insertions(+), 13013 deletions(-) create mode 100644 .markdownlint.json create mode 100644 internal/agd/devicefinder.go create mode 100644 internal/agd/ratelimit.go delete mode 100644 internal/agd/version.go create mode 100644 internal/backendpb/metrics.go create mode 100644 internal/billstat/metrics.go create mode 100644 internal/cmd/builder.go create mode 100644 internal/cmd/context.go create mode 100644 internal/cmd/plugin/plugin.go create mode 100644 internal/cmd/tickrot.go create mode 100644 internal/debugsvc/cache.go delete mode 100644 internal/debugsvc/handler.go delete mode 100644 internal/debugsvc/middleware.go create mode 100644 internal/debugsvc/route.go delete mode 100644 internal/dnscheck/httpkv_internal_test.go delete mode 100644 internal/dnscheck/httpkv_test.go rename internal/dnscheck/{consul.go => remotekv.go} (67%) rename internal/dnscheck/{consul_test.go => remotekv_test.go} (60%) delete mode 100644 internal/dnsserver/prometheus/initsyncmap.go delete mode 100644 internal/dnsserver/prometheus/initsyncmap_internal_test.go create mode 100644 internal/dnsserver/ratelimit/counter.go delete mode 100644 internal/dnsserver/ratelimit/rps.go delete mode 100644 internal/dnssvc/internal/accessmw/access.go delete mode 100644 internal/dnssvc/internal/accessmw/access_test.go create mode 100644 internal/dnssvc/internal/devicefinder/device.go create mode 100644 internal/dnssvc/internal/devicefinder/device_test.go rename internal/dnssvc/internal/{devicesetter => devicefinder}/devicedata.go (73%) rename internal/dnssvc/internal/{devicesetter => devicefinder}/devicedata_test.go (57%) create mode 100644 internal/dnssvc/internal/devicefinder/devicefinder.go rename internal/dnssvc/internal/{devicesetter/devicesetter_test.go => devicefinder/devicefinder_test.go} (50%) rename internal/dnssvc/internal/{devicesetter => devicefinder}/error.go (81%) rename internal/dnssvc/internal/{devicesetter => devicefinder}/humanid.go (88%) rename internal/dnssvc/internal/{devicesetter => devicefinder}/humanid_test.go (55%) delete mode 100644 internal/dnssvc/internal/devicesetter/device.go delete mode 100644 internal/dnssvc/internal/devicesetter/device_test.go delete mode 100644 internal/dnssvc/internal/devicesetter/devicesetter.go delete mode 100644 internal/dnssvc/internal/dnssvctest/interface.go delete mode 100644 internal/dnssvc/internal/initial/location.go create mode 100644 internal/dnssvc/internal/mainmw/metrics.go create mode 100644 internal/dnssvc/internal/ratelimitmw/access.go create mode 100644 internal/dnssvc/internal/ratelimitmw/access_test.go create mode 100644 internal/dnssvc/internal/ratelimitmw/limit.go create mode 100644 internal/dnssvc/internal/ratelimitmw/metrics.go create mode 100644 internal/dnssvc/internal/ratelimitmw/ratelimitmw.go create mode 100644 internal/dnssvc/internal/ratelimitmw/requestinfo.go delete mode 100644 internal/filter/internal/internal_test.go delete mode 100644 internal/metrics/dnsmsg_test.go create mode 100644 internal/metrics/mainmw.go create mode 100644 internal/metrics/metrics_test.go create mode 100644 internal/metrics/profiledb.go create mode 100644 internal/metrics/ratelimitmw.go create mode 100644 internal/metrics/rediskv.go rename internal/metrics/{usercount_internal_test.go => usercount_test.go} (74%) create mode 100644 internal/optslog/optslog.go create mode 100644 internal/profiledb/internal/filecachepb/filecachepb_test.go create mode 100644 internal/profiledb/metrics.go create mode 100644 internal/remotekv/cachekv.go create mode 100644 internal/remotekv/cachekv_test.go rename internal/{dnscheck => remotekv/consulkv}/consulkv.go (50%) create mode 100644 internal/remotekv/consulkv/consulkv_test.go create mode 100644 internal/remotekv/consulkv/error.go create mode 100644 internal/remotekv/keynamespace.go create mode 100644 internal/remotekv/keynamespace_test.go create mode 100644 internal/remotekv/rediskv/metrics.go create mode 100644 internal/remotekv/rediskv/rediskv.go create mode 100644 internal/remotekv/remotekv.go delete mode 100644 internal/tools/doc.go create mode 100644 internal/version/norace.go create mode 100644 internal/version/race.go create mode 100644 internal/version/version.go create mode 100644 internal/websvc/testdata/block_page.html create mode 100644 scripts/make/md-lint.sh create mode 100644 scripts/make/sh-lint.sh diff --git a/.gitignore b/.gitignore index 01ef28c..41cb9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,11 +10,12 @@ *.test /bin/ /filters/ -/test/ /github-mirror/ +/test/ AdGuardDNS +agdns asn.mmdb config.yaml country.mmdb -querylog.jsonl profilecache.json +querylog.jsonl diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..445ee06 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,25 @@ +{ + "ul-indent": { + "indent": 4 + }, + "ul-style": { + "style": "dash" + }, + "emphasis-style": { + "style": "asterisk" + }, + "no-duplicate-heading": { + "siblings_only": true + }, + "no-inline-html": { + "allowed_elements": [ + "a" + ] + }, + "no-trailing-spaces": { + "br_spaces": 0 + }, + "line-length": false, + "no-bare-urls": false, + "link-fragments": false +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c5805..1525252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,88 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does [kec]: https://keepachangelog.com/en/1.0.0/ [sem]: https://semver.org/spec/v2.0.0.html +## AGDNS-2468 / Build 869 + +- The environment variable `PROFILES_MAX_RESP_SIZE` has been added. It sets the maximum size of the response from the profiles endpoint of the backend API. The default value is `8MB`. + +## AGDNS-2427 / Build 854 + +- The environment variables `REDIS_ADDR`, `REDIS_KEY_PREFIX`, `REDIS_MAX_ACTIVE`, `REDIS_MAX_IDLE`, `REDIS_IDLE_TIMEOUT`, and `REDIS_PORT` have been added. + +- The property `ttl` within the `check` is replaced by the object `kv` containing the previous `ttl` and the new property `type`. So replace this: + + ```yaml + check: + # … + ttl: 30s + ``` + + with this: + + ```yaml + check: + kv: + type: 'consul' + ttl: 30s + # … + ``` + +## AGDNS-2331 / Build 818 + +- Profile's file cache version was incremented. The new field `RateLimit` has been added to profile's object. + +## AGDNS-2008 / Build 809 + +- The environment variables `WEB_STATIC_DIR` and `WEB_STATIC_DIR_ENABLED` have been added. If `WEB_STATIC_DIR_ENABLED` is set to `1`, `WEB_STATIC_DIR` must point to a directory, from which static files are served. The `web.static_content` property in the configuration file is also ignored when `WEB_STATIC_DIR_ENABLED` is set to `1`. + +## AGDNS-2316 / Build 808 + +- The environment variables `BLOCKED_SERVICE_ENABLED`, `GENERAL_SAFE_SEARCH_ENABLED`, and `YOUTUBE_SAFE_SEARCH_ENABLED` have been added. If they are set to `0`, their corresponding `*_URL` environment variables can be empty. + +## AGDNS-2312 / Build 807 + +- The environment variables `BILLSTAT_URL` and `PROFILES_URL` no longer required if there are no server groups with profiles enabled. + +## AGDNS-2312 / Build 802 + +- The environment variables `ADULT_BLOCKING_ENABLED`, `NEW_REG_DOMAINS_ENABLED`, and `SAFE_BROWSING_ENABLED` have been added. If they are set to `0`, their corresponding `*_URL` environment variables can be empty. + +## AGDNS-2302 / Build 801 + +- The environment variable `METRICS_NAMESPACE` has been added. + +## AGDNS-2292 / Build 794 + +- The environment variable `PROFILES_ENABLED` has been removed. + +- The objects within the `server_groups` array have a new property `profiles_enabled`. So replace this: + + ```yaml + server_groups: + - name: 'default' + # … + - name: 'client' + # … + ``` + + with this: + + ```yaml + server_groups: + - name: 'default' + # … + profiles_enabled: false + - name: 'client' + # … + profiles_enabled: true + ``` + +## AGDNS-2289 / Build 793 + +- The environment variable `FILTER_INDEX_URL` now accepts `file://` URIs to use local files as filtering-rule list indexes. + +- All other `*_URL` environment variables are now validated to be HTTP(s) or gRPC(S) more strictly. + ## AGDNS-2254 / Build 779 - The environment variables `BILLSTAT_API_KEY` and `PROFILES_API_KEY` have been added. @@ -21,11 +103,11 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does ## AGDNS-2022 / Build 746 -- The property `block_page_redirect` of objects within `server_groups` array has been removed. +- The property `block_page_redirect` of objects within the `server_groups` array has been removed. ## AGDNS-1981 / Build 744 -- The objects within `server_groups` array had a change in their `block_page_redirect` configuration, it now supports arrays of IP addresses in `ipv4` and `ipv6` fields. +- The objects within the `server_groups` array had a change in their `block_page_redirect` configuration, it now supports arrays of IP addresses in `ipv4` and `ipv6` fields. - Profile's file cache version was incremented. In case of `BlockingModeCustomIP` the `profile.blocking_mode` IPv4/IPv6 fields are now arrays of IP addresses. @@ -83,7 +165,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does ## AGDNS-1954 / Build 719 -- The objects within `server_groups` array have a new property `block_page_redirect`: +- The objects within the `server_groups` array have a new property `block_page_redirect`: ```yaml block_page_redirect: @@ -106,8 +188,8 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does probability: 0.01 ``` - > [!NOTE] - > For `ipv4` and `ipv6` only one address is currently supported. + > [!NOTE] + > For `ipv4` and `ipv6` only one address is currently supported. For server groups that do not require a block-page redirect, set: @@ -557,7 +639,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does This means that DNSDB is disabled by default. -- The default configuration file path has been changed from `config.yml` to ./config.yaml for consistency with other services. +- The default configuration file path has been changed from `./config.yml` to `./config.yaml` for consistency with other services. ## AGDNS-916 / Build 456 @@ -1483,7 +1565,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project **does ## AGDNS-246 / Build 83 -- The new environment variable `RULESTAT_URL` has been added. Its default value is , which means that no statistics are gathered. Adjust the value, if necessary. +- The new environment variable `RULESTAT_URL` has been added. Its default value is an empty string, which means that no statistics are gathered. Adjust the value, if necessary. ## AGDNS-245 / Build 74 diff --git a/HACKING.md b/HACKING.md index 4e1b89b..edb2427 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1 +1,3 @@ +# Code guidelines + See the [Adguard Code Guidelines](https://github.com/AdguardTeam/CodeGuidelines/). diff --git a/Makefile b/Makefile index d9e9238..d75f046 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # Makefile. Bump this number every time a significant change is made to # this Makefile. # -# AdGuard-Project-Version: 5 +# AdGuard-Project-Version: 9 # Don't name these macros "GO" etc., because GNU Make apparently makes # them exported environment variables with the literal value of @@ -20,25 +20,33 @@ GO.MACRO = $${GO:-go} VERBOSE.MACRO = $${VERBOSE:-0} -BRANCH = $$( git rev-parse --abbrev-ref HEAD ) +BRANCH = $${BRANCH:-$$(git rev-parse --abbrev-ref HEAD)} GOAMD64 = v1 GOPROXY = https://proxy.golang.org|direct -GOTOOLCHAIN = go1.22.5 +GOTELEMETRY = off +GOTOOLCHAIN = go1.23.1 RACE = 0 -REVISION = $$( git rev-parse --short HEAD ) +REVISION = $${REVISION:-$$(git rev-parse --short HEAD)} VERSION = 0 -ENV = env\ - BRANCH="$(BRANCH)"\ - GO="$(GO.MACRO)"\ - GOAMD64='$(GOAMD64)'\ - GOPROXY='$(GOPROXY)'\ - GOTOOLCHAIN='$(GOTOOLCHAIN)'\ - PATH="$${PWD}/bin:$$( "$(GO.MACRO)" env GOPATH )/bin:$${PATH}"\ - RACE='$(RACE)'\ - REVISION="$(REVISION)"\ - VERBOSE="$(VERBOSE.MACRO)"\ - VERSION="$(VERSION)"\ +ENV = env \ + BRANCH="$(BRANCH)" \ + GO="$(GO.MACRO)" \ + GOAMD64='$(GOAMD64)' \ + GOPROXY='$(GOPROXY)' \ + GOTELEMETRY='$(GOTELEMETRY)' \ + GOTOOLCHAIN='$(GOTOOLCHAIN)' \ + PATH="$${PWD}/bin:$$("$(GO.MACRO)" env GOPATH)/bin:$${PATH}" \ + RACE='$(RACE)' \ + REVISION="$(REVISION)" \ + VERBOSE="$(VERBOSE.MACRO)" \ + VERSION="$(VERSION)" \ + +# Keep the line above blank. + +ENV_MISC = env \ + PATH="$${PWD}/bin:$$("$(GO.MACRO)" env GOPATH)/bin:$${PATH}" \ + VERBOSE="$(VERBOSE.MACRO)" \ # Keep the line above blank. @@ -50,29 +58,34 @@ init: ; git config core.hooksPath ./scripts/hooks test: go-test -go-bench: ; $(ENV) "$(SHELL)" ./scripts/make/go-bench.sh -go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh -go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh -go-fuzz: ; $(ENV) "$(SHELL)" ./scripts/make/go-fuzz.sh -go-gen: ; $(ENV) "$(SHELL)" ./scripts/make/go-gen.sh -go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh -go-test: ; $(ENV) RACE='1' "$(SHELL)" ./scripts/make/go-test.sh -go-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-tools.sh - -go-upd-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-upd-tools.sh +go-bench: ; $(ENV) "$(SHELL)" ./scripts/make/go-bench.sh +go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh +go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh +go-env: ; $(ENV) "$(GO.MACRO)" env +go-fuzz: ; $(ENV) "$(SHELL)" ./scripts/make/go-fuzz.sh +go-gen: ; $(ENV) "$(SHELL)" ./scripts/make/go-gen.sh +go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh +go-test: ; $(ENV) RACE='1' "$(SHELL)" ./scripts/make/go-test.sh +go-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-tools.sh +go-upd-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-upd-tools.sh go-check: go-tools go-lint go-test # A quick check to make sure that all operating systems relevant to the # development of the project can be typechecked and built successfully. go-os-check: - env GOOS='darwin' "$(GO.MACRO)" vet ./internal/... ./internal/dnsserver/... - env GOOS='linux' "$(GO.MACRO)" vet ./internal/... ./internal/dnsserver/... + $(ENV) GOOS='darwin' "$(GO.MACRO)" vet ./internal/... ./internal/dnsserver/... + $(ENV) GOOS='linux' "$(GO.MACRO)" vet ./internal/... ./internal/dnsserver/... # Additionally, check the AdGuard Home OSs in the dnsserver module. - env GOOS='freebsd' "$(GO.MACRO)" vet ./internal/dnsserver/... - env GOOS='openbsd' "$(GO.MACRO)" vet ./internal/dnsserver/... - env GOOS='windows' "$(GO.MACRO)" vet ./internal/dnsserver/... + $(ENV) GOOS='freebsd' "$(GO.MACRO)" vet ./internal/dnsserver/... + $(ENV) GOOS='openbsd' "$(GO.MACRO)" vet ./internal/dnsserver/... + $(ENV) GOOS='windows' "$(GO.MACRO)" vet ./internal/dnsserver/... -txt-lint: ; $(ENV) "$(SHELL)" ./scripts/make/txt-lint.sh +txt-lint: ; $(ENV) "$(SHELL)" ./scripts/make/txt-lint.sh + +md-lint: ; $(ENV_MISC) "$(SHELL)" ./scripts/make/md-lint.sh +sh-lint: ; $(ENV_MISC) "$(SHELL)" ./scripts/make/sh-lint.sh + +# Targets related to AdGuard DNS start here. sync-github: ; $(ENV) "$(SHELL)" ./scripts/make/github-sync.sh diff --git a/README.md b/README.md index 358f379..e4d20a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ - # AdGuard DNS +# AdGuard DNS + +
@@ -15,85 +17,60 @@

-AdGuard DNS is an alternative solution for tracker blocking, privacy protection, -and parental control. Easy to set up and free to use, it provides a necessary -minimum of best protection against online ads, trackers, and phishing, no matter -what platform and device you use. + +AdGuard DNS is an alternative solution for tracker blocking, privacy protection, and parental control. Easy to set up and free to use, it provides a necessary minimum of best protection against online ads, trackers, and phishing, no matter what platform and device you use. +## DNS privacy -## DNS Privacy +If you use regular client-server protocol, you are at risk of your DNS requests being intercepted and, subsequently, eavesdropped and/or altered. For instance, in the US the Senate voted to eliminate rules that restricted ISPs from selling their users' browsing data. Moreover, DNS is often used for censorship and surveillance purposes on the government level. -If you use regular client-server protocol, you are at risk of your DNS requests -being intercepted and, subsequently, eavesdropped and/or altered. For instance, -in the US the Senate voted to eliminate rules that restricted ISPs from selling -their users' browsing data. Moreover, DNS is often used for censorship and -surveillance purposes on the government level. +All of this is possible due to the lack of encryption, and AdGuard DNS provides a solution. It supports all known DNS encryption protocols including DNS-over-HTTPS, DNS-over-TLS, DNS-over-QUIC, and DNSCrypt. -All of this is possible due to the lack of encryption, and AdGuard DNS provides -a solution. It supports all known DNS encryption protocols including -DNS-over-HTTPS, DNS-over-TLS, DNS-over-QUIC, and DNSCrypt. - -On top of that, AdGuard DNS provides no-logs [privacy policy] which means we do -not record logs of your browsing activity. +On top of that, AdGuard DNS provides no-logs [privacy policy] which means we do not record logs of your browsing activity. [privacy policy]: https://adguard-dns.io/privacy.html +## Basic features +- **Blocking trackers network-wide** with no additional software required. You can even set it up on your router to block ads on all devices connected to your home Wi-Fi network. -## Basic Features +- Protection from phishing and hazardous websites and malvertising (malicious ads). - * **Blocking trackers network-wide** with no additional software required. - You can even set it up on your router to block ads on all devices connected - to your home Wi-Fi network. +- Use the **Family protection** mode of AdGuard DNS to block access to all websites with adult content and enforce safe search in the browser, in addition to the regular perks of ad blocking and browsing security. - * Protection from phishing and hazardous websites and malvertising (malicious - ads). +### Can AdGuard DNS replace a traditional blocker? - * Use the **Family protection** mode of AdGuard DNS to block access to all - websites with adult content and enforce safe search in the browser, in - addition to the regular perks of ad blocking and browsing security. +It depends. DNS-level blocking lacks the flexibility of the traditional ad blockers. For instance, there is no cosmetic pages processing. So in general, traditional blockers provide higher quality. - ### Can AdGuard DNS replace a traditional blocker? +## Personal DNS server -It depends. DNS-level blocking lacks the flexibility of the traditional ad -blockers. For instance, there is no cosmetic pages processing. So in general, -traditional blockers provide higher quality. - - - -## Personal DNS server +
A screenshot of the AdGuard DNS dashboard
AdGuard DNS dashboard
-You can sign up for a personal AdGuard DNS account and get access to the -following features: + - * Manage devices and their settings in one place. +You can sign up for a personal AdGuard DNS account and get access to the following features: - * Manage blocklists that are used to block ads. +- Manage devices and their settings in one place. - * View statistics on the DNS queries, companies, countries your devices try to - connect to. +- Manage blocklists that are used to block ads. - * You can also maintain your own set of rules in the "User rules" section. - AdGuard DNS provides a flexible [rules system]. +- View statistics on the DNS queries, companies, countries your devices try to connect to. - * AdGuard DNS also provides an [API] that can be used to integrate with it if - you need that. +- You can also maintain your own set of rules in the "User rules" section. AdGuard DNS provides a flexible [rules system]. + +- AdGuard DNS also provides an [API] that can be used to integrate with it if you need that. [rules system]: https://adguard-dns.io/kb/general/dns-filtering-syntax/ [API]: https://adguard-dns.io/kb/private-dns/api/overview/ +## Software license +Copyright (C) 2022-2024 AdGuard Software Ltd. -## Software License - -Copyright (C) 2022-2023 AdGuard Software Ltd. - -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU Affero General Public License as published by the Free -Software Foundation, version 3. +This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3. diff --git a/config.dist.yaml b/config.dist.yaml index a7fdcc0..170a4ed 100644 --- a/config.dist.yaml +++ b/config.dist.yaml @@ -166,6 +166,13 @@ geoip: # DNS checking configuration. check: # Domains to use for DNS checking. + kv: + # Defines the type of remote kay-value storage. Allowed values are + # "consul" and "redis". + type: 'consul' + # For how long to keep the information about the client. + ttl: 30s + # Domains to use for DNS checking. domains: - dnscheck.adguard-dns.com - dnscheck.adguard.com @@ -180,8 +187,6 @@ check: ipv6: - 1234::cdee - 1234::cdef - # For how long to keep the information about the client. - ttl: 30s # Web/HTTP(S) service configuration. All non-root requests to the main service # not matching the static_content map are shown a 404 page. In special @@ -462,6 +467,7 @@ server_groups: resolver_public: '9327C5E64783E19C339BD6B680A56DB85521CC6E4E0CA5DF5274E2D3CE026C6B' es_version: 1 certificate_ttl: 8760h + profiles_enabled: true # Connectivity check configuration. connectivity_check: diff --git a/doc/configuration.md b/doc/configuration.md index 5231360..c1655f5 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,178 +1,126 @@ - # AdGuard DNS configuration file +# AdGuard DNS configuration file -Besides the [environment][env], AdGuard DNS uses a [YAML][yaml] file to store -configuration. See file [`config.dist.yml`][dist] for a full example of a -configuration file with comments. +Besides the [environment][env], AdGuard DNS uses a [YAML][yaml] file to store configuration. See file [`config.dist.yml`][dist] for a full example of a configuration file with comments. -## Contents +## Contents - * [Recommended values and notes](#recommended) - * [Result cache sizes](#recommended-result_cache) - * [`SO_RCVBUF` and `SO_SNDBUF` on Linux](#recommended-buffers) - * [Connection limiter](#recommended-connection_limit) - * [Rate limiting](#ratelimit) - * [Stream connection limit](#ratelimit-connection_limit) - * [Cache](#cache) - * [Upstream](#upstream) - * [Healthcheck](#upstream-healthcheck) - * [Common DNS settings](#dns) - * [DNSDB](#dnsdb) - * [Backend](#backend) - * [Query log](#query_log) - * [GeoIP database](#geoip) - * [DNS-server check](#check) - * [Web API](#web) - * [Safe browsing](#safe_browsing) - * [Adult-content blocking](#adult_blocking) - * [Filters](#filters) - * [Filtering groups](#filtering_groups) - * [Network interface listeners](#interface_listeners) - * [Server groups](#server_groups) - * [DDR](#server_groups-*-ddr) - * [TLS](#server_groups-*-tls) - * [Servers](#server_groups-*-servers-*) - * [Connectivity check](#connectivity-check) - * [Network settings](#network) - * [Access settings](#access) - * [Additional metrics information](#additional_metrics_info) +- [Recommended values and notes](#recommended) + - [Result cache sizes](#recommended-result_cache) + - [`SO_RCVBUF` and `SO_SNDBUF` on Linux](#recommended-buffers) + - [Connection limiter](#recommended-connection_limit) +- [Rate limiting](#ratelimit) + - [Stream connection limit](#ratelimit-connection_limit) +- [Cache](#cache) +- [Upstream](#upstream) + - [Healthcheck](#upstream-healthcheck) +- [Common DNS settings](#dns) +- [DNSDB](#dnsdb) +- [Backend](#backend) +- [Query log](#query_log) +- [GeoIP database](#geoip) +- [DNS-server check](#check) +- [Web API](#web) +- [Safe browsing](#safe_browsing) +- [Adult-content blocking](#adult_blocking) +- [Filters](#filters) +- [Filtering groups](#filtering_groups) +- [Network interface listeners](#interface_listeners) +- [Server groups](#server_groups) + - [DDR](#server_groups-*-ddr) + - [TLS](#server_groups-*-tls) + - [Servers](#server_groups-*-servers-*) +- [Connectivity check](#connectivity-check) +- [Network settings](#network) +- [Access settings](#access) +- [Additional metrics information](#additional_metrics_info) [dist]: ../config.dist.yml [env]: environment.md [yaml]: https://yaml.org/ +## Recommended values and notes - -## Recommended values and notes - - ### Result cache sizes +### Result cache sizes According to AdGuard DNS usage data: - * requests for the top **1000** most commonly requested domains account for - approximately **85 %** of all queries; - * requests for the top **10 000** most commonly requested domains account for - approximately **95 %** of all queries; - * requests for the top **50 000** most commonly requested domains account for - approximately **98 %** of all queries; - * requests for the top **100 000** most commonly requested domains account - for approximately **99 %** of all queries. +- requests for the top **1000** most commonly requested domains account for approximately **85 %** of all queries; +- requests for the top **10 000** most commonly requested domains account for approximately **95 %** of all queries; +- requests for the top **50 000** most commonly requested domains account for approximately **98 %** of all queries; +- requests for the top **100 000** most commonly requested domains account for approximately **99 %** of all queries. -But these statistics are only about domain names and do not account for -differences in question types (`A`, `AAAA`, `HTTPS`, etc.) or whether a cached -record is a question or an answer. +But these statistics are only about domain names and do not account for differences in question types (`A`, `AAAA`, `HTTPS`, etc.) or whether a cached record is a question or an answer. -So, for example, if you want to reach 95 % cache-hit rate for a cache that -includes a question type in its cache key and also caches questions separately -from answers, you'll need to multiply the value from the statistic by 5 or 6. +So, for example, if you want to reach 95 % cache-hit rate for a cache that includes a question type in its cache key and also caches questions separately from answers, you'll need to multiply the value from the statistic by 5 or 6. +### `SO_RCVBUF` and `SO_SNDBUF` on Linux +On Linux OSs the values for these socket options coming from the configuration file (parameters [`network.so_rcvbuf`](#network-so_rcvbuf) and [`network.so_sndbuf`](#network-so_sndbuf)) is doubled, and the maximum and minimum values are controlled by the values in `/proc/`. See `man 7 socket`: - ### `SO_RCVBUF` and `SO_SNDBUF` on Linux +> `SO_RCVBUF` +> +> \[…\] The kernel doubles this value (to allow space for bookkeeping overhead) when it is set using setsockopt(2), and this doubled value is returned by getsockopt(2). The default value is set by the `/proc/sys/net/core/rmem_default` file, and the maximum allowed value is set by the `/proc/sys/net/core/rmem_max` file. The minimum (doubled) value for this option is `256`. +> +> \[…\] +> +> `SO_SNDBUF` +> +> \[…\] The default value is set by the `/proc/sys/net/core/wmem_default` file, and the maximum allowed value is set by the `/proc/sys/net/core/wmem_max` file. The minimum (doubled) value for this option is `2048`. -On Linux OSs the values for these socket options coming from the configuration -file (parameters [`network.so_rcvbuf`](#network-so_rcvbuf) and -[`network.so_sndbuf`](#network-so_sndbuf)) is doubled, and the maximum and -minimum values are controlled by the values in `/proc/`. See `man 7 socket`: +The maximum value for these parameters is the maximum value of a 32-bit signed integer (`2147483647`). - > `SO_RCVBUF` - > - > \[…\] The kernel doubles this value (to allow space for bookkeeping - > overhead) when it is set using setsockopt(2), and this doubled value is - > returned by getsockopt(2). The default value is set by the - > `/proc/sys/net/core/rmem_default` file, and the maximum allowed value is set - > by the `/proc/sys/net/core/rmem_max` file. The minimum (doubled) value for - > this option is `256`. - > - > \[…\] - > - > `SO_SNDBUF` - > - > \[…\] The default value is set by the `/proc/sys/net/core/wmem_default` - > file, and the maximum allowed value is set by the - > `/proc/sys/net/core/wmem_max` file. The minimum (doubled) value for this - > option is `2048`. +### Stream connection limit +Currently, there are the following recommendations for parameters [`ratelimit.connection_limit.stop`](#ratelimit-connection_limit-stop) and [`ratelimit.connection_limit.resume`](#ratelimit-connection_limit-resume): +- `stop` should be about 25 % above the current maximum daily number of used TCP sockets. That is, if the instance currently has a maximum of 100 000 TCP sockets in use every day, `stop` should be set to about `125000`. - ### Stream connection limit +- `resume` should be about 20 % above the current maximum daily number of used TCP sockets. That is, if the instance currently has a maximum of 100 000 TCP sockets in use every day, `resume` should be set to about `120000`. -Currently, there are the following recommendations for parameters -[`ratelimit.connection_limit.stop`](#ratelimit-connection_limit-stop) and -[`ratelimit.connection_limit.resume`](#ratelimit-connection_limit-resume): - - * `stop` should be about 25 % above the current maximum daily number of used - TCP sockets. That is, if the instance currently has a maximum of 100 000 - TCP sockets in use every day, `stop` should be set to about `125000`. - - * `resume` should be about 20 % above the current maximum daily number of used - TCP sockets. That is, if the instance currently has a maximum of 100 000 - TCP sockets in use every day, `resume` should be set to about `120000`. - - > [!NOTE] - > The number of active stream-connections includes sockets that are in the - > process of accepting new connections but have not yet accepted one. That - > means that `resume` should be greater than the number of bound addresses. +> [!NOTE] +> The number of active stream-connections includes sockets that are in the process of accepting new connections but have not yet accepted one. That means that `resume` should be greater than the number of bound addresses. These recommendations are to be revised based on the metrics. - - -## Rate limiting +## Rate limiting The `ratelimit` object has the following properties: - * `refuseany`: - If true, refuse DNS queries with the `ANY` (aka `*`) type. +- `refuseany`: If true, refuse DNS queries with the `ANY` (aka `*`) type. **Example:** `true`. - * `response_size_estimate`: - The size of one DNS response for the purposes of rate limiting. If a DNS - response is larger than this value, it is counted as several responses. +- `response_size_estimate`: The size of one DNS response for the purposes of rate limiting. If a DNS response is larger than this value, it is counted as several responses. **Example:** `1KB`. - * `backoff_period`: - The time during which to count the number of requests that a client has sent - over the RPS. +- `backoff_period`: The time during which to count the number of requests that a client has sent over the RPS. **Example:** `10m`. - * `backoff_duration`: - How long a client that has hit the RPS too often stays in the backoff state. +- `backoff_duration`: How long a client that has hit the RPS too often stays in the backoff state. **Example:** `30m`. - * `ipv4`: - The ipv4 configuration object. It has the following fields: +- `ipv4`: The ipv4 configuration object. It has the following fields: - * `rps`: - The rate of requests per second for one subnet. Requests above this are - counted in the backoff count. + - `rps`: The rate of requests per second for one subnet. Requests above this are counted in the backoff count. **Example:** `30`. - * `ipv4-subnet_key_len`: - The length of the subnet prefix used to calculate rate limiter bucket keys. + - `ipv4-subnet_key_len`: The length of the subnet prefix used to calculate rate limiter bucket keys. **Example:** `24`. - * `ipv6`: - The `ipv6` configuration object has the same properties as the `ipv4` one - above. +- `ipv6`: The `ipv6` configuration object has the same properties as the `ipv4` one above. - * `backoff_count`: - Maximum number of requests a client can make above the RPS within a - `backoff_period`. When a client exceeds this limit, requests aren't allowed - from client's subnet until `backoff_duration` ends. +- `backoff_count`: Maximum number of requests a client can make above the RPS within a `backoff_period`. When a client exceeds this limit, requests aren't allowed from client's subnet until `backoff_duration` ends. **Example:** `1000`. - * `allowlist`: - The allowlist configuration object. It has the following fields: +- `allowlist`: The allowlist configuration object. It has the following fields: - * `list`: - The array of the allowed IPs or CIDRs. + - `list`: The array of the allowed IPs or CIDRs. **Property example:** @@ -182,109 +130,77 @@ The `ratelimit` object has the following properties: - '192.175.2.1/16' ``` - * `refresh_interval`: - How often AdGuard DNS refreshes the dynamic part of its allowlist from - the data received from the - [`CONSUL_ALLOWLIST_URL`][env-consul_allowlist_url], as a human-readable - duration. + - `refresh_interval`: How often AdGuard DNS refreshes the dynamic part of its allowlist from the data received from the [`CONSUL_ALLOWLIST_URL`][env-consul_allowlist_url], as a human-readable duration. **Example:** `30s`. -For example, if `backoff_period` is `1m`, `backoff_count` is `10`, and -`ipv4-rps` is `5`, a client (meaning all IP addresses within the subnet defined -by `ipv4-subnet_key_len`) that made 15 requests in one second or 6 requests -(one above `rps`) every second for 10 seconds within one minute, the client is -blocked for `backoff_duration`. +For example, if `backoff_period` is `1m`, `backoff_count` is `10`, and `ipv4-rps` is `5`, a client (meaning all IP addresses within the subnet defined by `ipv4-subnet_key_len`) that made 15 requests in one second or 6 requests (one above `rps`) every second for 10 seconds within one minute, the client is blocked for `backoff_duration`. - ### Stream connection limit +### Stream connection limit The `connection_limit` object has the following properties: - * `enabled`: - Whether or not the stream-connection limit should be enforced. +- `enabled`: Whether or not the stream-connection limit should be enforced. **Example:** `true`. - * `stop`: - The point at which the limiter stops accepting new connections. Once the - number of active connections reaches this limit, new connections wait for - the number to decrease to or below `resume`. +- `stop`: The point at which the limiter stops accepting new connections. Once the number of active connections reaches this limit, new connections wait for the number to decrease to or below `resume`. **Example:** `1000`. - * `resume`: - The point at which the limiter starts accepting new connections again after - reaching `stop`. +- `resume`: The point at which the limiter starts accepting new connections again after reaching `stop`. **Example:** `800`. See also [notes on these parameters](#recommended-connection_limit). - ### QUIC rate limiting +### QUIC rate limiting The `quic` object has the following properties: - * `enabled`: - Whether or not the QUIC connections rate limiting should be enforced. +- `enabled`: Whether or not the QUIC connections rate limiting should be enforced. **Example:** `true`. - * `max_streams_per_peer`: - The maximum number of concurrent streams that a peer is allowed to open. +- `max_streams_per_peer`: The maximum number of concurrent streams that a peer is allowed to open. **Example:** `1000`. - ### TCP rate limiting +### TCP rate limiting The `tcp` object has the following properties: - * `enabled`: - Whether or not the TCP rate limiting should be enforced. +- `enabled`: Whether or not the TCP rate limiting should be enforced. **Example:** `true`. - * `max_pipeline_count`: - The maximum number of simultaneously processing TCP messages per one - connection. +- `max_pipeline_count`: The maximum number of simultaneously processing TCP messages per one connection. **Example:** `1000`. [env-consul_allowlist_url]: environment.md#CONSUL_ALLOWLIST_URL - - -## Cache +## Cache The `cache` object has the following properties: - * `type`: - The type of cache to use. Can be `simple` (a simple LRU cache) or `ecs` (a - ECS-aware LRU cache). If set to `ecs`, `ecs_size` must be greater than - zero. +- `type`: The type of cache to use. Can be `simple` (a simple LRU cache) or `ecs` (a ECS-aware LRU cache). If set to `ecs`, `ecs_size` must be greater than zero. **Example:** `simple`. - * `size`: - The total number of items in the cache for hostnames with no ECS support. - Must be greater than or equal to zero. If zero, cache is disabled. +- `size`: The total number of items in the cache for hostnames with no ECS support. Must be greater than or equal to zero. If zero, cache is disabled. **Example:** `10000`. - * `ecs_size`: - The total number of items in the cache for hostnames with ECS support. +- `ecs_size`: The total number of items in the cache for hostnames with ECS support. **Example:** `10000`. - * `ttl_override`: - The object describes cache TTL override mechanics. It has the following - properties: +- `ttl_override`: The object describes cache TTL override mechanics. It has the following properties: - * `enabled`: - If true, the TTL overrides are enabled. + - `enabled`: If true, the TTL overrides are enabled. - * `min`: - The minimum duration for TTL for cache items of both caches, with and - without ECS support. The recommended value is `60s`. + - `min`: The minimum duration for TTL for cache items of both caches, with and without ECS support. The recommended value is `60s`. **Property example:** @@ -294,16 +210,11 @@ The `cache` object has the following properties: 'min': 60s ``` - - -## Upstream +## Upstream The `upstream` object has the following properties: - * `servers`: - The array of the main upstream servers URLs, in the `[scheme://]ip:port` - format and its timeouts for main upstream DNS requests, as a human-readable - duration. +- `servers`: The array of the main upstream servers URLs, in the `[scheme://]ip:port` format and its timeouts for main upstream DNS requests, as a human-readable duration. **Property example:** @@ -320,15 +231,9 @@ The `upstream` object has the following properties: timeout: 2s ``` - * `fallback`: - Fallback servers configuration. It has the following properties: +- `fallback`: Fallback servers configuration. It has the following properties: - * `servers`: - The array of the fallback upstream servers URLs, in the - `[scheme://]ip:port` format and its timeouts for upstream DNS requests, - as a human-readable duration. These are use used in case a network error - occurs while requesting the main upstream server. This property has the - same format as [`upstream-servers`](#upstream-servers) above. + - `servers`: The array of the fallback upstream servers URLs, in the `[scheme://]ip:port` format and its timeouts for upstream DNS requests, as a human-readable duration. These are use used in case a network error occurs while requesting the main upstream server. This property has the same format as [`upstream-servers`](#upstream-servers) above. **Property example:** @@ -340,161 +245,101 @@ The `upstream` object has the following properties: timeout: 2s ``` - * `healthcheck`: Healthcheck configuration. See - [below](#upstream-healthcheck). +- `healthcheck`: Healthcheck configuration. See [below](#upstream-healthcheck). +### Healthcheck +If `enabled` is true, the upstream healthcheck is enabled. The healthcheck worker probes the main upstream with an `A` query for a domain created from `domain_template`. If there is an error, timeout, or a response different from a `NOERROR` one then the main upstream is considered down, and all requests are redirected to fallback upstream servers for the time set by `backoff_duration`. Afterwards, if a worker probe is successful, AdGuard DNS considers the connection to the main upstream as restored, and requests are routed back to it. - ### Healthcheck - -If `enabled` is true, the upstream healthcheck is enabled. The healthcheck -worker probes the main upstream with an `A` query for a domain created from -`domain_template`. If there is an error, timeout, or a response different from -a `NOERROR` one then the main upstream is considered down, and all requests are -redirected to fallback upstream servers for the time set by `backoff_duration`. -Afterwards, if a worker probe is successful, AdGuard DNS considers the -connection to the main upstream as restored, and requests are routed back to it. - - * `enabled`: - - If true, the upstream healthcheck is enabled. +- `enabled`: If true, the upstream healthcheck is enabled. **Example:** `true`. - * `interval`: - How often AdGuard DNS makes upstream healthcheck requests, as a - human-readable duration. +- `interval`: How often AdGuard DNS makes upstream healthcheck requests, as a human-readable duration. **Example:** `2s`. - * `timeout`: - Timeout for all outgoing healthcheck requests, as a human-readable duration. +- `timeout`: Timeout for all outgoing healthcheck requests, as a human-readable duration. **Example:** `1s`. - * `backoff_duration`: - Backoff duration after failed healthcheck request, as a human-readable - duration. If the main upstream is down, AdGuardDNS does not return back to - using it until this time has passed. The healthcheck is still performed, - and each failed check advances the backoff. +- `backoff_duration`: Backoff duration after failed healthcheck request, as a human-readable duration. If the main upstream is down, AdGuardDNS does not return back to using it until this time has passed. The healthcheck is still performed, and each failed check advances the backoff. **Example:** `30s`. - * `domain_template`: - The template for domains used to perform healthcheck queries. If the - `domain_template` contains the string `${RANDOM}`, all occurrences of this - string are replaced with a random string (currently, a hexadecimal form of a - 64-bit integer) on every healthcheck query. Queries must return a `NOERROR` - response. +- `domain_template`: The template for domains used to perform healthcheck queries. If the `domain_template` contains the string `${RANDOM}`, all occurrences of this string are replaced with a random string (currently, a hexadecimal form of a 64-bit integer) on every healthcheck query. Queries must return a `NOERROR` response. **Example:** `${RANDOM}.neverssl.com`. - - -## DNS +## DNS The `dns` object has the following properties: - * `read_timeout`: - The timeout for any read from a UDP connection or the first read from - a TCP/TLS connection, as a human-readable duration. It currently doesn't - affect DNSCrypt, QUIC, or HTTPS. +- `read_timeout`: The timeout for any read from a UDP connection or the first read from a TCP/TLS connection, as a human-readable duration. It currently doesn't affect DNSCrypt, QUIC, or HTTPS. **Example:** `2s`. - * `tcp_idle_timeout`: - The timeout for consecutive reads from a TCP/TLS connection, as a - human-readable duration. It currently doesn't affect DNSCrypt, QUIC, or - HTTPS. +- `tcp_idle_timeout`: The timeout for consecutive reads from a TCP/TLS connection, as a human-readable duration. It currently doesn't affect DNSCrypt, QUIC, or HTTPS. **Example:** `30s`. - * `write_timeout`: - The timeout for writing to a UDP or TCP/TLS connection, as a human-readable - duration. It currently doesn't affect DNSCrypt, QUIC, or HTTPS. +- `write_timeout`: The timeout for writing to a UDP or TCP/TLS connection, as a human-readable duration. It currently doesn't affect DNSCrypt, QUIC, or HTTPS. **Example:** `2s`. - * `handle_timeout`: - The timeout for the entire handling of a single query, as a human-readable - duration. +- `handle_timeout`: The timeout for the entire handling of a single query, as a human-readable duration. **Example:** `1s`. - * `max_udp_response_size`: - The maximum size of DNS response over UDP protocol. +- `max_udp_response_size`: The maximum size of DNS response over UDP protocol. **Example:** `1024B`. - - -## DNSDB +## DNSDB The `dnsdb` object has the following properties: - * `enabled`: - If true, the DNSDB memory buffer is enabled. +- `enabled`: If true, the DNSDB memory buffer is enabled. **Example:** `true`. - * `max_size`: - The maximum number of records in the in-memory buffer. The record key is a - combination of the target hostname from the question and the resource-record - type of the question or the answer. +- `max_size`: The maximum number of records in the in-memory buffer. The record key is a combination of the target hostname from the question and the resource-record type of the question or the answer. **Example:** `500000`. - - -## Backend +## Backend The `backend` object has the following properties: - * `timeout`: - Timeout for all outgoing HTTP requests to the backend, as a human-readable - duration. Set to `0s` to disable timeouts. +- `timeout`: Timeout for all outgoing HTTP requests to the backend, as a human-readable duration. Set to `0s` to disable timeouts. **Example:** `10s`. - * `refresh_interval`: - How often AdGuard DNS checks the backend for data updates, as a - human-readable duration. +- `refresh_interval`: How often AdGuard DNS checks the backend for data updates, as a human-readable duration. **Example:** `1m`. - * `full_refresh_interval`: - How often AdGuard DNS performs a full profile refresh, as a human-readable - duration. If [`PROFILES_CACHE_PATH`][env-profiles_cache_path] is set, the - profile cache is also saved after a full refresh. +- `full_refresh_interval`: How often AdGuard DNS performs a full profile refresh, as a human-readable duration. If [`PROFILES_CACHE_PATH`][env-profiles_cache_path] is set, the profile cache is also saved after a full refresh. **Example:** `24h`. - * `full_refresh_retry_interval`: - How long to wait before attempting a new full profile synchronization after - a failure, as a human-readable duration. It is recommended to keep this - value greater than [`refresh_interval`](#backend-refresh_interval). +- `full_refresh_retry_interval`: How long to wait before attempting a new full profile synchronization after a failure, as a human-readable duration. It is recommended to keep this value greater than [`refresh_interval`](#backend-refresh_interval). **Example:** `1h`. - * `bill_stat_interval`: - How often AdGuard DNS sends the billing statistics to the backend, as - a human-readable duration. +- `bill_stat_interval`: How often AdGuard DNS sends the billing statistics to the backend, as a human-readable duration. **Example:** `1m`. [env-profiles_cache_path]: environment.md#PROFILES_CACHE_PATH - - -## Query log +## Query log The `query_log` object has the following properties: - * `file`: - The file query log configuration object. It has the following properties: +- `file`: The file query log configuration object. It has the following properties: - * `enabled`: - If true, the JSONL file query logging is enabled. + - `enabled`: If true, the JSONL file query logging is enabled. **Property example:** @@ -503,35 +348,41 @@ The `query_log` object has the following properties: 'enabled': true ``` - - -## GeoIP database +## GeoIP database The `geoip` object has the following properties: - * `host_cache_size`: - The size of the host lookup cache, in entries. +- `host_cache_size`: The size of the host lookup cache, in entries. **Example:** `100000`. - * `ip_cache_size`: - The size of the IP lookup cache, in entries. +- `ip_cache_size`: The size of the IP lookup cache, in entries. **Example:** `100000`. - * `refresh_interval`: - Interval between the GeoIP database refreshes, as a human-readable duration. +- `refresh_interval`: Interval between the GeoIP database refreshes, as a human-readable duration. **Example:** `5m`. - - -## DNS-server check +## DNS-server check The `check` object has the following properties: - * `domains`: - The domain suffixes to which random IDs are prepended using a hyphen. +- `kv`: Remote key-value storage settings. It has the following properties: + + - `type`: Type of the remote KV storage. Allowed values are `consul` and `redis`. + + **Example:** `consul`. + + - `ttl`: For how long to keep the information about a single user in remote KV, as a human-readable duration. + + For `consul`, the TTL must be between `10s` and `1d`. Note that the actual TTL can be up to twice as long. + + For `redis`, the TTL must be greater than or equal to `1ms`. + + **Example:** `30s`. + +- `domains`: The domain suffixes to which random IDs are prepended using a hyphen. **Property example:** @@ -541,29 +392,15 @@ The `check` object has the following properties: - 'checkdns.example.com' ``` - * `node_location`: - The location code of this server node. +- `node_location`: The location code of this server node. **Example:** `ams`. - * `node_name`: - The name of this server node. +- `node_name`: The name of this server node. **Example:** `eu-1.dns.example.com`. - * `ttl`: - For how long to keep the information about a single user in Consul KV, as - a human-readable duration. Note the actual TTL may be up to twice as long - due to Consul's peculiarities. - - **Example:** `30s`. - - * `ipv4` and `ipv6`: - Arrays of IPv4 or IPv6 addresses with which to respond to `A` and `AAAA` - queries correspondingly. Generally, those should be the IP addresses of the - AdGuard DNS [main HTTP API][http-dnscheck] for the DNS server check feature - to work properly. In a development setup, that means the localhost - addresses. +- `ipv4` and `ipv6`: Arrays of IPv4 or IPv6 addresses with which to respond to `A` and `AAAA` queries correspondingly. Generally, those should be the IP addresses of the AdGuard DNS [main HTTP API][http-dnscheck] for the DNS server check feature to work properly. In a development setup, that means the localhost addresses. **Property examples:** @@ -578,22 +415,15 @@ The `check` object has the following properties: [http-dnscheck]: http.md#dnscheck-test - - -## Web API +## Web API The optional `web` object has the following properties: - * `linked_ip`: - The optional linked IP and dynamic DNS (DDNS, DynDNS) web server - configuration. The [static content](#web-static_content) is not served on - these addresses. +- `linked_ip`: The optional linked IP and dynamic DNS (DDNS, DynDNS) web server configuration. The [static content](#web-static_content) is not served on these addresses. - See the [full description of this API][http-linked-ip-proxy] on the HTTP API - page. + See the [full description of this API][http-linked-ip-proxy] on the HTTP API page. - Property `bind` has the same format as [`non_doh_bind`](#web-non_doh_bind) - below. + Property `bind` has the same format as [`non_doh_bind`](#web-non_doh_bind) below. **Property example:** @@ -607,22 +437,13 @@ The optional `web` object has the following properties: 'key': './test/cert.key' ``` - * `safe_browsing`: - The optional safe browsing block-page web server configurations. Every - request is responded with the content from the file to which the - `block_page` property points. +- `safe_browsing`: The optional safe browsing block-page web server configurations. Every request is responded with the content from the file to which the `block_page` property points. - See the [full description of this API][http-block-pages] on the HTTP API - page. + See the [full description of this API][http-block-pages] on the HTTP API page. - Property `bind` has the same format as [`non_doh_bind`](#web-non_doh_bind) - below. The addresses should be different from the `adult_blocking` server, - and the same as the ones of the `block_host` property in the - [`safe_browsing`](#safe_browsing) and [`adult_blocking`](#adult_blocking) - objects correspondingly. + Property `bind` has the same format as [`non_doh_bind`](#web-non_doh_bind) below. The addresses should be different from the `adult_blocking` server, and the same as the ones of the `block_host` property in the [`safe_browsing`](#safe_browsing) and [`adult_blocking`](#adult_blocking) objects correspondingly. - While this object is optional, both `bind` and `block_page` properties - within them are required. + While this object is optional, both `bind` and `block_page` properties within them are required. **Property examples:** @@ -637,23 +458,11 @@ The optional `web` object has the following properties: 'block_page': '/var/www/block_page.html' ``` - * `adult_blocking`: - The optional adult block-page web server configuration. The format of the - values is the same as in the [`safe_browsing`](#web-safe_browsing) object - above. +- `adult_blocking`: The optional adult block-page web server configuration. The format of the values is the same as in the [`safe_browsing`](#web-safe_browsing) object above. - * `general_blocking`: - The optional general block-page web server configuration. The format of - the values is the same as in the [`safe_browsing`](#web-safe_browsing) - object above. +- `general_blocking`: The optional general block-page web server configuration. The format of the values is the same as in the [`safe_browsing`](#web-safe_browsing) object above. - * `non_doh_bind`: - The optional listen addresses and optional TLS configuration for the web - service in addition to the ones in the DNS-over-HTTPS handlers. The - `certificates` array has the same format as the one in a server group's [TLS - settings](#server_groups-*-tls). In the special case of `GET /robots.txt` - requests, a special response is served; this response could be overwritten - with static content. +- `non_doh_bind`: The optional listen addresses and optional TLS configuration for the web service in addition to the ones in the DNS-over-HTTPS handlers. The `certificates` array has the same format as the one in a server group's [TLS settings](#server_groups-*-tls). In the special case of `GET /robots.txt` requests, a special response is served; this response could be overwritten with static content. **Property example:** @@ -666,15 +475,12 @@ The optional `web` object has the following properties: 'key': './test/cert.key' ``` - * `static_content`: - The optional inline static content mapping. Not served on the `linked_ip`, - `safe_browsing` and `adult_blocking` servers. Paths must not duplicate the - ones used by the DNS-over-HTTPS server. +- `static_content`: The optional inline static content mapping. Not served on the `linked_ip`, `safe_browsing` and `adult_blocking` servers. Paths must not duplicate the ones used by the DNS-over-HTTPS server. - Inside of the `headers` map, the header `Content-Type` is required. + > [!NOTE] + > This field is ignored if [`WEB_STATIC_DIR_ENABLED`][env-WEB_STATIC_DIR_ENABLED] is set to `1`. - > [!NOTE] - > Paths are case-sensitive. + Inside of the `headers` map, the header `Content-Type` is required. The paths are case-sensitive. **Property example:** @@ -687,259 +493,177 @@ The optional `web` object has the following properties: - 'image/x-icon' ``` - * `root_redirect_url`: - The optional URL to which non-DNS and non-Debug HTTP requests are - redirected. If not set, AdGuard DNS will respond with a 404 status to all - such requests. +- `root_redirect_url`: The optional URL to which non-DNS and non-Debug HTTP requests are redirected. If not set, AdGuard DNS will respond with a 404 status to all such requests. **Example:** `https://adguard-dns.com/`. - * `error_404` and `error_500`: - The optional paths to the 404 page and the 500 page HTML files - correspondingly. If not set, a simple plain text 404 or 500 page is served. +- `error_404` and `error_500`: The optional paths to the 404 page and the 500 page HTML files correspondingly. If not set, a simple plain text 404 or 500 page is served. **Example:** `/var/www/404.html`. - * `timeout`: - The timeout for server operations, as a human-readable duration. +- `timeout`: The timeout for server operations, as a human-readable duration. **Example:** `30s`. -[http-block-pages]: http.md#block-pages -[http-linked-ip-proxy]: http.md#linked-ip-proxy +[env-WEB_STATIC_DIR_ENABLED]: environment.md#WEB_STATIC_DIR_ENABLED +[http-block-pages]: http.md#block-pages +[http-linked-ip-proxy]: http.md#linked-ip-proxy - - -## Safe browsing +## Safe browsing The `safe_browsing` object has the following properties: - * `block_host`: - The host with which to respond to any requests that match the filter. +- `block_host`: The host with which to respond to any requests that match the filter. **Example:** `standard-block.dns.adguard.com`. - * `cache_size`: - The size of the response cache, in entries. +- `cache_size`: The size of the response cache, in entries. **WARNING: CURRENTLY IGNORED!** See AGDNS-398. **Example:** `1024`. - * `cache_ttl`: - The TTL of the response cache, as a human-readable duration. +- `cache_ttl`: The TTL of the response cache, as a human-readable duration. **Example:** `1h`. - * `url`: - The URL from which the contents can be updated. The URL must reply with - a 200 status code. +- `url`: The URL from which the contents can be updated. The URL must reply with a 200 status code. **Example:** `https://example.com/safe_browsing.txt`. - * `refresh_interval`: - How often AdGuard DNS refreshes the filter. +- `refresh_interval`: How often AdGuard DNS refreshes the filter. **Example:** `1m`. - * `refresh_timeout`: - The timeout for the update operation, as a human-readable duration. +- `refresh_timeout`: The timeout for the update operation, as a human-readable duration. **Example:** `1m`. +## Adult-content blocking +The `adult_blocking` object has the same properties as the [`safe_browsing`](#safe_browsing) one above. -## Adult-content blocking +## Filter Lists -The `adult_blocking` object has the same properties as the -[`safe_browsing`](#safe_browsing) one above. - - - -## Filter Lists - -**TODO(a.garipov):** Add the timeout for the blocked-service index refresh. It -is currently hardcoded to 3 minutes. +**TODO(a.garipov):** Add the timeout for the blocked-service index refresh. It is currently hardcoded to 3 minutes. The `filters` object has the following properties: - * `response_ttl`: - The default TTL to set for responses to queries for blocked or modified - domains, as a human-readable duration. It is used for anonymous users. For - users with profiles, the TTL from their profile settings are used. +- `response_ttl`: The default TTL to set for responses to queries for blocked or modified domains, as a human-readable duration. It is used for anonymous users. For users with profiles, the TTL from their profile settings are used. **Example:** `10s`. - * `custom_filter_cache_size`: - The size of the LRU cache of compiled filtering rule engines for profiles - with custom filtering rules, in entries. Zero means no caching, which slows +- `custom_filter_cache_size`: The size of the LRU cache of compiled filtering rule engines for profiles with custom filtering rules, in entries. Zero means no caching, which slows down queries. **Example:** `1024`. - * `safe_search_cache_size`: - The size of the LRU cache of the safe-search filtering results. This value - applies to both general and YouTube safe-search. +- `safe_search_cache_size`: The size of the LRU cache of the safe-search filtering results. This value applies to both general and YouTube safe-search. **Example:** `1024`. - * `refresh_interval`: - How often AdGuard DNS refreshes the rule-list filters from the filter index, - as well as the blocked services list from the [blocked list - index][env-blocked_services]. +- `refresh_interval`: How often AdGuard DNS refreshes the rule-list filters from the filter index, as well as the blocked services list from the [blocked list index][env-blocked_services]. **Example:** `1h`. - * `refresh_timeout`: - The timeout for the *entire* filter update operation, as a human-readable - duration. Note that filter rule-list index and each filter rule-list - update operations have their own timeouts, see - [`index_refresh_timeout`](#filters-index_refresh_timeout) and - [`rule_list_refresh_timeout`](#filters-rule_list_refresh_timeout). +- `refresh_timeout`: The timeout for the *entire* filter update operation, as a human-readable duration. Note that filter rule-list index and each filter rule-list update operations have their own timeouts, see [`index_refresh_timeout`](#filters-index_refresh_timeout) and [`rule_list_refresh_timeout`](#filters-rule_list_refresh_timeout). **Example:** `5m`. - * `index_refresh_timeout`: - The timeout for the filter rule-list index update operation, as a - human-readable duration. See also - [`refresh_timeout`](#filters-refresh_timeout) for the entire filter update - operation. +- `index_refresh_timeout`: The timeout for the filter rule-list index update operation, as a human-readable duration. See also [`refresh_timeout`](#filters-refresh_timeout) for the entire filter update operation. **Example:** `1m`. - * `rule_list_refresh_timeout`: - The timeout for the filter update operation of each rule-list, including the - safe-search ones, as a human-readable duration. See also - [`refresh_timeout`](#filters-refresh_timeout) for the entire filter update - operation. +- `rule_list_refresh_timeout`: The timeout for the filter update operation of each rule-list, including the safe-search ones, as a human-readable duration. See also [`refresh_timeout`](#filters-refresh_timeout) for the entire filter update operation. **Example:** `1m`. - * `max_size`: - The maximum size of the downloadable content for a rule-list in a - human-readable format. +- `max_size`: The maximum size of the downloadable content for a rule-list in a human-readable format. **Example:** `256MB`. - * `rule_list_cache`: - Rule lists cache settings. It has the following properties: +- `rule_list_cache`: Rule lists cache settings. It has the following properties: - * `enabled`: - If true, use the rule-list filtering result cache. This cache is not - used for users' custom rules. + - `enabled`: If true, use the rule-list filtering result cache. This cache is not used for users' custom rules. **Example:** `true`. - * `rule_list_cache-size`: - The size of the LRU cache of the rule-list filtering results. + - `rule_list_cache-size`: The size of the LRU cache of the rule-list filtering results. **Example:** `10000`. [env-blocked_services]: environment.md#BLOCKED_SERVICE_INDEX_URL - - -## Filtering groups +## Filtering groups The items of the `filtering_groups` array have the following properties: - * `id`: - The unique ID of this filtering group. - +- `id`: The unique ID of this filtering group. **Example:** `default`. - * `rule_lists`: - Filtering rule lists settings. This object has the following properties: +- `rule_lists`: Filtering rule lists settings. This object has the following properties: - * `enabled`: - Shows if rule-list filtering should be enforced. If it is set to - `false`, the rest of the settings are ignored. + - `enabled`: Shows if rule-list filtering should be enforced. If it is set to `false`, the rest of the settings are ignored. **Example:** `true`. - * `ids`: - The array of rule-list IDs used in this filtering group. + - `ids`: The array of rule-list IDs used in this filtering group. **Example:** `[adguard_dns_default]`. - * `parental`: - Parental protection settings. This object has the following properties: +- `parental`: Parental protection settings. This object has the following properties: - * `enabled`: - Shows if any kind of parental protection filtering should be enforced at - all. If it is set to `false`, the rest of the settings are ignored. + - `enabled`: Shows if any kind of parental protection filtering should be enforced at all. If it is set to `false`, the rest of the settings are ignored. **Example:** `true`. - * `block_adult`: - If true, adult content blocking is enabled for this filtering group by - default. Requires `enabled` to also be true. + - `block_adult`: If true, adult content blocking is enabled for this filtering group by default. Requires `enabled` to also be true. **Example:** `true`. - * `general_safe_search`: - If true, general safe search is enabled for this filtering group by - default. Requires `enabled` to also be true. + - `general_safe_search`: If true, general safe search is enabled for this filtering group by default. Requires `enabled` to also be true. **Example:** `true`. - * `youtube_safe_search`: - If true, YouTube safe search is enabled for this filtering group by - default. Requires `enabled` to also be true. + - `youtube_safe_search`: If true, YouTube safe search is enabled for this filtering group by default. Requires `enabled` to also be true. **Example:** `true`. - * `safe_browsing`: - General safe browsing settings. This object has the following properties: +- `safe_browsing`: General safe browsing settings. This object has the following properties: - * `enabled`: - Shows if the general safe browsing filtering should be enforced. If it - is set to `false`, the rest of the settings are ignored. + - `enabled`: Shows if the general safe browsing filtering should be enforced. If it is set to `false`, the rest of the settings are ignored. **Example:** `true`. - * `block_dangerous_domains`: - Shows if the dangerous domains filtering should be enforced. + - `block_dangerous_domains`: Shows if the dangerous domains filtering should be enforced. **Example:** `true`. - * `block_newly_registered_domains`: - Shows if the newly registered domains filtering should be enforced. + - `block_newly_registered_domains`: Shows if the newly registered domains filtering should be enforced. **Example:** `true`. - * `private_relay`: - If true, Apple Private Relay queries are blocked for requests using this - filtering group. +- `private_relay`: If true, Apple Private Relay queries are blocked for requests using this filtering group. **Example:** `false`. - * `block_firefox_canary`: - If true, Firefox canary domain queries are blocked for requests using this - filtering group. +- `block_firefox_canary`: If true, Firefox canary domain queries are blocked for requests using this filtering group. **Example:** `true`. +## Network interface listeners - -## Network interface listeners - - > [!NOTE] - > The network interface listening works only on Linux with `SO_BINDTODEVICE` - > support (2.0.30 and later) and properly setup IP routes. See the [section - > on testing `SO_BINDTODEVICE` using Docker][dev-btd]. +> [!NOTE] +> The network interface listening works only on Linux with `SO_BINDTODEVICE` support (2.0.30 and later) and properly setup IP routes. See the [section on testing `SO_BINDTODEVICE` using Docker][dev-btd]. The `interface_listeners` object has the following properties: - * `channel_buffer_size`: - The size of the buffers of the channels used to dispatch TCP connections and - UDP sessions. +- `channel_buffer_size`: The size of the buffers of the channels used to dispatch TCP connections and UDP sessions. **Example:** `1000`. - * `list`: - The mapping of interface-listener IDs to their configuration. +- `list`: The mapping of interface-listener IDs to their configuration. **Property example:** @@ -955,81 +679,54 @@ The `interface_listeners` object has the following properties: [dev-btd]: development.md#testing-bindtodevice - - -## Server groups +## Server groups The items of the `server_groups` array have the following properties: - * `name`: - The unique name of this server group. +- `name`: The unique name of this server group. **Example:** `adguard_dns_default`. - * `filtering_group`: - The default filtering group for this server group. It is used for anonymous - users. +- `filtering_group`: The default filtering group for this server group. It is used for anonymous users. **Example:** `default`. - * `ddr`: The DDR configuration object. See [below](#server_groups-*-ddr). +- `ddr`: The DDR configuration object. See [below](#server_groups-*-ddr). - * `tls`: The TLS configuration object. See [below](#server_groups-*-tls). +- `tls`: The TLS configuration object. See [below](#server_groups-*-tls). - > [!NOTE] - > The `tls` object is optional unless the [`servers` - > array](#server_groups-*-servers-*) contains at least one item with an - > encrypted protocol. + > [!NOTE] + > The `tls` object is optional unless the [`servers` array](#server_groups-*-servers-*) contains at least one item with an encrypted protocol. - * `servers`: Server configuration for this filtering group. See - [below](#server_groups-*-servers-*). - - - - ### DDR - -The DDR configuration object. Many of these data duplicate data from objects in -the [`servers`](#server_groups-*-servers-*) array. This was done because there -was an opinion that a more restrictive configuration that automatically -collected the required data was not self-describing and flexible enough. - - * `enabled`: - Shows if DDR queries are processed. If it is set to `false`, DDR domain - name queries receive an `NXDOMAIN` response. +- `profiles_enabled`: If true, enable recognition of user devices and profiles for this server group. **Example:** `true`. - * `device_records`: - The device ID wildcard to record template mapping. The keys should - generally be kept in sync with the - [`device_id_wildcards`](#sg-*-tls-device_id_wildcards) field of the `tls` - object. +- `servers`: Server configuration for this filtering group. See [below](#server_groups-*-servers-*). + +### DDR + +The DDR configuration object. Many of these data duplicate data from objects in the [`servers`](#server_groups-*-servers-*) array. This was done because there was an opinion that a more restrictive configuration that automatically collected the required data was not self-describing and flexible enough. + +- `enabled`: Shows if DDR queries are processed. If it is set to `false`, DDR domain name queries receive an `NXDOMAIN` response. + + **Example:** `true`. + +- `device_records`: The device ID wildcard to record template mapping. The keys should generally be kept in sync with the [`device_id_wildcards`](#sg-*-tls-device_id_wildcards) field of the `tls` object. The values have the following properties: - * `doh_path`: - The path template for the DoH DDR SVCB records. It is optional, unless - `https_port` below is set. + - `doh_path`: The path template for the DoH DDR SVCB records. It is optional, unless `https_port` below is set. - * `https_port`: - The optional port to use in DDR responses about the DoH resolver. If it - is zero, the DoH resolver address is not included into the answer. A - non-zero `https_port` should not be the same as `tls_port` below. + - `https_port`: The optional port to use in DDR responses about the DoH resolver. If it is zero, the DoH resolver address is not included into the answer. A non-zero `https_port` should not be the same as `tls_port` below. - * `quic_port`: - The optional port to use in DDR responses about the DoQ resolver. If it - is zero, the DoQ resolver address is not included into the answer. + - `quic_port`: The optional port to use in DDR responses about the DoQ resolver. If it is zero, the DoQ resolver address is not included into the answer. - * `tls_port`: - The optional port to use in DDR responses about the DoT resolver. If it - is zero, the DoT resolver address is not included into the answer. A - non-zero `tls_port` should not be the same as `https_port` above. + - `tls_port`: The optional port to use in DDR responses about the DoT resolver. If it is zero, the DoT resolver address is not included into the answer. A non-zero `tls_port` should not be the same as `https_port` above. - * `ipv4_hints`: - The optional hints about the IPv4-addresses of the server. + - `ipv4_hints`: The optional hints about the IPv4-addresses of the server. - * `ipv6_hints`: - The optional hints about the IPv6-addresses of the server. + - `ipv6_hints`: The optional hints about the IPv6-addresses of the server. **Property example:** @@ -1055,19 +752,12 @@ collected the required data was not self-describing and flexible enough. - '2001::5678' ``` - - * `public_records`: - The public domain name to DDR record template mapping. The format of the - values is the same as in the [`device_records`](#sg-*-ddr-device_records) +- `public_records`: The public domain name to DDR record template mapping. The format of the values is the same as in the [`device_records`](#sg-*-ddr-device_records) above. +### TLS - - ### TLS - - * `certificates`: - The array of objects with paths to the certificate and the private key for - this server group. +- `certificates`: The array of objects with paths to the certificate and the private key for this server group. **Property example:** @@ -1077,9 +767,7 @@ collected the required data was not self-describing and flexible enough. 'key': '/etc/dns/cert.key' ``` - * `session_keys`: - The array of file paths from which the each server's TLS session keys are - updated. Session ticket key files must contain at least 32 bytes. +- `session_keys`: The array of file paths from which the each server's TLS session keys are updated. Session ticket key files must contain at least 32 bytes. **Property example:** @@ -1089,9 +777,7 @@ collected the required data was not self-describing and flexible enough. - './private/key_2' ``` - * `device_id_wildcards`: - The array of domain name wildcards to use to detect clients' device IDs. - Use this to prevent conflicts when using certificates for subdomains. +- `device_id_wildcards`: The array of domain name wildcards to use to detect clients' device IDs. Use this to prevent conflicts when using certificates for subdomains. **Property example:** @@ -1100,44 +786,35 @@ collected the required data was not self-describing and flexible enough. - '*.d.dns.example.com' ``` - - - ### Servers +### Servers The items of the `servers` array have the following properties: - * `name`: - The unique name of this server. +- `name`: The unique name of this server. **Example:** `default_dns`. - * `protocol`: - The protocol to use on this server. The following values are supported: +- `protocol`: The protocol to use on this server. The following values are supported: - * `dns` - * `dnscrypt` - * `https` - * `quic` - * `tls` + - `dns` + - `dnscrypt` + - `https` + - `quic` + - `tls` **Example:** `dns`. - * `linked_ip_enabled`: - If true, use the profiles' linked IPs to detect. +- `linked_ip_enabled`: If true, use the profiles' linked IPs to detect. **Default:** `false`. **Example:** `true`. - * `bind_addresses`: - The array of `ip:port` addresses to listen on. If `bind_addresses` is set, - `bind_interfaces` (see below) should not be set. +- `bind_addresses`: The array of `ip:port` addresses to listen on. If `bind_addresses` is set, `bind_interfaces` (see below) should not be set. **Example:** `[127.0.0.1:53, 192.168.1.1:53]`. - * `bind_interfaces`: - The array of [interface listener](#ifl-list) data. If `bind_interfaces` is - set, `bind_addresses` (see above) should not be set. +- `bind_interfaces`: The array of [interface listener](#ifl-list) data. If `bind_interfaces` is set, `bind_addresses` (see above) should not be set. **Property example:** @@ -1151,21 +828,15 @@ The items of the `servers` array have the following properties: - '172.17.0.0/16' ``` - * `dnscrypt`: - The optional DNSCrypt configuration object. It has the following - properties: +- `dnscrypt`: The optional DNSCrypt configuration object. It has the following properties: - * `config_path`: - The path to the DNSCrypt configuration file. See the [configuration - section][dnscconf] of the DNSCrypt module. + - `config_path`: The path to the DNSCrypt configuration file. See the [configuration section][dnscconf] of the DNSCrypt module. Must not be set if `inline` is set. **Example:** `/etc/dns/dnscrypt.yml` - * `inline`: - The DNSCrypt configuration, inline. See the [configuration - section][dnscconf] of the DNSCrypt module. + - `inline`: The DNSCrypt configuration, inline. See the [configuration section][dnscconf] of the DNSCrypt module. Must not be set if `config_path` is set. @@ -1184,71 +855,51 @@ The items of the `servers` array have the following properties: [dnscconf]: https://github.com/ameshkov/dnscrypt/blob/master/README.md#configure - - -## Connectivity check +## Connectivity check The `connectivity_check` object has the following properties: - * `probe_ipv4`: - The IPv4 address with port to which a connectivity check is performed. +- `probe_ipv4`: The IPv4 address with port to which a connectivity check is performed. **Example:** `8.8.8.8:53`. - * `probe_ipv6`: - The optional IPv6 address with port to which a connectivity check is - performed. This field is required in case of any IPv6 address in - [`bind_addresses`](#sg-s-*-bind_addresses). +- `probe_ipv6`: The optional IPv6 address with port to which a connectivity check is performed. This field is required in case of any IPv6 address in [`bind_addresses`](#sg-s-*-bind_addresses). **Example:** `[2001:4860:4860::8888]:53`. - - -## Network settings +## Network settings The `network` object has the following properties: - * `so_rcvbuf`: - The size of socket receive buffer (`SO_RCVBUF`), in a human-readable format. - Default is zero, which means use the default system settings. +- `so_rcvbuf`: The size of socket receive buffer (`SO_RCVBUF`), in a human-readable format. Default is zero, which means use the default system settings. See also [notes on these parameters](#recommended-buffers). **Example:** `1MB`. - * `so_sndbuf`: - The size of socket send buffer (`SO_SNDBUF`), in a human-readable format. - Default is zero, which means use the default system settings. +- `so_sndbuf`: The size of socket send buffer (`SO_SNDBUF`), in a human-readable format. Default is zero, which means use the default system settings. See also [notes on these parameters](#recommended-buffers). **Example:** `1MB`. - - ## Access settings The `access` object has the following properties: -* `blocked_question_domains`: - The list of domains or AdBlock rules to block requests. +- `blocked_question_domains`: The list of domains or AdBlock rules to block requests. **Examples:** `test.org`, `||example.org^$dnstype=AAAA`. -* `blocked_client_subnets`: - The list of IP addresses or CIDR-es to block. +- `blocked_client_subnets`: The list of IP addresses or CIDR-es to block. **Example:** `127.0.0.1`. +## Additional metrics information +The `additional_metrics_info` object is a map of strings with extra information which is exposed by `dns_app_additional_info` metric. -## Additional metrics information - -The `additional_metrics_info` object is a map of strings with extra information -which is exposed by `dns_app_additional_info` metric. - -Map keys must match regular expression `^[a-zA-Z_][a-zA-Z0-9_]*$`. See -[Prometheus documentation on valid labels][prom-label]. +Map keys must match regular expression `^[a-zA-Z_][a-zA-Z0-9_]*$`. See [Prometheus documentation on valid labels][prom-label]. **Property example:** @@ -1258,7 +909,7 @@ Map keys must match regular expression `^[a-zA-Z_][a-zA-Z0-9_]*$`. See 'info_key_2': 'info_value_2' ``` -The Prometheus metrics key is `additional_info`. For example: +The Prometheus metrics key is `additional_info`. For example: ```none # HELP dns_app_additional_info A metric with a constant '1' value labeled by additional info provided in configuration diff --git a/doc/debugdns.md b/doc/debugdns.md index 30405ed..0e69c26 100644 --- a/doc/debugdns.md +++ b/doc/debugdns.md @@ -1,4 +1,6 @@ - # AdGuard DNS Query Debugging API +# AdGuard DNS query debugging API + + You can debug AdGuard DNS queries by performing a query with the `CHAOS` class: @@ -41,8 +43,7 @@ In the `ANSWER SECTION`, the usual `IN` reply is returned. In the `ADDITIONAL SECTION`, the following debug information is returned: - * `client-ip`: - The IP address of the client. The full name is `client-ip.adguard-dns.com`. +- `client-ip`: The IP address of the client. The full name is `client-ip.adguard-dns.com`. **Example:** @@ -50,8 +51,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: client-ip.adguard-dns.com. 10 CH TXT "127.0.0.1" ``` - * `server-ip`: - The IP address of the server. The full name is `server-ip.adguard-dns.com`. +- `server-ip`: The IP address of the server. The full name is `server-ip.adguard-dns.com`. **Example:** @@ -59,9 +59,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: server-ip.adguard-dns.com. 10 CH TXT "127.0.0.1" ``` - * `device-id`: - The ID of the device as detected by the server, if any. The full name is - `device-id.adguard-dns.com`. +- `device-id`: The ID of the device as detected by the server, if any. The full name is `device-id.adguard-dns.com`. **Example:** @@ -69,9 +67,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: device-id.adguard-dns.com. 10 CH TXT "dev1234" ``` - * `profile-id`: - The ID of the profile (aka “DNS server” on the UI) of the AdGuard DNS - server. The full name is `profile-id.adguard-dns.com`. +- `profile-id`: The ID of the profile (aka “DNS server” on the UI) of the AdGuard DNS server. The full name is `profile-id.adguard-dns.com`. **Example:** @@ -79,8 +75,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: profile-id.adguard-dns.com. 10 CH TXT "prof1234" ``` - * `country`: - User's country code. The full name is `country.adguard-dns.com`. +- `country`: User's country code. The full name is `country.adguard-dns.com`. **Example:** @@ -88,9 +83,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: country.adguard-dns.com. 10 CH TXT "CY" ``` - * `asn`: - User's autonomous system number (ASN). The full name is - `asn.adguard-dns.com`. +- `asn`: User's autonomous system number (ASN). The full name is `asn.adguard-dns.com`. **Example:** @@ -98,9 +91,7 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: asn.adguard-dns.com. 10 CH TXT "1234" ``` - * `subdivision`: - User's location subdivision code. This field could be empty even if user's - country code is present. The full name is `subdivision.adguard-dns.com`. +- `subdivision`: User's location subdivision code. This field could be empty even if user's country code is present. The full name is `subdivision.adguard-dns.com`. **Example:** @@ -109,26 +100,17 @@ In the `ADDITIONAL SECTION`, the following debug information is returned: subdivision.adguard-dns.com. 10 CH TXT "CA" ``` -The following debug records can have one of two prefixes: `req` or `resp`. The -prefix depends on whether the filtering was applied to the request or the -response. +The following debug records can have one of two prefixes: `req` or `resp`. The prefix depends on whether the filtering was applied to the request or the response. - * `res-type`: - The `type` of response. The full name is - `(req|resp).res-type.adguard-dns.com`. Can be the following types: +- `res-type`: The `type` of response. The full name is `(req|resp).res-type.adguard-dns.com`. Can be the following types: - * `normal`: - The request or response was not filtered. + - `normal`: The request or response was not filtered. - * `blocked`: - The request or response was blocked by a filter list or parental - protection. + - `blocked`: The request or response was blocked by a filter list or parental protection. - * `allowed`: - The request or response was allowed by an exception rule. + - `allowed`: The request or response was allowed by an exception rule. - * `modified`: - The query has been rewritten by a rewrite rule or parental protection. + - `modified`: The query has been rewritten by a rewrite rule or parental protection. **Example:** @@ -136,10 +118,7 @@ response. req.res-type.adguard-dns.com. 10 CH TXT "blocked" ``` - * `rule`: - The rule that was applied to the query. The full name is - `(req|resp).rule.adguard-dns.com`. Rules that are longer than 255 bytes are - split into several consecutive strings. +- `rule`: The rule that was applied to the query. The full name is `(req|resp).rule.adguard-dns.com`. Rules that are longer than 255 bytes are split into several consecutive strings. **Example:** @@ -156,10 +135,7 @@ response. "heregoesthesecondpartoftherule" ``` - * `rule-list-id`: - The ID of the rule list that was applied, if any. The full name is - `(req|resp).rule-list-id.adguard-dns.com`. - +- `rule-list-id`: The ID of the rule list that was applied, if any. The full name is `(req|resp).rule-list-id.adguard-dns.com`. **Example:** @@ -167,7 +143,6 @@ response. req.rule-list-id.adguard-dns.com. 10 CH TXT "adguard_dns_filter" ``` -The TTL of these responses is taken from parameter -[`filters.response_ttl`][conf-filters-ttl] in the configuration file. +The TTL of these responses is taken from parameter [`filters.response_ttl`][conf-filters-ttl] in the configuration file. [conf-filters-ttl]: configuration.md#filters-response_ttl diff --git a/doc/debughttp.md b/doc/debughttp.md index 871f990..03bccd5 100644 --- a/doc/debughttp.md +++ b/doc/debughttp.md @@ -7,6 +7,7 @@ The AdGuard DNS debug HTTP API is served on [`LISTEN_PORT`][env-listen_port] and - [`GET /health-check`](#health-check) - [`GET /metrics`](#metrics) - [`GET /debug/pprof`](#pprof) +- [`POST /debug/api/cache/clear`](#api-cache-clear) - [`POST /debug/api/refresh`](#api-refresh) - [`POST /dnsdb/csv`](#dnsdb-csv) @@ -14,11 +15,11 @@ The AdGuard DNS debug HTTP API is served on [`LISTEN_PORT`][env-listen_port] and ## `GET /health-check` -A simple health check API. Always responds with a `200 OK` status and the plain-text body `OK`. +A simple health check API. Always responds with a `200 OK` status and the plain-text body `OK`. ## `GET /metrics` -Prometheus metrics HTTP API. See the [metrics page][metrics] for more details. +Prometheus metrics HTTP API. See the [metrics page][metrics] for more details. [metrics]: metrics.md @@ -28,14 +29,14 @@ The HTTP interface of Go's [PProf HTTP API][pprof api]. [pprof api]: https://pkg.go.dev/net/http/pprof -## `POST /debug/api/refresh` +## `POST /debug/api/cache/clear` -Run some refresh jobs manually. This refresh does not alter the time of the next automatic refresh. +Run some cache purges manually. The `ids` is an array of path patterns to match the cache IDs. Example request: ```sh -curl -d '{"ids":["*"]}' -v "http://${LISTEN_ADDR}:${LISTEN_PORT}/debug/api/refresh" +curl -d '{"ids":["filters/rulelist/*"]}' -v "http://${LISTEN_ADDR}:${LISTEN_PORT}/debug/api/cache/clear" ``` Request body example: @@ -43,35 +44,91 @@ Request body example: ```json { "ids": [ - "filter_storage", - "adult_blocking" + "filters/hashprefix/adult_blocking", + "filters/custom" ] } ``` Supported IDs: -- `adult_blocking`; -- `filter_storage`; -- `newly_registered_domains`; -- `safe_browsing`. +- `dns/ecscache_no_ecs` +- `dns/ecscache_with_ecs` +- `filters/blocked_service/*` +- `filters/custom` +- `filters/hashprefix/adult_blocking` +- `filters/hashprefix/newly_registered_domains` +- `filters/hashprefix/safe_browsing` +- `filters/rulelist/*` +- `filters/safe_search/general_safe_search` +- `geoip/host` +- `geoip/ip` -The special ID `*`, when used alone, causes all available refresh tasks to be performed. Use with caution. +Note that you can clear the cache of any individual blocked service, e.g. `filters/blocked_service/youtube`, and any filter rule list, e.g. `filters/rulelist/adguard_dns_filter`. + +The special ID `*`, when used alone, causes all available caches to be purged. Use with caution. Response body example: ```json { "results": { - "adult_blocking": "ok", - "filter_storage": "ok" + "filters/hashprefix/adult_blocking": "ok", + "filters/custom": "ok" + } +} +``` + +## `POST /debug/api/refresh` + +Run some refresh jobs manually. The `ids` is an array of path patterns to match the refreshers IDs. This refresh does not alter the time of the next automatic refresh. + +Example request: + +```sh +curl -d '{"ids":["filters/*"]}' -v "http://${LISTEN_ADDR}:${LISTEN_PORT}/debug/api/refresh" +``` + +Request body example: + +```json +{ + "ids": [ + "filters/hashprefix/adult_blocking", + "filters/storage" + ] +} +``` + +Supported IDs: + +- `allowlist` +- `billstat` +- `filters/hashprefix/adult_blocking` +- `filters/hashprefix/newly_registered_domains` +- `filters/hashprefix/safe_browsing` +- `filters/storage` +- `geoip` +- `profiledb` +- `rulestat` +- `ticket_rotator` + +The special ID `*`, when used alone, causes all available refresh tasks to be performed. Use with caution. + +Response body example: + +```json +{ + "results": { + "filters/hashprefix/adult_blocking": "ok", + "filters/storage": "ok" } } ``` ## `POST /dnsdb/csv` -The CSV dump of the current DNSDB statistics. Example of the output: +The CSV dump of the current DNSDB statistics. Example of the output: ```csv example.com,A,NOERROR,93.184.216.34,42 diff --git a/doc/development.md b/doc/development.md index bb25bc9..385c433 100644 --- a/doc/development.md +++ b/doc/development.md @@ -1,134 +1,74 @@ - # AdGuard DNS Development Setup +# AdGuard DNS Development Setup -## Contents +## Contents - * [Initial setup](#init) - * [Common Makefile macros and targets](#makefile) - * [How to run AdGuard DNS](#run) - * [Testing](#testing) +- [Initial setup](#init) +- [Common Makefile macros and targets](#makefile) +- [How to run AdGuard DNS](#run) +- [Testing](#testing) - - -## Initial setup +## Initial setup Development is supported on Linux and macOS (aka Darwin) systems. -1. Install Go 1.22 or later. +1. Install Go 1.23 or later. -1. Call `make init` to set up the Git pre-commit hook. +2. Call `make init` to set up the Git pre-commit hook. -1. Call `make go-tools` to install analyzers and other tools into the `bin` - directory. +3. Call `make go-tools` to install analyzers and other tools into the `bin` directory. +## Common Makefile macros and targets +Most development tasks are done through the use of our Makefile. Please keep the Makefile POSIX-compliant and portable. -## Common Makefile macros and targets +### Macros -Most development tasks are done through the use of our Makefile. Please keep -the Makefile POSIX-compliant and portable. +This is not an extensive list. See `../Makefile` and the scripts in the `../scripts/make/` directory. +- `OUT`: The name of the binary to build. Default: `./AdGuardDNS`. +- `RACE`: Set to `1` to enable the race detector. The race detector is always enabled for `make go-test`. - ### Macros +- `VERBOSE`: Set to `1` to enable verbose mode. Default: `0`. -This is not an extensive list. See `../Makefile` and the scripts in the -`../scripts/make/` directory. +### Targets -
-
OUT
-
- The name of the binary to build. Default: ./AdGuardDNS. -
-
RACE
-
- Set to 1 to enable the race detector. The race detector is - always enabled for make go-test. -
-
VERBOSE
-
- Set to 1 to enable verbose mode. Default: 0. -
-
+This is not an extensive list. See `../Makefile`. +- `make init`: Set up the pre-commit hook that runs checks, linters, and tests. +- `make go-build`: Build the binary. See also the `OUT` and `RACE` macros. - ### Targets +- `make go-gen`: Regenerate the automatically generated Go files that need to be periodically updated. Those generated files are: -This is not an extensive list. See `../Makefile`. + - `../internal/backendpb/dns.pb.go` + - `../internal/backendpb/dns_grpc.pb.go` + - `../internal/ecscache/ecsblockilist_generate.go` + - `../internal/geoip/asntops_generate.go` + - `../internal/geoip/country_generate.go` + - `../internal/profiledb/internal/filecachepb/filecache.pb.go` -
-
make init
-
- Set up the pre-commit hook that runs checks, linters, and tests. -
-
make go-build
-
- Build the binary. See also the OUT and RACE - macros. -
-
make go-gen
-
-

- Regenerate the automatically generated Go files that need to be - periodically updated. Those generated files are: -

-
    -
  • - ../internal/geoip/country_generate.go; -
    • -
  • - ../internal/geoip/asntops_generate.go; -
    • -
  • - ../internal/ecscache/ecsblockilist_generate.go; -
    • -
  • - ../internal/profiledb/internal/filecachepb/filecache.pb.go. -
    • -
-

- You'll need to - install protoc - for the last one. -

-
-
make go-lint
-
- Run Go checkers and static analysis. -
-
make go-test
-
- Run Go tests. -
-
make go-bench
-
- Run Go benchmarks. -
-
make go-tools
-
- Install the Go static analysis tools locally. -
-
make test
-
- Currently does the same thing as make go-test but is - defined both because it's a common target and also in case code in - another language appears in the future. -
-
make txt-lint
-
- Run plain text checkers. -
-
+ You'll need to [install `protoc`][protoc] for the last one. +- `make go-lint`: Run Go checkers and static analysis. +- `make go-test`: Run Go tests. -## How to run AdGuard DNS +- `make go-bench`: Run Go benchmarks. + +- `make go-tools`: Install the Go static analysis tools locally. + +- `make test`: Currently does the same thing as `make go-test` but is defined both because it's a common target and also in case code in another language appears in the future. + +- `make txt-lint`: Run plain text checkers. + +[protoc]: https://protobuf.dev/getting-started/gotutorial/#compiling-protocol-buffers + +## How to run AdGuard DNS This is an example on how to run AdGuard DNS locally. - - - ### Step 1: prepare the TLS certificate and the key +### Step 1: prepare the TLS certificate and the key Keeping the test files in the `test` directory since it's added to `.gitignore`: @@ -150,20 +90,17 @@ openssl rand 32 > ./tls_key_1 openssl rand 32 > ./tls_key_2 ``` - - - ### Step 2: prepare the DNSCrypt configuration +### Step 2: prepare the DNSCrypt configuration Install the [`dnscrypt`][dnsc] tool: - * On macOS, install from Brew: +- On macOS, install from Brew: ```sh brew install ameshkov/tap/dnscrypt ``` - * On other unixes, such as Linux, [download][dnscdl] and install the latest - release manually. +- On other unixes, such as Linux, [download][dnscdl] and install the latest release manually. Then, generate the configuration: @@ -171,18 +108,14 @@ Then, generate the configuration: dnscrypt generate -p testdns -o ./dnscrypt.yml ``` - - - ### Step 3: prepare the configuration file +### Step 3: prepare the configuration file ```sh cd ../ cp -f config.dist.yaml config.yaml ``` - - - ### Step 4: prepare the test data +### Step 4: prepare the test data ```sh echo 'General content ahead' > ./test/block_page_general.html @@ -192,17 +125,13 @@ echo 'Error 404' > ./test/error_404.html echo 'Error 500' > ./test/error_500.html ``` - - - ### Step 5: compile AdGuard DNS +### Step 5: compile AdGuard DNS ```sh make build ``` - - - ### Step 6: prepare cache data and GeoIP +### Step 6: prepare cache data and GeoIP We'll use the test versions of the GeoIP databases here. @@ -214,45 +143,39 @@ curl 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2 curl 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-ISP-Test.mmdb' -o ./test/GeoIP2-ISP-Test.mmdb ``` - - - ### Step 7: run AdGuard DNS +### Step 7: run AdGuard DNS You'll need to supply the following: - * [`ADULT_BLOCKING_URL`][env-ADULT_BLOCKING_URL] - * [`BILLSTAT_URL`][env-BILLSTAT_URL] - * [`CONSUL_ALLOWLIST_URL`][env-CONSUL_ALLOWLIST_URL] - * [`GENERAL_SAFE_SEARCH_URL`][env-GENERAL_SAFE_SEARCH_URL] - * [`LINKED_IP_TARGET_URL`][env-LINKED_IP_TARGET_URL] - * [`NEW_REG_DOMAINS_URL`][env-NEW_REG_DOMAINS_URL] - * [`PROFILES_URL`][env-PROFILES_URL] - * [`SAFE_BROWSING_URL`][env-SAFE_BROWSING_URL] - * [`YOUTUBE_SAFE_SEARCH_URL`][env-YOUTUBE_SAFE_SEARCH_URL] +- [`ADULT_BLOCKING_URL`][env-ADULT_BLOCKING_URL] +- [`BILLSTAT_URL`][env-BILLSTAT_URL] +- [`CONSUL_ALLOWLIST_URL`][env-CONSUL_ALLOWLIST_URL] +- [`GENERAL_SAFE_SEARCH_URL`][env-GENERAL_SAFE_SEARCH_URL] +- [`LINKED_IP_TARGET_URL`][env-LINKED_IP_TARGET_URL] +- [`NEW_REG_DOMAINS_URL`][env-NEW_REG_DOMAINS_URL] +- [`PROFILES_URL`][env-PROFILES_URL] +- [`SAFE_BROWSING_URL`][env-SAFE_BROWSING_URL] +- [`YOUTUBE_SAFE_SEARCH_URL`][env-YOUTUBE_SAFE_SEARCH_URL] See the [external HTTP API documentation][externalhttp]. -You may use `go run ./scripts/backend` to start mock GRPC server for -`BILLSTAT_URL` and `PROFILES_URL` endpoints. +You may use `go run ./scripts/backend` to start mock GRPC server for `BILLSTAT_URL` and `PROFILES_URL` endpoints. -You may need to change the listen ports in `config.yaml` which are less than -1024 to some other ports. Otherwise, `sudo` or `doas` is required to run -`AdGuardDNS`. +You may need to change the listen ports in `config.yaml` which are less than 1024 to some other ports. Otherwise, `sudo` or `doas` is required to run `AdGuardDNS`. Examples below are for the configuration with the following changes: - * Plain DNS: `53` → `5354` - * DoT: `853` → `8853` - * DoH: `443` → `8443` - * DoQ: `853` → `8853` +- Plain DNS: `53` → `5354` +- DoT: `853` → `8853` +- DoH: `443` → `8443` +- DoQ: `853` → `8853` You may also need to remove `probe_ipv6` if your network does not support IPv6. If you're using an OS different from Linux, you also need to make these changes: - * Remove the `interface_listeners` section. - * Remove `bind_interfaces` from the `default_dns` server configuration and - replace it with `bind_addresses`. +- Remove the `interface_listeners` section. +- Remove `bind_interfaces` from the `default_dns` server configuration and replace it with `bind_addresses`. ```sh env \ @@ -293,9 +216,7 @@ env \ [externalhttp]: externalhttp.md - - - ### Step 8: test your instance +### Step 8: test your instance Plain DNS: @@ -323,11 +244,10 @@ VERIFY=0 dnslookup example.org quic://127.0.0.1:8853 Open `http://127.0.0.1:8081/metrics` to see the server's metrics. -DNSCrypt is a bit trickier. You need to open `dnscrypt.yml` and use values from -there to generate an SDNS stamp on . +DNSCrypt is a bit trickier. You need to open `dnscrypt.yml` and use values from there to generate an SDNS stamp on . - > [!NOTE] - > The example below is for a test configuration that won't work for you. +> [!NOTE] +> The example below is for a test configuration that won't work for you. ```sh dnslookup example.org sdns://AQcAAAAAAAAADjEyNy4wLjAuMTo1NDQzIAbKgP3dmXybr1DaKIFgKjsc8zSFX4rgT_hFgymSq6w1FzIuZG5zY3J5cHQtY2VydC50ZXN0ZG5z @@ -336,53 +256,35 @@ dnslookup example.org sdns://AQcAAAAAAAAADjEyNy4wLjAuMTo1NDQzIAbKgP3dmXybr1DaKIF [dnsc]: https://github.com/ameshkov/dnscrypt [dnscdl]: https://github.com/ameshkov/dnscrypt/releases +## Testing +The `go-bench` and `go-test` targets [described earlier](#makefile-targets) should generally be enough, but there are cases where additional testing setup is required. One such case is package `bindtodevice`. -## Testing +### Testing `SO_BINDTODEVICE` features -The `go-bench` and `go-test` targets [described earlier](#makefile-targets) -should generally be enough, but there are cases where additional testing setup -is required. One such case is package `bindtodevice`. - - - - ### Testing `SO_BINDTODEVICE` features - -The `SO_BINDTODEVICE` features require a Linux machine with a particular IP -routing set up. In order to test these features on architectures other than -Linux, this repository has a Dockerfile and a convenient script to use it, see -`scripts/test/bindtodevice.sh`. +The `SO_BINDTODEVICE` features require a Linux machine with a particular IP routing set up. In order to test these features on architectures other than Linux, this repository has a Dockerfile and a convenient script to use it, see `scripts/test/bindtodevice.sh`. A simple example: - * If your Docker is installed in a way that doesn't require `sudo` to use it: +- If your Docker is installed in a way that doesn't require `sudo` to use it: ```sh sh ./scripts/test/bindtodevice.sh ``` - * Otherwise: +- Otherwise: ```sh env USE_SUDO=1 sh ./scripts/test/bindtodevice.sh ``` -This will build the image and open a shell within the container. The container -environment is defined by `scripts/test/bindtodevice.docker`, and has all -utilities required to build the `AdGuardDNS` binary and test it. The working -directory is also shared with the container through the `/test` directory inside -it. The container also routes all IP connections to any address in the -`172.17.0.0/16` subnet to the `eth0` network interface. So, calling `make -go-test` or a similar command from within the container will actually test the -`SO_BINDTODEVICE` features: +This will build the image and open a shell within the container. The container environment is defined by `scripts/test/bindtodevice.docker`, and has all utilities required to build the `AdGuardDNS` binary and test it. The working directory is also shared with the container through the `/test` directory inside it. The container also routes all IP connections to any address in the `172.17.0.0/16` subnet to the `eth0` network interface. So, calling `make go-test` or a similar command from within the container will actually test the `SO_BINDTODEVICE` features: ```sh go test --cover -v ./internal/bindtodevice/ ``` -If you want to open an additional terminal (for example to launch `AdGuardDNS` -in one and `dig` it in the other), use `docker exec` like this (you may need -`sudo` for that): +If you want to open an additional terminal (for example to launch `AdGuardDNS` in one and `dig` it in the other), use `docker exec` like this (you may need `sudo` for that): ```sh docker exec -i -t agdns_bindtodevice_test /bin/sh diff --git a/doc/environment.md b/doc/environment.md index db52b91..cda8b27 100644 --- a/doc/environment.md +++ b/doc/environment.md @@ -4,9 +4,11 @@ AdGuard DNS uses [environment variables][wiki-env] to store some of the more sen ## Contents +- [`ADULT_BLOCKING_ENABLED`](#ADULT_BLOCKING_ENABLED) - [`ADULT_BLOCKING_URL`](#ADULT_BLOCKING_URL) - [`BILLSTAT_API_KEY`](#BILLSTAT_API_KEY) - [`BILLSTAT_URL`](#BILLSTAT_URL) +- [`BLOCKED_SERVICE_ENABLED`](#BLOCKED_SERVICE_ENABLED) - [`BLOCKED_SERVICE_INDEX_URL`](#BLOCKED_SERVICE_INDEX_URL) - [`CONFIG_PATH`](#CONFIG_PATH) - [`CONSUL_ALLOWLIST_URL`](#CONSUL_ALLOWLIST_URL) @@ -14,33 +16,51 @@ AdGuard DNS uses [environment variables][wiki-env] to store some of the more sen - [`CONSUL_DNSCHECK_SESSION_URL`](#CONSUL_DNSCHECK_SESSION_URL) - [`FILTER_CACHE_PATH`](#FILTER_CACHE_PATH) - [`FILTER_INDEX_URL`](#FILTER_INDEX_URL) +- [`GENERAL_SAFE_ENABLED`](#GENERAL_SAFE_SEARCH_ENABLED) - [`GENERAL_SAFE_SEARCH_URL`](#GENERAL_SAFE_SEARCH_URL) - [`GEOIP_ASN_PATH` and `GEOIP_COUNTRY_PATH`](#GEOIP_ASN_PATH) - [`LINKED_IP_TARGET_URL`](#LINKED_IP_TARGET_URL) - [`LISTEN_ADDR`](#LISTEN_ADDR) - [`LISTEN_PORT`](#LISTEN_PORT) - [`LOG_TIMESTAMP`](#LOG_TIMESTAMP) +- [`METRICS_NAMESPACE`](#METRICS_NAMESPACE) +- [`NEW_REG_DOMAINS_ENABLED`](#NEW_REG_DOMAINS_ENABLED) - [`NEW_REG_DOMAINS_URL`](#NEW_REG_DOMAINS_URL) - [`PROFILES_API_KEY`](#PROFILES_API_KEY) - [`PROFILES_CACHE_PATH`](#PROFILES_CACHE_PATH) -- [`PROFILES_ENABLED`](#PROFILES_ENABLED) - [`PROFILES_URL`](#PROFILES_URL) +- [`REDIS_ADDR`](#REDIS_ADDR) +- [`REDIS_KEY_PREFIX`](#REDIS_KEY_PREFIX) +- [`REDIS_MAX_ACTIVE`](#REDIS_MAX_ACTIVE) +- [`REDIS_MAX_IDLE`](#REDIS_MAX_IDLE) +- [`REDIS_IDLE_TIMEOUT`](#REDIS_IDLE_TIMEOUT) +- [`REDIS_PORT`](#REDIS_PORT) - [`QUERYLOG_PATH`](#QUERYLOG_PATH) - [`RULESTAT_URL`](#RULESTAT_URL) +- [`SAFE_BROWSING_ENABLED`](#SAFE_BROWSING_ENABLED) - [`SAFE_BROWSING_URL`](#SAFE_BROWSING_URL) - [`SENTRY_DSN`](#SENTRY_DSN) - [`SSL_KEY_LOG_FILE`](#SSL_KEY_LOG_FILE) - [`VERBOSE`](#VERBOSE) +- [`WEB_STATIC_DIR_ENABLED`](#WEB_STATIC_DIR_ENABLED) +- [`WEB_STATIC_DIR`](#WEB_STATIC_DIR) +- [`YOUTUBE_SAFE_SEARCH_ENABLED`](#YOUTUBE_SAFE_SEARCH_ENABLED) - [`YOUTUBE_SAFE_SEARCH_URL`](#YOUTUBE_SAFE_SEARCH_URL) [conf]: configuration.md [wiki-env]: https://en.wikipedia.org/wiki/Environment_variable +## `ADULT_BLOCKING_ENABLED` + +When set to `1`, enable the adult-blocking hash-prefix filter. When set to `0`, disable it. + +**Default:** `1`. + ## `ADULT_BLOCKING_URL` -The URL of source list of rules for adult blocking filter. +The HTTP(S) URL of source list of rules for adult blocking filter. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `ADULT_BLOCKING_ENABLED` is set to `1`. ## `BILLSTAT_API_KEY` @@ -52,17 +72,24 @@ The API key to use when authenticating queries to the billing statistics API, if ## `BILLSTAT_URL` -The base backend URL for backend billing statistics uploader API. Supports GRPC (`grpc://` and`grpcs://`) URLs. See the [external HTTP API requirements section][ext-billstat]. +The base backend URL for backend billing statistics uploader API. Supports gRPC(S) (`grpc://` and`grpcs://`) URLs. See the [external HTTP API requirements section][ext-billstat]. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if there is at least one [server group][conf-sg] with profiles enabled. +[conf-sg]: configuration.md#server_groups [ext-billstat]: externalhttp.md#backend-billstat +## `BLOCKED_SERVICE_ENABLED` + +When set to `1`, enable the blocked service filter. When set to `0`, disable it. + +**Default:** `1`. + ## `BLOCKED_SERVICE_INDEX_URL` -The URL of the blocked service index file server. See the [external HTTP API requirements section][ext-blocked] on the expected format of the response. +The HTTP(S) URL of the blocked service index file server. See the [external HTTP API requirements section][ext-blocked] on the expected format of the response. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `BLOCKED_SERVICE_ENABLED` is set to `1`. [ext-blocked]: externalhttp.md#filters-blocked-services @@ -74,7 +101,7 @@ The path to the configuration file. ## `CONSUL_ALLOWLIST_URL` -The URL of the Consul instance serving the dynamic part of the rate-limit allowlist. See the [external HTTP API requirements section][ext-consul] on the expected format of the response. +The HTTP(S) URL of the Consul instance serving the dynamic part of the rate-limit allowlist. See the [external HTTP API requirements section][ext-consul] on the expected format of the response. **Default:** No default value, the variable is **required.** @@ -82,7 +109,7 @@ The URL of the Consul instance serving the dynamic part of the rate-limit allowl ## `CONSUL_DNSCHECK_KV_URL` -The URL of the KV API of the Consul instance used as a key-value database for the DNS server checking. It must end with `/kv/` where `` is any non-empty namespace. If not specified, the [`CONSUL_DNSCHECK_SESSION_URL`](#CONSUL_DNSCHECK_SESSION_URL) is also omitted. +The HTTP(S) URL of the KV API of the Consul instance used as a key-value database for the DNS server checking. It must end with `/kv/` where `` is any non-empty namespace. If not specified, the [`CONSUL_DNSCHECK_SESSION_URL`](#CONSUL_DNSCHECK_SESSION_URL) is also omitted. **Default:** **Unset.** @@ -90,7 +117,7 @@ The URL of the KV API of the Consul instance used as a key-value database for th ## `CONSUL_DNSCHECK_SESSION_URL` -The URL of the session API of the Consul instance used as a key-value database for the DNS server checking. If not specified, the [`CONSUL_DNSCHECK_KV_URL`](#CONSUL_DNSCHECK_KV_URL) is also omitted. +The HTTP(S) URL of the session API of the Consul instance used as a key-value database for the DNS server checking. If not specified, the [`CONSUL_DNSCHECK_KV_URL`](#CONSUL_DNSCHECK_KV_URL) is also omitted. **Default:** **Unset.** @@ -104,17 +131,23 @@ The path to the directory used to store the cached version of all filters and fi ## `FILTER_INDEX_URL` -The URL of the filtering rule index file server. See the [external HTTP API requirements section][ext-lists] on the expected format of the response. +The HTTP(S) URL or a hostless file URI (e.g. `file:///tmp/filters.json`) of the filtering rule index file server. See the [external HTTP API requirements section][ext-lists] on the expected format of the response. **Default:** No default value, the variable is **required.** [ext-lists]: externalhttp.md#filters-lists +## `GENERAL_SAFE_SEARCH_ENABLED` + +When set to `1`, enable the general safe search filter. When set to `0`, disable it. + +**Default:** `1`. + ## `GENERAL_SAFE_SEARCH_URL` -The URL of the list of general safe search rewriting rules. See the [external HTTP API requirements section][ext-general] on the expected format of the response. +The HTTP(S) URL of the list of general safe search rewriting rules. See the [external HTTP API requirements section][ext-general] on the expected format of the response. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `GENERAL_SAFE_SEARCH_ENABLED` is set to `1`. [ext-general]: externalhttp.md#filters-safe-search @@ -126,7 +159,7 @@ Paths to the files containing MaxMind GeoIP databases: for ASNs and for countrie ## `LINKED_IP_TARGET_URL` -The target URL to which linked IP API requests are proxied. In case [linked IP and dynamic DNS][conf-web-linked_ip] web server is configured, the variable is required. See the [external HTTP API requirements section][ext-linked_ip]. +The target HTTP(S) URL to which linked IP API requests are proxied. In case [linked IP and dynamic DNS][conf-web-linked_ip] web server is configured, the variable is required. See the [external HTTP API requirements section][ext-linked_ip]. **Default:** **Unset.** @@ -153,11 +186,23 @@ If `1`, show timestamps in the plain text logs. If `0`, don't show the timestamp **Default:** `1`. +## `METRICS_NAMESPACE` + +The namespace to be used for Prometheus metrics. It must be a valid Prometheus metric label. + +**Default:** `dns`. + +## `NEW_REG_DOMAINS_ENABLED` + +When set to `1`, enable the newly-registered domains hash-prefix filter. When set to `0`, disable it. + +**Default:** `1`. + ## `NEW_REG_DOMAINS_URL` -The URL of source list of rules for newly registered domains safe browsing filter. +The HTTP(S) URL of source list of rules for newly registered domains safe browsing filter. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `NEW_REG_DOMAINS_ENABLED` is set to `1`. ## `PROFILES_API_KEY` @@ -189,20 +234,58 @@ The profile cache is read on start and is later updated on every [full refresh][ [conf-backend-full_refresh_interval]: configuration.md#backend-full_refresh_interval -## `PROFILES_ENABLED` +## `PROFILES_MAX_RESP_SIZE` -If `0`, disables user profiles and devices recognition and billing. +The maximum size of the response from the profiles API in a human-readable format. -**Default:** `1`. +**Default:** `8MB`. ## `PROFILES_URL` -The base backend URL for profiles API. Supports GRPC (`grpc://` and`grpcs://`) URLs. See the [external API requirements section][ext-profiles]. +The base backend URL for profiles API. Supports gRPC(S) (`grpc://` and`grpcs://`) URLs. See the [external API requirements section][ext-profiles]. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if there is at least one [server group][conf-sg] with profiles enabled. [ext-profiles]: externalhttp.md#backend-profiles +## `REDIS_ADDR` + +Redis server address. Can be an IP address or a hostname. + +**Default:** No default value, the variable if required if the [type][conf-check-kv-type] of remote KV storage for DNS server checking is `redis` in the configuration file. + +[conf-check-kv-type]: configuration.md#check-kv-type + +## `REDIS_KEY_PREFIX` + +The prefix for Redis keys. + +**Default:** `agdns`. + +## `REDIS_MAX_ACTIVE` + +The maximum number of active Redis connections. + +**Default:** `10`. + +## `REDIS_MAX_IDLE` + +The maximum number of idle Redis connections. + +**Default:** `3`. + +## `REDIS_IDLE_TIMEOUT` + +How long until idle Redis connections are closed, as a human-readable duration. + +**Default:** `30s`. + +## `REDIS_PORT` + +Redis server port. + +**Default:** `6379`. + ## `QUERYLOG_PATH` The path to the file into which the query log is going to be written. @@ -211,7 +294,7 @@ The path to the file into which the query log is going to be written. ## `RULESTAT_URL` -The URL to send filtering rule list statistics to. If empty or unset, the collection of filtering rule statistics is disabled. See the [external HTTP API requirements section][ext-rulestat] on the expected format of the response. +The HTTP(S) URL to send filtering rule list statistics to. If empty or unset, the collection of filtering rule statistics is disabled. See the [external HTTP API requirements section][ext-rulestat] on the expected format of the response. **Default:** **Unset.** @@ -219,11 +302,17 @@ The URL to send filtering rule list statistics to. If empty or unset, the collec [ext-rulestat]: externalhttp.md#rulestat +## `SAFE_BROWSING_ENABLED` + +When set to `1`, enable the safe-browsing hash-prefix filter. When set to `0`, disable it. + +**Default:** `1`. + ## `SAFE_BROWSING_URL` -The URL of source list of rules for dangerous domains safe browsing filter. +The HTTP(S) URL of source list of rules for dangerous domains safe browsing filter. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `SAFE_BROWSING_ENABLED` is set to `1`. ## `SENTRY_DSN` @@ -239,12 +328,36 @@ If set, TLS key logs are written to this file to allow other programs (i.e. Wire ## `VERBOSE` -When set to `1`, enable verbose logging. When set to `0`, disable it. +- `2`: Enables trace logging. + +- `1`: Enables debug logging. + +- `0`: The default level of verbosity: only info logs are printed. **Default:** `0`. +## `WEB_STATIC_DIR_ENABLED` + +When set to `1`, use `WEB_STATIC_DIR` as the source of the static content. + +**Default:** `0`. + +## `WEB_STATIC_DIR` + +The absolute path to the directory used to serve static content. The directory must exist. + +The value of the `Content-Type` header is guessed from the files' contents. Other headers cannot be modified. If the content type of a file cannot be guessed, `text/plain` is used. + +**Default:** No default value, the variable is required if `WEB_STATIC_DIR_ENABLED` is set to `1`. + +## `YOUTUBE_SAFE_SEARCH_ENABLED` + +When set to `1`, enable the youtube safe search filter. When set to `0`, disable it. + +**Default:** `1`. + ## `YOUTUBE_SAFE_SEARCH_URL` -The URL of the list of YouTube-specific safe search rewriting rules. See the [external HTTP API requirements section][ext-general] on the expected format of the response. +The HTTP(S) URL of the list of YouTube-specific safe search rewriting rules. See the [external HTTP API requirements section][ext-general] on the expected format of the response. -**Default:** No default value, the variable is **required.** +**Default:** No default value, the variable is required if `YOUTUBE_SAFE_SEARCH_ENABLED` is set to `1`. diff --git a/doc/externalhttp.md b/doc/externalhttp.md index 6a16428..d50f09e 100644 --- a/doc/externalhttp.md +++ b/doc/externalhttp.md @@ -1,65 +1,46 @@ - # AdGuard DNS External HTTP API Requirements +# AdGuard DNS external HTTP API requirements -AdGuard DNS uses information from external HTTP APIs for filtering and other -pieces of its functionality. Whenever it makes requests to these services, -AdGuard DNS sets the `User-Agent` header. All services described in this -document should set the `Server` header in their replies. +AdGuard DNS uses information from external HTTP APIs for filtering and other pieces of its functionality. Whenever it makes requests to these services, AdGuard DNS sets the `User-Agent` header. All services described in this document should set the `Server` header in their replies. +## Contents +- [Backend billing statistics](#backend-billstat) +- [Backend profiles service](#backend-profiles) +- [Consul key-value storage](#consul) +- [Filtering](#filters) + - [Blocked services](#filters-blocked-services) + - [Filtering rule lists](#filters-lists) + - [Safe search](#filters-safe-search) +- [Proxied linked IP and dynamic DNS (DDNS) Endpoints](#backend-linkip) +- [Rule statistics service](#rulestat) -## Contents +## Backend billing statistics - * [Backend Billing Statistics](#backend-billstat) - * [Backend Profiles Service](#backend-profiles) - * [Consul Key-Value Storage](#consul) - * [Filtering](#filters) - * [Blocked Services](#filters-blocked-services) - * [Filtering Rule Lists](#filters-lists) - * [Safe Search](#filters-safe-search) - * [Proxied Linked IP and Dynamic DNS (DDNS) Endpoints](#backend-linkip) - * [Rule Statistics Service](#rulestat) +This is the service to which the [`BILLSTAT_URL`][env-billstat_url] environment variable points. Supports gRPC(s) URLs. The service must correspond to `./internal/backendpb/dns.proto`. - - -## Backend Billing Statistics - -This is the service to which the [`BILLSTAT_URL`][env-billstat_url] environment -variable points. Supports `grpc(s)` URLs. The service must correspond to -`./internal/backendpb/backend.proto`. This service can be disabled with the -[`PROFILES_ENABLED`][env-profiles_enabled] environment variable. +This service is disabled when all server groups have property [`profiles_enabled`][conf-srvgrp-prof] set to `false`. [env-billstat_url]: environment.md#BILLSTAT_URL -[env-profiles_enabled]: environment.md#PROFILES_ENABLED +[conf-srvgrp-prof]: configuration.md#sg-*-profiles_enabled +## Backend profiles service +This is the service to which the [`PROFILES_URL`][env-profiles_url] environment variable points. Supports gRPC(s) URLs. The service must correspond to `./internal/backendpb/dns.proto`. -## Backend Profiles Service - -This is the service to which the [`PROFILES_URL`][env-profiles_url] environment -variable points. Supports `grpc(s)` URLs. The service must correspond to -`./internal/backendpb/backend.proto`. This service can be disabled with the -[`PROFILES_ENABLED`][env-profiles_enabled] environment variable. +This service is disabled when all server groups have property [`profiles_enabled`][conf-srvgrp-prof] set to `false`. [env-profiles_url]: environment.md#PROFILES_URL +## Consul key-value storage +A [Consul][consul-io] service can be used for the DNS server check and dynamic rate-limit allowlist features. Currently used endpoints can be seen in the documentation of the [`CONSUL_ALLOWLIST_URL`][env-consul-allowlist], [`CONSUL_DNSCHECK_KV_URL`][env-consul-dnscheck-kv], and [`CONSUL_DNSCHECK_SESSION_URL`][env-consul-dnscheck-session] environment variables. -## Consul Key-Value Storage - -A [Consul][consul-io] service can be used for the DNS server check and dynamic -rate-limit allowlist features. Currently used endpoints can be seen in the -documentation of the [`CONSUL_ALLOWLIST_URL`][env-consul-allowlist], -[`CONSUL_DNSCHECK_KV_URL`][env-consul-dnscheck-kv], and -[`CONSUL_DNSCHECK_SESSION_URL`][env-consul-dnscheck-session] environment -variables. - -The `CONSUL_ALLOWLIST_URL` endpoint must respond with a `200 OK` response code -and a JSON document in the following format: +The `CONSUL_ALLOWLIST_URL` endpoint must respond with a `200 OK` response code and a JSON document in the following format: ```json [ @@ -76,15 +57,11 @@ and a JSON document in the following format: [env-consul-dnscheck-kv]: environment.md#CONSUL_DNSCHECK_KV_URL [env-consul-dnscheck-session]: environment.md#CONSUL_DNSCHECK_SESSION_URL +## Filtering +### Blocked services -## Filtering - - ### Blocked Services - -This endpoint, defined by [`BLOCKED_SERVICE_INDEX_URL`][env-services], must -respond with a `200 OK` response code and a JSON document in the following -format: +This endpoint, defined by [`BLOCKED_SERVICE_INDEX_URL`][env-services], must respond with a `200 OK` response code and a JSON document in the following format: ```json { @@ -100,15 +77,11 @@ format: } ``` -All properties must be filled with valid IDs and rules. Additional fields in -objects are ignored. +All properties must be filled with valid IDs and rules. Additional fields in objects are ignored. +### Filtering rule lists - - ### Filtering Rule Lists - -This endpoint, defined by [`FILTER_INDEX_URL`][env-filters], must respond with a -`200 OK` response code and a JSON document in the following format: +This endpoint, defined by [`FILTER_INDEX_URL`][env-filters], must respond with a `200 OK` response code and a JSON document in the following format: ```json { @@ -121,17 +94,11 @@ This endpoint, defined by [`FILTER_INDEX_URL`][env-filters], must respond with a } ``` -All properties must be filled with valid IDs and URLs. Additional fields in -objects are ignored. +All properties must be filled with valid IDs and URLs. Additional fields in objects are ignored. +### Safe search - - ### Safe Search - -These endpoints, defined by [`GENERAL_SAFE_SEARCH_URL`][env-general] and -[`YOUTUBE_SAFE_SEARCH_URL`][env-youtube], must respond with a `200 OK` response -code and filtering rule lists with [`$dnsrewrite`][rules-dnsrewrite] rules for -`A`, `AAAA`, or `CNAME` types. For example, for YouTube: +These endpoints, defined by [`GENERAL_SAFE_SEARCH_URL`][env-general] and [`YOUTUBE_SAFE_SEARCH_URL`][env-youtube], must respond with a `200 OK` response code and filtering rule lists with [`$dnsrewrite`][rules-dnsrewrite] rules for `A`, `AAAA`, or `CNAME` types. For example, for YouTube: ```none |m.youtube.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com @@ -149,31 +116,25 @@ code and filtering rule lists with [`$dnsrewrite`][rules-dnsrewrite] rules for + [rules-dnsrewrite]: https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#dnsrewrite +## Proxied linked iP and dynamic DNS (DDNS) endpoints +The service defined by the [`LINKED_IP_TARGET_URL`][env-linked_ip_target_url] environment variable should define the following endpoints: -## Proxied Linked IP and Dynamic DNS (DDNS) Endpoints +- `GET /linkip/{device_id}/{encrypted}/status` +- `GET /linkip/{device_id}/{encrypted}` +- `POST /ddns/{device_id}/{encrypted}/{domain}` +- `POST /linkip/{device_id}/{encrypted}` -The service defined by the [`LINKED_IP_TARGET_URL`][env-linked_ip_target_url] -environment variable should define the following endpoints: - - * `GET /linkip/{device_id}/{encrypted}/status`; - * `GET /linkip/{device_id}/{encrypted}`; - * `POST /ddns/{device_id}/{encrypted}/{domain}`; - * `POST /linkip/{device_id}/{encrypted}`. - -The AdGuard DNS proxy will add the `CF-Connecting-IP` header with the IP address -of the original client as well as set the `User-Agent` header to its own value. +The AdGuard DNS proxy will add the `X-Connecting-Ip` header with the IP address of the original client as well as set the `User-Agent` header to its own value. [env-linked_ip_target_url]: environment.md#LINKED_IP_TARGET_URL +## Rule statistics service - -## Rule Statistics Service - -This endpoint, defined by [`RULESTAT_URL`][env-rulestat], must respond with a -`200 OK` response code and accept a JSON document in the following format: +This endpoint, defined by [`RULESTAT_URL`][env-rulestat], must respond with a `200 OK` response code and accept a JSON document in the following format: ```json { diff --git a/doc/http.md b/doc/http.md index a8a34e6..2fbc062 100644 --- a/doc/http.md +++ b/doc/http.md @@ -1,30 +1,23 @@ - # AdGuard DNS HTTP API +# AdGuard DNS HTTP API -The main HTTP API is served on the same port as the DNS-over-HTTP servers as -well as on other addresses, if the [web configuration][conf-web] is set -appropriately. +The main HTTP API is served on the same port as the DNS-over-HTTP servers as well as on other addresses, if the [web configuration][conf-web] is set appropriately. -## Contents +## Contents - * [Block Pages](#block-pages) - * [DNS Server Check](#dnscheck-test) - * [Linked IP Proxy](#linked-ip-proxy) - * [Static Content](#static-content) +- [Block Pages](#block-pages) +- [DNS Server Check](#dnscheck-test) +- [Linked IP Proxy](#linked-ip-proxy) +- [Static Content](#static-content) [conf-web]: configuration.md#web +## Block Pages +The safe-browsing, adult-blocking, and popup-blocking servers. Every request is responded with the content from the configured file, with the exception of `GET /favicon.ico` and `GET /robots.txt` requests, which are handled separately: -## Block Pages +- `GET /favicon.ico` requests are responded with a plain-text `404 Not Found` response. -The safe-browsing, adult-blocking, and popup-blocking servers. Every request is -responded with the content from the configured file, with the exception of `GET -/favicon.ico` and `GET /robots.txt` requests, which are handled separately: - - * `GET /favicon.ico` requests are responded with a plain-text `404 Not Found` - response. - - * `GET /robots.txt` requests are responded with: +- `GET /robots.txt` requests are responded with: ```none User-agent: * @@ -33,14 +26,9 @@ responded with the content from the configured file, with the exception of `GET The [static content](#static-content) is not served on these servers. +## DNS Server Check - -## DNS Server Check - -`GET /dnscheck/test` is the DNS server check HTTP API. It should be requested -with a random ID prepended to one of the [check domains][conf-check-domains] -with a hyphen. The random ID must have from 4 to 63 characters and only include -the alphanumeric characters and a hyphen. +`GET /dnscheck/test` is the DNS server check HTTP API. It should be requested with a random ID prepended to one of the [check domains][conf-check-domains] with a hyphen. The random ID must have from 4 to 63 characters and only include the alphanumeric characters and a hyphen.