From 150f2d733f0ec6b598abbb22ba199b27be6e93db Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Thu, 4 Jan 2024 19:22:32 +0300 Subject: [PATCH] Sync v2.5.1 --- CHANGELOG.md | 256 +- Makefile | 10 + config.dist.yaml | 57 +- doc/configuration.md | 150 +- doc/development.md | 18 +- doc/environment.md | 19 +- doc/externalhttp.md | 98 +- doc/querylog.md | 7 + go.mod | 52 +- go.sum | 146 +- go.work | 2 +- go.work.sum | 64 +- internal/access/access.go | 77 +- internal/access/access_test.go | 24 +- internal/access/engine.go | 77 + internal/access/engine_internal_test.go | 107 + internal/access/profile.go | 137 + internal/access/profile_test.go | 265 + internal/agd/agd.go | 3 - internal/agd/agd_test.go | 6 - internal/agd/context.go | 10 +- internal/agd/error.go | 24 - internal/agd/errorcollector.go | 24 - internal/agd/profile.go | 15 +- internal/agd/server.go | 133 +- internal/agd/server_test.go | 152 + internal/agd/service.go | 25 - internal/agdhttp/url.go | 12 - internal/agdhttp/url_test.go | 7 - internal/agdio/agdio.go | 58 - internal/agdio/agdio_test.go | 69 - internal/agdnet/agdnet.go | 13 + internal/agdnet/agdnet_test.go | 9 + internal/agdnet/prefixaddr.go | 32 +- internal/agdnet/prefixaddr_example_test.go | 14 +- .../prefixaddr_test.go} | 31 +- internal/agdnet/resolver.go | 6 +- internal/agdservice/agdservice.go | 38 + .../agdservice_test.go} | 6 +- internal/{agd => agdservice}/refresh.go | 117 +- internal/{agd => agdservice}/refresh_test.go | 47 +- internal/agdsync/agdsync.go | 37 - internal/agdtest/agdtest.go | 22 +- internal/agdtest/interface.go | 91 +- internal/backend/backend.go | 11 - internal/backend/billstat.go | 131 - internal/backend/billstat_test.go | 103 - internal/backend/profiledb.go | 561 -- internal/backend/profiledb_test.go | 209 - internal/backend/testdata/profiles.json | 121 - internal/backendpb/backend.pb.go | 478 +- internal/backendpb/backend.proto | 20 +- internal/backendpb/backend_grpc.pb.go | 2 +- internal/backendpb/backendpb.go | 6 +- internal/backendpb/billstat.go | 5 +- internal/backendpb/billstat_test.go | 3 +- internal/backendpb/profiledb.go | 115 +- internal/backendpb/profiledb_internal_test.go | 160 +- internal/billstat/billstat.go | 13 +- internal/billstat/runtime.go | 12 +- internal/billstat/runtime_test.go | 9 +- .../bindtodevice_internal_test.go | 3 +- internal/bindtodevice/bindtodevice_test.go | 4 + internal/bindtodevice/connindex_linux.go | 3 +- .../connindex_linux_internal_test.go | 2 +- .../bindtodevice/interfacelistener_linux.go | 26 +- internal/bindtodevice/listenconfig_linux.go | 8 +- .../listenconfig_linux_internal_test.go | 6 +- internal/bindtodevice/listenconfig_others.go | 20 +- internal/bindtodevice/manager.go | 4 +- internal/bindtodevice/manager_linux.go | 59 +- internal/bindtodevice/manager_linux_test.go | 4 +- internal/bindtodevice/manager_others.go | 35 +- internal/bindtodevice/prefixaddr_linux.go | 33 - .../socket_linux_internal_test.go | 15 +- internal/cmd/backend.go | 69 +- internal/cmd/check.go | 4 +- internal/cmd/cmd.go | 66 +- internal/cmd/config.go | 6 + internal/cmd/conncheck.go | 14 +- internal/cmd/ddr.go | 13 +- internal/cmd/dns.go | 58 + internal/cmd/dnsdb.go | 6 +- internal/cmd/env.go | 43 +- internal/cmd/filter.go | 17 +- internal/cmd/geoip.go | 13 +- internal/cmd/ifacelistener.go | 4 +- internal/cmd/ratelimit.go | 108 +- internal/cmd/safebrowsing.go | 28 +- internal/cmd/server.go | 70 +- internal/cmd/servergroup.go | 9 +- internal/cmd/signal.go | 6 +- internal/cmd/tls.go | 19 +- internal/cmd/upstream.go | 141 +- internal/cmd/validation.go | 11 + internal/cmd/websvc.go | 10 +- internal/connlimiter/conn.go | 2 +- internal/connlimiter/limiter.go | 2 +- internal/connlimiter/limiter_test.go | 2 +- internal/connlimiter/listener.go | 8 +- internal/consul/allowlist.go | 7 +- internal/debugsvc/debugsvc.go | 15 +- internal/debugsvc/debugsvc_test.go | 10 +- internal/dnscheck/consul.go | 28 +- internal/dnscheck/consul_test.go | 15 +- internal/dnscheck/error.go | 64 + internal/dnscheck/httpkv_test.go | 8 +- internal/dnsdb/dnsdb.go | 5 +- internal/dnsdb/http.go | 4 +- internal/dnsmsg/blockingmode.go | 151 - internal/dnsmsg/blockingmode_example_test.go | 158 - internal/dnsmsg/cloner.go | 540 +- internal/dnsmsg/cloner_test.go | 319 +- internal/dnsmsg/clonerstat.go | 20 + internal/dnsmsg/constructor.go | 177 +- internal/dnsmsg/constructor_test.go | 18 +- internal/dnsmsg/dnsmsg.go | 22 +- internal/dnsmsg/error.go | 4 + internal/dnsmsg/error_test.go | 9 + internal/dnsmsg/httpscloner.go | 241 + internal/dnsmsg/optcloner.go | 108 + internal/dnsmsg/rrconstructor.go | 142 + internal/dnsmsg/svcbmsg_test.go | 4 +- internal/dnsserver/cache/cache.go | 7 +- internal/dnsserver/context.go | 194 +- internal/dnsserver/context_test.go | 23 +- internal/dnsserver/disposer.go | 25 + internal/dnsserver/dnsserver.go | 11 +- internal/dnsserver/dnsservertest/handler.go | 5 +- .../dnsserver/dnsservertest/quictracer.go | 105 +- internal/dnsserver/dnsservertest/server.go | 2 + internal/dnsserver/doc.go | 6 +- internal/dnsserver/example_test.go | 6 +- internal/dnsserver/forward/error.go | 21 +- internal/dnsserver/forward/example_test.go | 15 +- internal/dnsserver/forward/forward.go | 168 +- internal/dnsserver/forward/forward_test.go | 28 +- internal/dnsserver/forward/healthcheck.go | 140 +- .../dnsserver/forward/healthcheck_test.go | 16 +- internal/dnsserver/forward/metrics.go | 22 +- internal/dnsserver/forward/upstream.go | 8 +- internal/dnsserver/forward/upstreamplain.go | 129 +- .../dnsserver/forward/upstreamplain_test.go | 26 +- internal/dnsserver/go.mod | 40 +- internal/dnsserver/go.sum | 99 +- internal/dnsserver/metrics.go | 82 +- internal/dnsserver/msg.go | 19 +- internal/dnsserver/netext/packetconn_linux.go | 12 +- internal/dnsserver/normalize.go | 41 +- internal/dnsserver/prometheus/cache_test.go | 5 +- internal/dnsserver/prometheus/forward.go | 5 +- internal/dnsserver/prometheus/forward_test.go | 6 +- internal/dnsserver/prometheus/helper.go | 2 +- .../dnsserver/prometheus/prometheus_test.go | 2 +- .../dnsserver/prometheus/ratelimit_test.go | 7 +- internal/dnsserver/prometheus/server.go | 43 +- internal/dnsserver/prometheus/server_test.go | 22 +- internal/dnsserver/protocol.go | 3 +- internal/dnsserver/querylog/querylog.go | 9 +- internal/dnsserver/querylog/querylog_test.go | 8 +- internal/dnsserver/ratelimit/backoff.go | 56 +- internal/dnsserver/ratelimit/ratelimit.go | 11 +- .../dnsserver/ratelimit/ratelimit_test.go | 9 +- internal/dnsserver/serverbase.go | 113 +- internal/dnsserver/serverbench_test.go | 40 +- internal/dnsserver/serverdns.go | 104 +- internal/dnsserver/serverdns_test.go | 177 +- internal/dnsserver/serverdnscrypt.go | 14 +- internal/dnsserver/serverdnstcp.go | 300 +- internal/dnsserver/serverdnsudp.go | 116 +- internal/dnsserver/serverhttps.go | 54 +- internal/dnsserver/serverhttps_test.go | 42 +- internal/dnsserver/serverquic.go | 205 +- internal/dnsserver/serverquic_test.go | 58 +- internal/dnsserver/servertls.go | 9 +- internal/dnsserver/ttl.go | 24 +- internal/dnssvc/dnssvc.go | 248 +- internal/dnssvc/dnssvc_test.go | 87 +- internal/dnssvc/errcoll.go | 9 +- ...middleware_test.go => integration_test.go} | 109 +- internal/dnssvc/internal/accessmw/access.go | 16 +- .../dnssvc/internal/accessmw/access_test.go | 7 +- .../dnssvc/internal/dnssvctest/dnssvctest.go | 84 +- internal/dnssvc/internal/initial/deviceid.go | 6 +- .../initial/deviceid_internal_test.go | 9 +- internal/dnssvc/internal/initial/initial.go | 95 +- .../dnssvc/internal/initial/initial_test.go | 459 +- internal/dnssvc/internal/initial/profile.go | 59 +- .../internal/initial/profile_internal_test.go | 138 +- .../dnssvc/internal/initial/specialdomain.go | 8 +- .../internal/initial/specialdomain_test.go | 9 +- .../dnssvc/{ => internal/mainmw}/debug.go | 60 +- .../mainmw}/debug_internal_test.go | 156 +- internal/dnssvc/internal/mainmw/error.go | 40 + internal/dnssvc/internal/mainmw/filter.go | 220 + .../mainmw/filter_internal_test.go} | 45 +- internal/dnssvc/internal/mainmw/mainmw.go | 287 + .../dnssvc/internal/mainmw/mainmw_test.go | 726 ++ internal/dnssvc/internal/mainmw/record.go | 177 + .../dnssvc/internal/preservice/preservice.go | 129 + .../preservice/preservice_test.go} | 62 +- .../internal/preupstream/preupstream.go | 205 + .../preupstream/preupstream_test.go} | 71 +- internal/dnssvc/middleware.go | 222 - internal/dnssvc/presvcmw.go | 126 - internal/dnssvc/preupstreammw.go | 168 - internal/dnssvc/record.go | 207 - internal/dnssvc/resp.go | 122 - internal/ecscache/cache.go | 48 +- internal/ecscache/cache_internal_test.go | 2 +- internal/ecscache/ecsblocklist.go | 8566 +++++++++++++++++ internal/ecscache/ecsblocklist_generate.go | 74 + internal/ecscache/ecscache.go | 111 +- internal/ecscache/ecscache_test.go | 262 +- internal/errcoll/errcoll.go | 29 +- internal/errcoll/sentry.go | 21 +- internal/errcoll/sentry_test.go | 2 +- internal/errcoll/writer.go | 36 +- internal/errcoll/writer_test.go | 2 +- internal/filter/filter_test.go | 1 + internal/filter/hashprefix/filter.go | 29 +- internal/filter/hashprefix/filter_test.go | 3 + internal/filter/hashprefix/matcher.go | 13 +- internal/filter/hashprefix/matcher_test.go | 2 +- internal/filter/hashprefix/storage.go | 79 +- internal/filter/hashprefix/storage_test.go | 26 +- internal/filter/index.go | 7 +- .../filter/internal/composite/composite.go | 56 +- .../composite/composite_internal_test.go | 52 + .../internal/composite/composite_test.go | 37 +- .../filter/internal/composite/dnsrewrite.go | 10 +- internal/filter/internal/custom/custom.go | 15 +- .../filter/internal/filtertest/filtertest.go | 2 +- internal/filter/internal/refreshable.go | 8 +- internal/filter/internal/result.go | 10 +- .../filter/internal/safesearch/safesearch.go | 81 +- .../internal/safesearch/safesearch_test.go | 8 +- .../filter/internal/serviceblock/index.go | 5 +- .../internal/serviceblock/serviceblock.go | 5 +- internal/filter/storage.go | 26 +- internal/filter/storage_test.go | 39 +- internal/geoip/asntops.go | 1491 ++- internal/geoip/asntops_generate.go | 18 +- internal/{agd => geoip}/country.go | 2 +- internal/{agd => geoip}/country_generate.go | 4 +- internal/geoip/error.go | 27 + internal/geoip/file.go | 207 +- internal/geoip/file_test.go | 37 +- internal/geoip/filescanner.go | 214 +- internal/geoip/geoip.go | 11 +- internal/geoip/geoip_test.go | 38 +- internal/{agd => geoip}/location.go | 2 +- internal/geoip/testdata/GeoIP2-City-Test.mmdb | Bin 22084 -> 22883 bytes .../geoip/testdata/GeoIP2-Country-Test.mmdb | Bin 19394 -> 19962 bytes internal/geoip/testdata/GeoIP2-ISP-Test.mmdb | Bin 0 -> 76623 bytes .../geoip/testdata/GeoLite2-ASN-Test.mmdb | Bin 12358 -> 0 bytes internal/metrics/access.go | 28 +- internal/metrics/backend.go | 25 +- internal/metrics/dnscheck.go | 14 +- internal/metrics/dnsmsg.go | 45 + internal/metrics/dnssvc.go | 9 + internal/metrics/metrics.go | 13 +- internal/metrics/research.go | 92 +- internal/metrics/tls.go | 25 +- internal/metrics/tls_test.go | 26 +- .../internal/filecachepb/filecache.pb.go | 393 +- .../internal/filecachepb/filecache.proto | 15 + .../internal/filecachepb/filecachepb.go | 111 +- .../profiledb/internal/filecachepb/storage.go | 9 +- internal/profiledb/internal/internal.go | 2 +- .../internal/profiledbtest/profiledbtest.go | 18 +- internal/profiledb/profiledb.go | 166 +- internal/profiledb/profiledb_test.go | 32 +- internal/querylog/entry.go | 29 +- internal/querylog/fs.go | 13 +- internal/querylog/fs_test.go | 4 +- internal/querylog/querylog_test.go | 5 +- internal/rulestat/http.go | 8 +- internal/tools/go.mod | 28 +- internal/tools/go.sum | 74 +- internal/websvc/handler_test.go | 6 +- internal/websvc/linkip.go | 7 +- internal/websvc/websvc.go | 18 +- internal/websvc/websvc_test.go | 6 +- scripts/backend/main.go | 162 + scripts/make/go-fuzz.sh | 56 + scripts/make/go-gen.sh | 30 +- scripts/make/go-lint.sh | 97 +- scripts/make/helper.sh | 4 +- scripts/make/txt-lint.sh | 9 +- staticcheck.conf | 8 + 291 files changed, 20054 insertions(+), 7857 deletions(-) create mode 100644 internal/access/engine.go create mode 100644 internal/access/engine_internal_test.go create mode 100644 internal/access/profile.go create mode 100644 internal/access/profile_test.go delete mode 100644 internal/agd/errorcollector.go create mode 100644 internal/agd/server_test.go delete mode 100644 internal/agd/service.go delete mode 100644 internal/agdio/agdio.go delete mode 100644 internal/agdio/agdio_test.go create mode 100644 internal/agdnet/agdnet_test.go rename internal/{bindtodevice/prefixaddr_linux_internal_test.go => agdnet/prefixaddr_test.go} (68%) create mode 100644 internal/agdservice/agdservice.go rename internal/{backend/backend_test.go => agdservice/agdservice_test.go} (50%) rename internal/{agd => agdservice}/refresh.go (51%) rename internal/{agd => agdservice}/refresh_test.go (72%) delete mode 100644 internal/agdsync/agdsync.go delete mode 100644 internal/backend/backend.go delete mode 100644 internal/backend/billstat.go delete mode 100644 internal/backend/billstat_test.go delete mode 100644 internal/backend/profiledb.go delete mode 100644 internal/backend/profiledb_test.go delete mode 100644 internal/backend/testdata/profiles.json delete mode 100644 internal/bindtodevice/prefixaddr_linux.go create mode 100644 internal/cmd/dns.go create mode 100644 internal/dnscheck/error.go delete mode 100644 internal/dnsmsg/blockingmode_example_test.go create mode 100644 internal/dnsmsg/clonerstat.go create mode 100644 internal/dnsmsg/error_test.go create mode 100644 internal/dnsmsg/httpscloner.go create mode 100644 internal/dnsmsg/optcloner.go create mode 100644 internal/dnsmsg/rrconstructor.go create mode 100644 internal/dnsserver/disposer.go rename internal/dnssvc/{middleware_test.go => integration_test.go} (80%) rename internal/dnssvc/{ => internal/mainmw}/debug.go (75%) rename internal/dnssvc/{ => internal/mainmw}/debug_internal_test.go (56%) create mode 100644 internal/dnssvc/internal/mainmw/error.go create mode 100644 internal/dnssvc/internal/mainmw/filter.go rename internal/dnssvc/{resp_internal_test.go => internal/mainmw/filter_internal_test.go} (68%) create mode 100644 internal/dnssvc/internal/mainmw/mainmw.go create mode 100644 internal/dnssvc/internal/mainmw/mainmw_test.go create mode 100644 internal/dnssvc/internal/mainmw/record.go create mode 100644 internal/dnssvc/internal/preservice/preservice.go rename internal/dnssvc/{presvcmw_internal_test.go => internal/preservice/preservice_test.go} (69%) create mode 100644 internal/dnssvc/internal/preupstream/preupstream.go rename internal/dnssvc/{preupstreammw_internal_test.go => internal/preupstream/preupstream_test.go} (78%) delete mode 100644 internal/dnssvc/middleware.go delete mode 100644 internal/dnssvc/presvcmw.go delete mode 100644 internal/dnssvc/preupstreammw.go delete mode 100644 internal/dnssvc/record.go delete mode 100644 internal/dnssvc/resp.go create mode 100644 internal/ecscache/ecsblocklist.go create mode 100644 internal/ecscache/ecsblocklist_generate.go create mode 100644 internal/filter/internal/composite/composite_internal_test.go rename internal/{agd => geoip}/country.go (99%) rename internal/{agd => geoip}/country_generate.go (98%) create mode 100644 internal/geoip/error.go rename internal/{agd => geoip}/location.go (99%) create mode 100644 internal/geoip/testdata/GeoIP2-ISP-Test.mmdb delete mode 100644 internal/geoip/testdata/GeoLite2-ASN-Test.mmdb create mode 100644 internal/metrics/dnsmsg.go create mode 100644 scripts/backend/main.go create mode 100644 scripts/make/go-fuzz.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 29b5bc7..2b806e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,247 @@ The format is **not** based on [Keep a Changelog][kec], since the project +## AGDNS-1761 / Build 702 + + * The property `upstream` has been modified. Its property `timeout` has been + replaced with the new property `servers.timeout` for each server in the + `servers` list. Concomitantly the `fallback.timeout` has been replaced with + `fallback.servers.timeout` for each fallback server. The `fallback.servers` + now supports not only the addresses of the servers, but URLs in the + `[scheme://]ip:port` format like it's done with the main servers. So replace + this: + + ```yaml + upstream: + # … + servers: + - 'tcp://1.1.1.1:53' + - '127.0.0.1:5358' + timeout: 2s + fallback: + servers: + - 8.8.4.4:53 + timeout: 1s + ``` + + with this: + + ```yaml + upstream: + # … + servers: + - address: 'tcp://1.1.1.1:53' + timeout: 2s + - address: '127.0.0.1:5358' + timeout: 2s + fallback: + servers: + - address: '8.8.4.4:53' + timeout: 1s + ``` + + Adjust the value and add new ones, if necessary. + + + +## AGDNS-698 / Build 701 + + * The object `dns` has new properties: `read_timeout`, `tcp_idle_timeout`, and + `write_timeout`. So replace this: + + ```yaml + dns: + max_udp_response_size: 1024B + ``` + + with this: + + ```yaml + dns: + read_timeout: 2s + tcp_idle_timeout: 30s + write_timeout: 2s + handle_timeout: 1s + max_udp_response_size: 1024B + ``` + + The values in the example are previous defaults. + + + +## AGDNS-1751 / Build 691 + + * The property `upstream.server` has been removed. Its former content is + moved to the newly added property `servers`, which now extended to contain + a list of URLs of main upstream servers. So replace this: + + ```yaml + upstream: + # … + server: `8.8.8.8:53` + ``` + + with this: + + ```yaml + upstream: + # … + servers: + - `8.8.8.8:53` + ``` + + Adjust the value and add new ones, if necessary. + + + +## AGDNS-1759 / Build 684 + + * The object `backend` has a new property, `full_refresh_retry_interval`. So + replace this: + + ```yaml + backend: + # … + full_refresh_interval: 24h + ``` + + with this: + + ```yaml + backend: + # … + full_refresh_interval: 24h + full_refresh_retry_interval: 1h + ``` + + Adjust the value, if necessary. + + + +## AGDNS-1744 / Build 681 + + * Metric `forward_request_total` has a new label `network`. This label + describes the network type (`tcp` or `udp`), over which an upstream has + finished processing request. + + + +## AGDNS-1738 / Build 678 + + * Object `dns` has a new property, describing maximum size of DNS response + over UDP protocol. + + ```yaml + dns: + max_udp_response_size: 1024B + handle_timeout: 1s + ``` + + + +## AGDNS-1735 / Build 677 + + * The property `upstream.fallback` has been changed. Its former content is + moved to the newly added property `servers`. The new property `timeout`, + which describes query timeout to fallback servers, was added. So replace + this: + + ```yaml + upstream: + fallback: + - 1.1.1.1:53 + - 8.8.8.8:53 + ``` + + with this: + + ```yaml + upstream: + fallback: + servers: + - 1.1.1.1:53 + - 8.8.8.8:53 + timeout: 1s + ``` + + Adjust the new values, if necessary. Note that the query timeout to fallback + servers was previously defined with `upstream.timeout` property, which now + describes the query timeout to the primary servers only. + + + +## AGDNS-1178 / Build 676 + + * The new object `dns` has been added: + + ```yaml + dns: + handle_timeout: 1s + ``` + + + +## AGDNS-1620 / Build 673 + + * Object `ratelimit` has two new properties: `quic` and `tcp`. They configure + QUIC and TCP connection limits. Example configuration: + + ```yaml + ratelimit: + # … + quic: + enabled: true + max_streams_per_peer: 100 + tcp: + enabled: true + max_pipeline_count: 100 + ``` + + + +## AGDNS-1684 / Build 661 + + * Profile's file cache version was incremented. The new field `access` has + been added. + + + +## AGDNS-1664 / Build 636 + + * The environment variables `BILLSTAT_URL` and `PROFILES_URL` no longer + support HTTP(s) endpoints. Use GRPC(S) instead. + + + +## AGDNS-1667 / Build 633 + +* `ratelimit` configuration properties `back_off_count`, `back_off_duration` + and `back_off_period` have been renamed to `backoff_count`, + `backoff_duration` and `backoff_period`. So replace this: + + ```yaml + ratelimit: + back_off_period: 10m + back_off_count: 1000 + back_off_duration: 30m + ``` + + with this: + + ```yaml + ratelimit: + backoff_period: 10m + backoff_count: 1000 + backoff_duration: 30m + ``` + + + ## AGDNS-1607 / Build 617 -* New configuration `access` has been added, it has an a list of AdBlock rules - to block requests, and a lists of client subnets to block access from. - Example configuration: + * New configuration `access` has been added, it has an a list of AdBlock rules + to block requests, and a lists of client subnets to block access from. + Example configuration: ```yaml access: @@ -31,10 +267,11 @@ The format is **not** based on [Keep a Changelog][kec], since the project ## AGDNS-1619 / Build 611 -* Added a new metric `bill_stat_upload_duration` that counts the duration of - billing statistics upload. -* The environment variable `BILLSTAT_URL`, which describes the endpoint for - backend billing statistics uploader API, now supports GRPC endpoints. + * Added a new metric `bill_stat_upload_duration` that counts the duration of + billing statistics upload. + + * The environment variable `BILLSTAT_URL`, which describes the endpoint for + backend billing statistics uploader API, now supports GRPC endpoints. @@ -57,7 +294,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project * The optional property `bind_interfaces` of `server_groups.*.servers` objects has been changed, property `subnet` is now an array and has been - ranamed to `subnets`. So replace this: + renamed to `subnets`. So replace this: ```yaml bind_interfaces: @@ -98,6 +335,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project ## AGDNS-1580 / Build 562 * The environment variable `DNSDB_PATH` has been removed. + * New configuration `dnsdb` has been added, it has an enabled/disabled flag and the property `max_size` which describes the maximum amount of records in the in-memory buffer. Example configuration: @@ -944,7 +1182,7 @@ The format is **not** based on [Keep a Changelog][kec], since the project identifiers, grouped by endpoint identifier and known server names. All unknown server names are grouped in `other` label: - ``` + ```none # TYPE dns_tls_handshake_total counter dns_tls_handshake_total{cipher_suite="TLS_AES_128_GCM_SHA256",did_resume="0",negotiated_proto="",proto="tls",server_name="default_dot: other",tls_version="tls1.3"} 4 ``` diff --git a/Makefile b/Makefile index 66973fe..d5d91df 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ VERBOSE.MACRO = $${VERBOSE:-0} BRANCH = $$( git rev-parse --abbrev-ref HEAD ) GOAMD64 = v1 GOPROXY = https://goproxy.cn|https://proxy.golang.org|direct +GOTOOLCHAIN = go1.21.5 RACE = 0 REVISION = $$( git rev-parse --short HEAD ) VERSION = 0 @@ -32,6 +33,7 @@ ENV = env\ GO="$(GO.MACRO)"\ GOAMD64='$(GOAMD64)'\ GOPROXY='$(GOPROXY)'\ + GOTOOLCHAIN='$(GOTOOLCHAIN)'\ PATH="$${PWD}/bin:$$( "$(GO.MACRO)" env GOPATH )/bin:$${PATH}"\ RACE='$(RACE)'\ REVISION="$(REVISION)"\ @@ -51,6 +53,7 @@ test: go-test go-bench: ; $(ENV) "$(SHELL)" ./scripts/make/go-bench.sh go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh +go-fuzz: ; $(ENV) "$(SHELL)" ./scripts/make/go-fuzz.sh go-gen: ; $(ENV) "$(SHELL)" ./scripts/make/go-gen.sh go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh go-test: ; $(ENV) RACE='1' "$(SHELL)" ./scripts/make/go-test.sh @@ -70,4 +73,11 @@ go-os-check: txt-lint: ; $(ENV) "$(SHELL)" ./scripts/make/txt-lint.sh +# TODO(a.garipov): Consider adding to scripts/ and the common project +# structure. +go-upd-tools: + cd ./internal/tools/ &&\ + "$(GO.MACRO)" get -u &&\ + "$(GO.MACRO)" mod tidy + sync-github: ; $(ENV) "$(SHELL)" ./scripts/make/github-sync.sh diff --git a/config.dist.yaml b/config.dist.yaml index 72763f3..378047b 100644 --- a/config.dist.yaml +++ b/config.dist.yaml @@ -24,15 +24,13 @@ ratelimit: subnet_key_len: 48 # The time during which to count the number of times a client has hit the # rate limit for a back off. - # - # TODO(a.garipov): Rename to "backoff_period" along with others. - back_off_period: 10m + backoff_period: 10m # How many times a client hits the rate limit before being held in the back # off. - back_off_count: 1000 + backoff_count: 1000 # How much a client that has hit the rate limit too often stays in the back # off. - back_off_duration: 30m + backoff_duration: 30m # Configuration for the allowlist. allowlist: @@ -52,6 +50,19 @@ ratelimit: stop: 1000 resume: 800 + # Configuration of QUIC streams limiting. + quic: + enabled: true + # The maximum number of concurrent streams that a peer is allowed to + # open. + max_streams_per_peer: 100 + + # Configuration of TCP pipeline limiting. + tcp: + enabled: true + # The maximum number of processing TCP messages per one connection. + max_pipeline_count: 100 + # Access settings. access: # Domains to block. @@ -78,11 +89,17 @@ cache: # DNS upstream configuration. upstream: - server: '8.8.8.8:53' - timeout: 2s + servers: + - address: 'tcp://1.1.1.1:53' + timeout: 2s + - address: '8.8.4.4:53' + timeout: 2s fallback: - - 1.1.1.1:53 - - 8.8.8.8:53 + servers: + - address: '1.1.1.1:53' + timeout: 1s + - address: '8.8.8.8:53' + timeout: 1s healthcheck: enabled: true interval: 2s @@ -90,6 +107,25 @@ upstream: backoff_duration: 30s domain_template: '${RANDOM}.neverssl.com' +# Common DNS settings. +# +# TODO(a.garipov): Consider making these settings per-server-group. +dns: + # The timeout for any read from a UDP connection or the first read from + # a TCP/TLS connection. It currently doesn't affect DNSCrypt, QUIC, or + # HTTPS. + read_timeout: 2s + # The timeout for consecutive reads from a TCP/TLS connection. It currently + # doesn't affect DNSCrypt, QUIC, or HTTPS. + tcp_idle_timeout: 30s + # The timeout for writing to a UDP or TCP/TLS connection. It currently + # doesn't affect DNSCrypt, QUIC, or HTTPS. + write_timeout: 2s + # The timeout for the entire handling of a single query. + handle_timeout: 1s + # UDP response size limit. + max_udp_response_size: 1024B + # DNSDB configuration. dnsdb: enabled: true @@ -106,6 +142,9 @@ backend: refresh_interval: 15s # How often AdGuard DNS performs full synchronization. full_refresh_interval: 24h + # How long to wait before attempting a new full synchronization after a + # failure. + full_refresh_retry_interval: 1h # How often AdGuard DNS sends the billing statistics to the backend. bill_stat_interval: 15s diff --git a/doc/configuration.md b/doc/configuration.md index 8ec88db..ee99059 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -15,6 +15,7 @@ configuration file with comments. * [Cache](#cache) * [Upstream](#upstream) * [Healthcheck](#upstream-healthcheck) + * [Common DNS settings](#dns) * [DNSDB](#dnsdb) * [Backend](#backend) * [Query log](#query_log) @@ -130,13 +131,13 @@ The `ratelimit` object has the following properties: **Example:** `1KB`. - * `back_off_period`: + * `backoff_period`: The time during which to count the number of requests that a client has sent over the RPS. **Example:** `10m`. - * `back_off_duration`: + * `backoff_duration`: How long a client that has hit the RPS too often stays in the backoff state. **Example:** `30m`. @@ -159,10 +160,10 @@ The `ratelimit` object has the following properties: The `ipv6` configuration object has the same properties as the `ipv4` one above. - * `back_off_count`: - Maximum number of requests a client can make above the RPS within - a `back_off_period`. When a client exceeds this limit, requests aren't - allowed from client's subnet until `back_off_duration` ends. + * `backoff_count`: + Maximum number of requests a client can make above the RPS within a + `backoff_period`. When a client exceeds this limit, requests aren't allowed + from client's subnet until `backoff_duration` ends. **Example:** `1000`. @@ -188,11 +189,11 @@ The `ratelimit` object has the following properties: **Example:** `30s`. -For example, if `back_off_period` is `1m`, `back_off_count` is `10`, and +For example, if `backoff_period` is `1m`, `backoff_count` is `10`, and `ipv4-rps` is `5`, a client (meaning all IP addresses within the subnet defined by `ipv4-subnet_key_len`) that made 15 requests in one second or 6 requests (one above `rps`) every second for 10 seconds within one minute, the client is -blocked for `back_off_duration`. +blocked for `backoff_duration`. ### Stream connection limit @@ -218,6 +219,35 @@ The `connection_limit` object has the following properties: See also [notes on these parameters](#recommended-connection_limit). + ### QUIC rate limiting + +The `quic` object has the following properties: + + * `enabled`: + Whether or not the QUIC connections rate limiting should be enforced. + + **Example:** `true`. + + * `max_streams_per_peer`: + The maximum number of concurrent streams that a peer is allowed to open. + + **Example:** `1000`. + + ### TCP rate limiting + +The `tcp` object has the following properties: + + * `enabled`: + Whether or not the TCP rate limiting should be enforced. + + **Example:** `true`. + + * `max_pipeline_count`: + The maximum number of simultaneously processing TCP messages per one + connection. + + **Example:** `1000`. + [env-consul_allowlist_url]: environment.md#CONSUL_ALLOWLIST_URL @@ -269,26 +299,45 @@ The `cache` object has the following properties: The `upstream` object has the following properties: - * `server`: - The URL of the main upstream server, in the `[scheme://]ip:port` format. + * `servers`: + The array of the main upstream servers URLs, in the `[scheme://]ip:port` + format and its timeouts for main upstream DNS requests, as a human-readable + duration. - **Examples:** + **Property example:** - - `8.8.8.8:53`: regular DNS (over UDP with TCP fallback). - - `tcp://1.1.1.1:53`: regular DNS (over TCP). - - `udp://1.1.1.1:53`: regular DNS (over UDP). - - * `timeout`: - Timeout for all outgoing DNS requests, as a human-readable duration. - - **Example:** `2s`. + ```yaml + 'servers': + # Regular DNS (over UDP with TCP fallback). + - address: '8.8.8.8:53' + timeout: 2s + # Regular DNS (over TCP). + - address: 'tcp://1.1.1.1:53' + timeout: 2s + # Regular DNS (over UDP). + - address: 'udp://1.1.1.1:53' + timeout: 2s + ``` * `fallback`: - The array of addresses of the fallback upstream servers, in the `ip:port` - format. These are use used in case a network error occurs while requesting - the main upstream server. + Fallback servers configuration. It has the following properties: - **Example:** `['1.1.1.1:53', '[2001:4860:4860::8888]:53']`. + * `servers`: + The array of the fallback upstream servers URLs, in the + `[scheme://]ip:port` format and its timeouts for upstream DNS requests, + as a human-readable duration. These are use used in case a network error + occurs while requesting the main upstream server. This property has the + same format as [`upstream-servers`](#upstream-servers) above. + + **Property example:** + + ```yaml + 'servers': + - address: '1.1.1.1:53' + timeout: 2s + - address: '[2001:4860:4860::8888]:53' + timeout: 2s + ``` * `healthcheck`: Healthcheck configuration. See [below](#upstream-healthcheck). @@ -341,9 +390,46 @@ connection to the main upstream as restored, and requests are routed back to it. +## DNS + +The `dns` object has the following properties: + + * `read_timeout`: + The timeout for any read from a UDP connection or the first read from + a TCP/TLS connection, as a human-readable duration. It currently doesn't + affect DNSCrypt, QUIC, or HTTPS. + + **Example:** `2s`. + + * `tcp_idle_timeout`: + The timeout for consecutive reads from a TCP/TLS connection, as a + human-readable duration. It currently doesn't affect DNSCrypt, QUIC, or + HTTPS. + + **Example:** `30s`. + + * `write_timeout`: + The timeout for writing to a UDP or TCP/TLS connection, as a human-readable + duration. It currently doesn't affect DNSCrypt, QUIC, or HTTPS. + + **Example:** `2s`. + + * `handle_timeout`: + The timeout for the entire handling of a single query, as a human-readable + duration. + + **Example:** `1s`. + + * `max_udp_response_size`: + The maximum size of DNS response over UDP protocol. + + **Example:** `1024B`. + + + ## DNSDB -The `DNSDB` object has the following properties: +The `dnsdb` object has the following properties: * `enabled`: If true, the DNSDB memory buffer is enabled. @@ -382,6 +468,13 @@ The `backend` object has the following properties: **Example:** `24h`. + * `full_refresh_retry_interval`: + How long to wait before attempting a new full profile synchronization after + a failure, as a human-readable duration. It is recommended to keep this + value greater than [`refresh_interval`](#backend-refresh_interval). + + **Example:** `1h`. + * `bill_stat_interval`: How often AdGuard DNS sends the billing statistics to the backend, as a human-readable duration. @@ -552,9 +645,9 @@ The optional `web` object has the following properties: The optional listen addresses and optional TLS configuration for the web service in addition to the ones in the DNS-over-HTTPS handlers. The `certificates` array has the same format as the one in a server group's [TLS - settings](#sg-*-tls). In the special case of `GET /robots.txt` requests, a - special response is served; this response could be overwritten with static - content. + settings](#server_groups-*-tls). In the special case of `GET /robots.txt` + requests, a special response is served; this response could be overwritten + with static content. **Property example:** @@ -604,7 +697,6 @@ The optional `web` object has the following properties: **Example:** `30s`. [http-block-pages]: http.md#block-pages -[http-dnscheck-test]: http.md#dhscheck-test [http-linked-ip-proxy]: http.md#linked-ip-proxy @@ -677,7 +769,7 @@ The `filters` object has the following properties: * `refresh_interval`: How often AdGuard DNS refreshes the rule-list filters from the filter index, as well as the blocked services list from the [blocked list - index][env-blocked_services)]. + index][env-blocked_services]. **Example:** `1h`. diff --git a/doc/development.md b/doc/development.md index 6b52219..5020058 100644 --- a/doc/development.md +++ b/doc/development.md @@ -13,7 +13,7 @@ Development is supported on Linux and macOS (aka Darwin) systems. -1. Install Go 1.20 or later. +1. Install Go 1.21 or later. 1. Call `make init` to set up the Git pre-commit hook. @@ -74,11 +74,14 @@ This is not an extensive list. See `../Makefile`.