Sync v2.2.1

.gitignore

@@ -16,6 +16,5 @@ AdGuardDNS
 asn.mmdb
 config.yaml
 country.mmdb
-dnsdb.bolt
 querylog.jsonl
 profilecache.json

CHANGELOG.md

@@ -11,6 +11,190 @@ The format is **not** based on [Keep a Changelog][kec], since the project
 
 
 
+##  AGDNS-1537 / Build 580
+
+ *  The optional property `bind_interfaces` of `server_groups.*.servers`
+    objects has been changed, property `subnet` is now an array and has been
+    ranamed to `subnets`.  So replace this:
+
+    ```yaml
+    bind_interfaces:
+      - id: 'dns'
+        subnet: '10.0.0.1/32'
+      - id: 'dns'
+        subnet: '10.0.0.2/32'
+      - id: 'dns'
+        subnet: '10.0.0.3/32'
+      - id: 'dns_secondary'
+        subnet: '10.0.0.1/32'
+    ```
+
+    with this:
+
+    ```yaml
+    bind_interfaces:
+      - id: 'dns'
+        subnets:
+          - '10.0.0.1/32'
+          - '10.0.0.2/32'
+          - '10.0.0.3/32'
+      - id: 'dns_secondary'
+        subnets:
+          - '10.0.0.1/32'
+    ```
+
+
+
+##  AGDNS-1537 / Build 566
+
+ *  The configuration property `filtering_groups.safe_browsing` has been changed,
+    new properties have been added: `block_dangerous_domains` and
+    `block_newly_registered_domains`.
+
+
+
+##  AGDNS-1580 / Build 562
+
+ *  The environment variable `DNSDB_PATH` has been removed.
+ *  New configuration `dnsdb` has been added, it has an enabled/disabled flag
+    and the property `max_size` which describes the maximum amount of records in
+    the in-memory buffer.  Example configuration:
+
+    ```yaml
+    dnsdb:
+        enabled: true
+        max_size: 500000
+    ```
+
+
+
+##  AGDNS-1537 / Build 559
+
+ *  Configuration properties `safe_browsing.url` and `adult_blocking.url` are
+    now removed.  Use newly added environment variables `ADULT_BLOCKING_URL` and
+    `SAFE_BROWSING_URL`.
+ *  New environment variable `NEW_REG_DOMAINS_URL` has been added, this is the
+    link to the source list of the newly registered domains.
+
+
+
+##  AGDNS-1567 / Build 557
+
+ *  The environment variable `BACKEND_ENDPOINT` was replaced with three
+    environment variables:
+
+     *  `LINKED_IP_TARGET_URL`: the target URL to which linked IP API requests
+        are proxied.
+     *  `PROFILES_URL`: the endpoint for profiles sync API.
+     *  `BILLSTAT_URL`: the endpoint for backend billing statistics uploader.
+
+
+
+##  AGDNS-1561 / Build 554
+
+ *  The `filters` object has a new property, `max_size`, which describes the
+    maximum size of the downloadable content for a rule-list in a human-readable
+    format.  Example configuration:
+
+    ```yaml
+    filters:
+        # …
+        max_size: 256MB
+    ```
+
+
+
+##  AGDNS-1561 / Build 550
+
+ *  Properties `so_sndbuf` and `so_rcvbuf` of object `network` have been changed.
+    Now they are in a human-readable format.  Example configuration:
+
+    ```yaml
+    network:
+        so_sndbuf: 2MB
+        so_rcvbuf: 0
+    ```
+
+ *  The object `filters` has been changed. Two properties,
+    `rule_list_cache_size` and `use_rule_list_cache` have been extracted to the
+    new object `rule_list_cache` and renamed to `size` and `enabled`.  So
+    replace this:
+
+    ```yaml
+    filters:
+        response_ttl: 5m
+        custom_filter_cache_size: 1024
+        safe_search_cache_size: 1024
+        rule_list_cache_size: 10000
+        refresh_interval: 1h
+        refresh_timeout: 5m
+        use_rule_list_cache: true
+    ```
+
+    with this:
+
+    ```yaml
+    filters:
+        response_ttl: 5m
+        custom_filter_cache_size: 1024
+        safe_search_cache_size: 1024
+        refresh_interval: 1h
+        refresh_timeout: 5m
+        rule_list_cache:
+            enabled: true
+            size: 10000
+    ```
+
+    Adjust the values, if necessary.
+
+
+
+##  AGDNS-1566 / Build 549
+
+ *  There is now a new env variable `RESEARCH_LOGS` that controls whether
+    logging of additional info for research purposes is enabled.  These log
+    records can be filtered out by `research:` prefix.  The default value is
+    `0`, i.e. additional logging is disabled.  The first thing that is logged
+    in this version is domains which responses have ECH config.  The log will
+    only be recorded when both `RESEARCH_LOGS` and `RESEARCH_METRICS` are set
+    to `1`.
+
+ *  Added a new research metric `dns_research_response_ech` that counts the
+    number of responses with a ECH configuration.
+
+
+
+##  AGDNS-1556 / Build 547
+
+ *  The object `cache` has a new property `ttl_override`.  It describes the TTL
+    override settings, such as the minimum TTL for cache items and the `enabled`
+    switch.  It overwrites the TTL from DNS response in case it's less than this
+    minimum value.  So replace this:
+
+    ```yaml
+    cache:
+        type: "simple"
+        size: 10000
+        ecs_size: 10000
+    ```
+
+    with this:
+
+    ```yaml
+    cache:
+        type: "simple"
+        size: 10000
+        ecs_size: 10000
+        ttl_override:
+            enabled: true
+            # The minimum duration of TTL for a cache item.
+            min: 60s
+    ```
+
+    Adjust the values, if necessary.
+
+
+
 ##  AGDNS-1498 / Build 527
 
  *  Object `ratelimit` has a new property, `connection_limit`, which allows

Makefile

@@ -10,7 +10,7 @@
 #
 # AdGuard-Project-Version: 2
 
-# Don't name these macros "GO" etc., because GNU Make apparenly makes
+# Don't name these macros "GO" etc., because GNU Make apparently makes
 # them exported environment variables with the literal value of
 # "${GO:-go}" and so on, which is not what we need.  Use a dot in the
 # name to make sure that users don't have an environment variable with

config.dist.yaml

@@ -62,6 +62,10 @@ cache:
     size: 10000
     # The total number of items in the cache for hostnames with ECS support.
     ecs_size: 10000
+    ttl_override:
+        enabled: true
+        # The minimum duration of TTL for a cache item.
+        min: 60s
 
 # DNS upstream configuration.
 upstream:
@@ -77,6 +81,11 @@ upstream:
         backoff_duration: 30s
         domain_template: '${RANDOM}.neverssl.com'
 
+# DNSDB configuration.
+dnsdb:
+    enabled: true
+    max_size: 500000
+
 # Common DNS HTTP backend service configuration.
 backend:
     # Timeout for all outgoing backend HTTP requests.  Set to `0s` to disable
@@ -192,7 +201,6 @@ web:
 
 # AdGuard general safe browsing filter configuration.
 safe_browsing:
-    url: 'https://raw.githubusercontent.com/ameshkov/PersonalFilters/master/safebrowsing_test.txt'
     block_host: 'standard-block.dns.adguard.com'
     cache_size: 1024
     cache_ttl: 1h
@@ -200,7 +208,6 @@ safe_browsing:
 
 # AdGuard adult content blocking filter configuration.
 adult_blocking:
-    url: 'https://raw.githubusercontent.com/ameshkov/PersonalFilters/master/adult_test.txt'
     block_host: 'family-block.dns.adguard.com'
     cache_size: 1024
     cache_ttl: 1h
@@ -215,16 +222,20 @@ filters:
     custom_filter_cache_size: 1024
     # The size of the LRU cache of safe-search filtering results.
     safe_search_cache_size: 1024
-    # The size of the LRU cache of rule-list filtering results.
-    rule_list_cache_size: 10000
     # How often to update filters from the index.  See the documentation for the
     # FILTER_INDEX_URL environment variable.
     refresh_interval: 1h
     # The timeout for the entire filter update operation.  Be aware that each
     # individual refresh operation also has its own hardcoded 30s timeout.
     refresh_timeout: 5m
+    # MaxSize is the maximum size of the downloadable filtering rule-list.
+    max_size: 256MB
+    # Rule list cache.
+    rule_list_cache:
         # If true, use filtering rule list result cache.
-    use_rule_list_cache: true
+        enabled: true
+        # The size of the LRU cache of rule-list filtering results.
+        size: 10000
 
 # Filtering groups are a set of different filtering configurations.  These
 # filtering configurations are then used by server_groups.
@@ -240,6 +251,8 @@ filtering_groups:
           - 'adguard_dns_filter'
     safe_browsing:
         enabled: true
+        block_dangerous_domains: true
+        block_newly_registered_domains: false
     block_private_relay: false
     block_firefox_canary: true
   - id: 'family'
@@ -254,6 +267,8 @@ filtering_groups:
           - 'adguard_dns_filter'
     safe_browsing:
         enabled: true
+        block_dangerous_domains: true
+        block_newly_registered_domains: false
     block_private_relay: false
     block_firefox_canary: true
   - id: 'non_filtering'
@@ -263,6 +278,8 @@ filtering_groups:
         enabled: false
     safe_browsing:
         enabled: false
+        block_dangerous_domains: true
+        block_newly_registered_domains: false
     block_private_relay: false
     block_firefox_canary: true
 
@@ -330,9 +347,11 @@ server_groups:
         # the plain-DNS servers.
         bind_interfaces:
           - id: 'eth0_plain_dns'
-            subnet: '127.0.0.0/8'
+            subnets:
+              - '127.0.0.0/8'
           - id: 'eth0_plain_dns_secondary'
-            subnet: '127.0.0.0/8'
+            subnets:
+              - '127.0.0.0/8'
       - name: 'default_dot'
         protocol: 'tls'
         linked_ip_enabled: false
@@ -383,9 +402,9 @@ additional_metrics_info:
 
 # Network settings.
 network:
-    # Defines the size of socket send buffer in bytes.  Default is zero (uses
+    # Defines the size of socket send buffer in a human-readable format.
-    # system settings).
+    # Default is zero (uses system settings).
     so_sndbuf: 0
-    # Defines the size of socket receive buffer in bytes.  Default is zero
+    # Defines the size of socket receive buffer in a human-readable format.
-    # (uses system settings).
+    #  Default is zero (uses system settings).
     so_rcvbuf: 0

doc/configuration.md

@@ -15,6 +15,7 @@ configuration file with comments.
  *  [Cache](#cache)
  *  [Upstream](#upstream)
      *  [Healthcheck](#upstream-healthcheck)
+ *  [DNSDB](#dnsdb)
  *  [Backend](#backend)
  *  [Query log](#query_log)
  *  [GeoIP database](#geoip)
@@ -242,6 +243,25 @@ The `cache` object has the following properties:
 
     **Example:** `10000`.
 
+ *  <a href="#cache-ttl_override" id="cache-ttl_override" name="cache-ttl_override">`ttl_override`</a>:
+    The object describes cache TTL override mechanics.  It has the following
+    properties:
+
+     *  <a href="cache-ttl_override-enabled">`enabled`</a>:
+        If true, the TTL overrides are enabled.
+
+     *  <a href="cache-ttl_override-min">`min`</a>:
+        The minimum duration for TTL for cache items of both caches, with and
+        without ECS support.  The recommended value is `60s`.
+
+    **Property example:**
+
+    ```yaml
+    'ttl_override':
+        'enabled': true
+        'min': 60s
+    ```
+
 
 
 ##  <a href="#upstream" id="upstream" name="upstream">Upstream</a>
@@ -320,6 +340,24 @@ connection to the main upstream as restored, and requests are routed back to it.
 
 
 
+##  <a href="#dnsdb" id="dnsdb" name="dnsdb">DNSDB</a>
+
+The `DNSDB` object has the following properties:
+
+ *  <a href="#dnsdb-enabled" id="dnsdb-enabled" name="dnsdb-enabled">`enabled`</a>:
+    If true, the DNSDB memory buffer is enabled.
+
+    **Example:** `true`.
+
+ *  <a href="#dnsdb-max_size" id="dnsdb-max_size" name="dnsdb-max_size">`max_size`</a>:
+    The maximum number of records in the in-memory buffer.  The record key is a
+    combination of the target hostname from the question and the resource-record
+    type of the question or the answer.
+
+    **Example:** `500000`.
+
+
+
 ##  <a href="#backend" id="backend" name="backend">Backend</a>
 
 The `backend` object has the following properties:
@@ -635,11 +673,6 @@ The `filters` object has the following properties:
 
     **Example:** `1024`.
 
- *  <a href="#filters-rule_list_cache_size" id="filters-rule_list_cache_size" name="filters-rule_list_cache_size">`rule_list_cache_size`</a>:
-    The size of the LRU cache of the rule-list filtering results.
-
-    **Example:** `10000`.
-
  *  <a href="#filters-refresh_interval" id="filters-refresh_interval" name="filters-refresh_interval">`refresh_interval`</a>:
     How often AdGuard DNS refreshes the rule-list filters from the filter index,
     as well as the blocked services list from the [blocked list
@@ -654,12 +687,26 @@ The `filters` object has the following properties:
 
     **Example:** `5m`.
 
- *  <a href="#filters-use_rule_list_cache" id="filters-use_rule_list_cache" name="filters-use_rule_list_cache">`use_rule_list_cache`</a>:
+ *  <a href="#filters-max_size" id="filters-max_size" name="filters-max_size">`max_size`</a>:
-    If true, use the rule-list filtering result cache.  This cache is not used
+    The maximum size of the downloadable content for a rule-list in a
-    for users' custom rules.
+    human-readable format.
+
+    **Example:** `256MB`.
+
+ *  <a href="#filters-rule_list_cache" id="filters-rule_list_cache" name="filters-rule_list_cache">`rule_list_cache`</a>:
+    Rule lists cache settings.  It has the following properties:
+
+     *  <a href="#filters-rule_list_cache-enabled" id="filters-rule_list_cache-enabled" name="filters-rule_list_cache-enabled">`enabled`</a>:
+        If true, use the rule-list filtering result cache.  This cache is not
+        used for users' custom rules.
 
         **Example:** `true`.
 
+     *  <a href="#filters-rule_list_cache-size" id="filters-rule_list_cache-size" name="filters-rule_list_cache-size">`rule_list_cache-size`</a>:
+        The size of the LRU cache of the rule-list filtering results.
+
+        **Example:** `10000`.
+
 [env-blocked_services]: environment.md#BLOCKED_SERVICE_INDEX_URL
 
 
@@ -724,6 +771,16 @@ The items of the `filtering_groups` array have the following properties:
 
         **Example:** `true`.
 
+     *  <a href="#fg-*-sb-block_dangerous_domains" id="fg-*-sb-block_dangerous_domains" name="fg-*-sb-block_dangerous_domains">`block_dangerous_domains`</a>:
+        Shows if the dangerous domains filtering should be enforced.
+
+        **Example:** `true`.
+
+     *  <a href="#fg-*-sb-block_newly_registered_domains" id="fg-*-sb-block_newly_registered_domains" name="fg-*-sb-block_newly_registered_domains">`block_newly_registered_domains`</a>:
+        Shows if the newly registered domains filtering should be enforced.
+
+        **Example:** `true`.
+
  *  <a href="#fg-*-block_private_relay" id="fg-*-block_private_relay" name="fg-*-block_private_relay">`private_relay`</a>:
     If true, Apple Private Relay queries are blocked for requests using this
     filtering group.
@@ -953,10 +1010,12 @@ The items of the `servers` array have the following properties:
 
     ```yaml
     'bind_interfaces':
-      - 'id': eth0_plain_dns'
+      - 'id': 'eth0_plain_dns'
-        'subnet': '172.17.0.0/16'
+        'subnets':
-      - 'id': eth0_plain_dns_secondary'
+          - '172.17.0.0/16'
-        'subnet': '172.17.0.0/16'
+      - 'id': 'eth0_plain_dns_secondary'
+        'subnets':
+          - '172.17.0.0/16'
     ```
 
  *  <a href="#sg-s-*-dnscrypt" id="sg-s-*-dnscrypt" name="sg-s-*-dnscrypt">`dnscrypt`</a>:
@@ -1017,20 +1076,20 @@ The `connectivity_check` object has the following properties:
 The `network` object has the following properties:
 
  *  <a href="#network-so_rcvbuf" id="network-so_rcvbuf" name="network-so_rcvbuf">`so_rcvbuf`</a>:
-    The size of socket receive buffer (`SO_RCVBUF`), in bytes.  Default is zero,
+    The size of socket receive buffer (`SO_RCVBUF`), in a human-readable format.
-    which means use the default system settings.
+    Default is zero, which means use the default system settings.
 
     See also [notes on these parameters](#recommended-buffers).
 
-    **Example:** `1048576`.
+    **Example:** `1MB`.
 
  *  <a href="#network-so_sndbuf" id="network-so_sndbuf" name="network-so_sndbuf">`so_sndbuf`</a>:
-    The size of socket send buffer (`SO_SNDBUF`), in bytes.  Default is zero,
+    The size of socket send buffer (`SO_SNDBUF`), in a human-readable format.
-    which means use the default system settings.
+    Default is zero, which means use the default system settings.
 
     See also [notes on these parameters](#recommended-buffers).
 
-    **Example:** `1048576`.
+    **Example:** `1MB`.
 
 
 

doc/debugdns.md

@@ -28,6 +28,7 @@ example.com.		17597	IN	A	93.184.216.34
 
 ;; ADDITIONAL SECTION:
 client-ip.adguard-dns.com. 10	CH	TXT	"127.0.0.1"
+server-ip.adguard-dns.com. 10	CH	TXT	"94.140.14.14"
 resp.res-type.adguard-dns.com. 10 CH TXT	"normal"
 
 ;; Query time: 26 msec
@@ -49,6 +50,15 @@ In the `ADDITIONAL SECTION`, the following debug information is returned:
     client-ip.adguard-dns.com. 10	CH	TXT	"127.0.0.1"
     ```
 
+ *  <a href="#additional-server-ip" id="additional-server-ip" name="additional-server-ip">`server-ip`</a>:
+    The IP address of the server.  The full name is `server-ip.adguard-dns.com`.
+
+    **Example:**
+
+    ```none
+    server-ip.adguard-dns.com. 10	CH	TXT	"127.0.0.1"
+    ```
+
  *  <a href="#additional-device-id" id="additional-device-id" name="additional-device-id">`device-id`</a>:
     The ID of the device as detected by the server, if any.  The full name is
     `device-id.adguard-dns.com`.

doc/development.md

@@ -216,9 +216,14 @@ curl 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoLit
 
 You'll need to supply the following:
 
- *  [`BACKEND_ENDPOINT`](#env-BACKEND_ENDPOINT)
+ *  [`ADULT_BLOCKING_URL`](#env-ADULT_BLOCKING_URL)
+ *  [`BILLSTAT_URL`](#env-BILLSTAT_URL)
  *  [`CONSUL_ALLOWLIST_URL`](#env-CONSUL_ALLOWLIST_URL)
  *  [`GENERAL_SAFE_SEARCH_URL`](#env-GENERAL_SAFE_SEARCH_URL)
+ *  [`LINKED_IP_TARGET_URL`](#env-LINKED_IP_TARGET_URL)
+ *  [`NEW_REG_DOMAINS_URL`](#env-NEW_REG_DOMAINS_URL)
+ *  [`PROFILES_URL`](#env-PROFILES_URL)
+ *  [`SAFE_BROWSING_URL`](#env-SAFE_BROWSING_URL)
  *  [`YOUTUBE_SAFE_SEARCH_URL`](#env-YOUTUBE_SAFE_SEARCH_URL)
 
 See the [external HTTP API documentation][externalhttp].
@@ -238,18 +243,22 @@ You may also need to remove `probe_ipv6` if your network does not support IPv6.
 
 ```sh
 env \
-    BACKEND_ENDPOINT='PUT BACKEND URL HERE' \
+    ADULT_BLOCKING_URL='https://raw.githubusercontent.com/ameshkov/PersonalFilters/master/adult_test.txt' \
-    BLOCKED_SERVICE_INDEX_URL='https://atropnikov.github.io/HostlistsRegistry/assets/services.json'\
+    BILLSTAT_URL='PUT BILLSTAT API BACKEND URL HERE' \
+    BLOCKED_SERVICE_INDEX_URL='https://adguardteam.github.io/HostlistsRegistry/assets/services.json'\
     CONSUL_ALLOWLIST_URL='PUT CONSUL ALLOWLIST URL HERE' \
     CONFIG_PATH='./config.yaml' \
-    DNSDB_PATH='./test/cache/dnsdb.bolt' \
+    FILTER_INDEX_URL='https://adguardteam.github.io/HostlistsRegistry/assets/filters.json' \
-    FILTER_INDEX_URL='https://atropnikov.github.io/HostlistsRegistry/assets/filters.json' \
     FILTER_CACHE_PATH='./test/cache' \
+    NEW_REG_DOMAINS_URL='PUT NEWLY REGISTERED DOMAINS FILTER URL HERE' \
     PROFILES_CACHE_PATH='./test/profilecache.json' \
+    PROFILES_URL='PUT PROFILES API BACKEND URL HERE' \
+    SAFE_BROWSING_URL='https://raw.githubusercontent.com/ameshkov/PersonalFilters/master/safebrowsing_test.txt' \
     GENERAL_SAFE_SEARCH_URL='https://adguardteam.github.io/HostlistsRegistry/assets/engines_safe_search.txt' \
     GEOIP_ASN_PATH='./test/GeoLite2-ASN-Test.mmdb' \
     GEOIP_COUNTRY_PATH='./test/GeoIP2-City-Test.mmdb' \
     QUERYLOG_PATH='./test/cache/querylog.jsonl' \
+    LINKED_IP_TARGET_URL='PUT LINKED IP TARGET URL HERE' \
     LISTEN_ADDR='127.0.0.1' \
     LISTEN_PORT='8081' \
     RULESTAT_URL='https://testchrome.adtidy.org/api/1.0/rulestats.html' \

doc/environment.md

@@ -6,24 +6,29 @@ sensitive configuration.  All other configuration is stored in the
 
 ##  Contents
 
- *  [`BACKEND_ENDPOINT`](#BACKEND_ENDPOINT)
+ *  [`ADULT_BLOCKING_URL`](#ADULT_BLOCKING_URL)
+ *  [`BILLSTAT_URL`](#BILLSTAT_URL)
  *  [`BLOCKED_SERVICE_INDEX_URL`](#BLOCKED_SERVICE_INDEX_URL)
  *  [`CONSUL_ALLOWLIST_URL`](#CONSUL_ALLOWLIST_URL)
  *  [`CONSUL_DNSCHECK_KV_URL`](#CONSUL_DNSCHECK_KV_URL)
  *  [`CONSUL_DNSCHECK_SESSION_URL`](#CONSUL_DNSCHECK_SESSION_URL)
  *  [`CONFIG_PATH`](#CONFIG_PATH)
- *  [`DNSDB_PATH`](#DNSDB_PATH)
  *  [`FILTER_INDEX_URL`](#FILTER_INDEX_URL)
  *  [`FILTER_CACHE_PATH`](#FILTER_CACHE_PATH)
- *  [`PROFILES_CACHE_PATH`](#PROFILES_CACHE_PATH)
  *  [`GENERAL_SAFE_SEARCH_URL`](#GENERAL_SAFE_SEARCH_URL)
  *  [`GEOIP_ASN_PATH` and `GEOIP_COUNTRY_PATH`](#GEOIP_ASN_PATH)
+ *  [`LINKED_IP_TARGET_URL`](#LINKED_IP_TARGET_URL)
  *  [`LISTEN_ADDR`](#LISTEN_ADDR)
  *  [`LISTEN_PORT`](#LISTEN_PORT)
  *  [`LOG_TIMESTAMP`](#LOG_TIMESTAMP)
+ *  [`NEW_REG_DOMAINS_URL`](#NEW_REG_DOMAINS_URL)
+ *  [`PROFILES_CACHE_PATH`](#PROFILES_CACHE_PATH)
+ *  [`PROFILES_URL`](#PROFILES_URL)
  *  [`QUERYLOG_PATH`](#QUERYLOG_PATH)
  *  [`RESEARCH_METRICS`](#RESEARCH_METRICS)
+ *  [`RESEARCH_LOGS`](#RESEARCH_LOGS)
  *  [`RULESTAT_URL`](#RULESTAT_URL)
+ *  [`SAFE_BROWSING_URL`](#SAFE_BROWSING_URL)
  *  [`SENTRY_DSN`](#SENTRY_DSN)
  *  [`SSL_KEY_LOG_FILE`](#SSL_KEY_LOG_FILE)
  *  [`VERBOSE`](#VERBOSE)
@@ -34,16 +39,26 @@ sensitive configuration.  All other configuration is stored in the
 
 
 
-##  <a href="#BACKEND_ENDPOINT" id="BACKEND_ENDPOINT" name="BACKEND_ENDPOINT">`BACKEND_ENDPOINT`</a>
+##  <a href="#ADULT_BLOCKING_URL" id="ADULT_BLOCKING_URL" name="ADULT_BLOCKING_URL">`ADULT_BLOCKING_URL`</a>
 
-The base backend URL to which API paths are appended.  The backend endpoints
+The URL of source list of rules for adult blocking filter.
-apart from the `/ddns/`and `/linkip/` ones must reply with a 200 status code on
-success.
 
 **Default:** No default value, the variable is **required.**
 
 
 
+##  <a href="#BILLSTAT_URL" id="BILLSTAT_URL" name="BILLSTAT_URL">`BILLSTAT_URL`</a>
+
+The base backend URL for backend billing statistics uploader API.  The backend
+endpoints must reply with a 200 status code on success.  See the [external HTTP
+API requirements section][ext-billstat]
+
+**Default:** No default value, the variable is **required.**
+
+[ext-billstat]: externalhttp.md#backend-billstat
+
+
+
 ##  <a href="#BLOCKED_SERVICE_INDEX_URL" id="BLOCKED_SERVICE_INDEX_URL" name="BLOCKED_SERVICE_INDEX_URL">`BLOCKED_SERVICE_INDEX_URL`</a>
 
 The URL of the blocked service index file server.  See the [external HTTP API
@@ -101,57 +116,15 @@ for the DNS server checking.  If not specified, the
 
 
 
-##  <a href="#DNSDB_PATH" id="DNSDB_PATH" name="DNSDB_PATH">`DNSDB_PATH`</a>
-
-The path to the DNSDB BoltDB database.  If empty or unset, DNSDB statistics
-collection is disabled.
-
-**Default:** **Unset.**
-
-**Example:** `./dnsdb.bolt`.
-
-
-
 ##  <a href="#FILTER_CACHE_PATH" id="FILTER_CACHE_PATH" name="FILTER_CACHE_PATH">`FILTER_CACHE_PATH`</a>
 
-The path to the directory with the filter lists cache.
+The path to the directory used to store the cached version of all filters and
+filter indexes.
 
 **Default:** `./filters/`.
 
 
 
-##  <a href="#PROFILES_CACHE_PATH" id="PROFILES_CACHE_PATH" name="PROFILES_CACHE_PATH">`PROFILES_CACHE_PATH`</a>
-
-The path to the profile cache file:
-
- *  `none` means that the profile caching is disabled.
-
- *  A file with the extension `.pb` means that the profiles are cached in the
-    protobuf format.
-
-    Use the following command to inspect the cache, assuming that the version is
-    correct:
-
-    ```sh
-    protoc\
-        --decode\
-        profiledb.FileCache\
-        ./internal/profiledb/internal/filecachepb/filecache.proto\
-        < /path/to/profilecache.pb
-    ```
-
- *  A file with the extension `.json` means that the profiles are cached in the
-    JSON format.  This format is **deprecated** and is not recommended.
-
-The profile cache is read on start and is later updated on every
-[full refresh][conf-backend-full_refresh_interval].
-
-**Default:** `./profilecache.json`.
-
-[conf-backend-full_refresh_interval]: configuration.md#backend-full_refresh_interval
-
-
-
 ##  <a href="#FILTER_INDEX_URL" id="FILTER_INDEX_URL" name="FILTER_INDEX_URL">`FILTER_INDEX_URL`</a>
 
 The URL of the filtering rule index file server.  See the [external HTTP API
@@ -184,12 +157,16 @@ countries and continents respectively.
 
 
 
-##  <a href="#LOG_TIMESTAMP" id="LOG_TIMESTAMP" name="LOG_TIMESTAMP">`LOG_TIMESTAMP`</a>
+##  <a href="#LINKED_IP_TARGET_URL" id="LINKED_IP_TARGET_URL" name="LINKED_IP_TARGET_URL">`LINKED_IP_TARGET_URL`</a>
 
-If `1`, show timestamps in the plain text logs.  If `0`, don't show the
+The target URL to which linked IP API requests are proxied.  In case [linked IP
-timestamps.
+and dynamic DNS][conf-web-linked_ip] web server is configured, the variable is
+required.  See the [external HTTP API requirements section][ext-linked_ip].
 
-**Default:** `1`.
+**Default:** **Unset.**
+
+[conf-web-linked_ip]: configuration.md#web-linked_ip
+[ext-linked_ip]: externalhttp.md#backend-linkip
 
 
 
@@ -212,6 +189,68 @@ health check, Prometheus, `pprof`, and other endpoints.
 
 
 
+##  <a href="#LOG_TIMESTAMP" id="LOG_TIMESTAMP" name="LOG_TIMESTAMP">`LOG_TIMESTAMP`</a>
+
+If `1`, show timestamps in the plain text logs.  If `0`, don't show the
+timestamps.
+
+**Default:** `1`.
+
+
+
+##  <a href="#NEW_REG_DOMAINS_URL" id="NEW_REG_DOMAINS_URL" name="NEW_REG_DOMAINS_URL">`NEW_REG_DOMAINS_URL`</a>
+
+The URL of source list of rules for newly registered domains safe browsing
+filter.
+
+**Default:** No default value, the variable is **required.**
+
+
+
+##  <a href="#PROFILES_CACHE_PATH" id="PROFILES_CACHE_PATH" name="PROFILES_CACHE_PATH">`PROFILES_CACHE_PATH`</a>
+
+The path to the profile cache file:
+
+*  `none` means that the profile caching is disabled.
+
+*  A file with the extension `.pb` means that the profiles are cached in the
+   protobuf format.
+
+   Use the following command to inspect the cache, assuming that the version is
+   correct:
+
+   ```sh
+   protoc\
+       --decode\
+       profiledb.FileCache\
+       ./internal/profiledb/internal/filecachepb/filecache.proto\
+       < /path/to/profilecache.pb
+   ```
+
+*  A file with the extension `.json` means that the profiles are cached in the
+   JSON format.  This format is **deprecated** and is not recommended.
+
+The profile cache is read on start and is later updated on every
+[full refresh][conf-backend-full_refresh_interval].
+
+**Default:** `./profilecache.json`.
+
+[conf-backend-full_refresh_interval]: configuration.md#backend-full_refresh_interval
+
+
+
+##  <a href="#PROFILES_URL" id="PROFILES_URL" name="PROFILES_URL">`PROFILES_URL`</a>
+
+The base backend URL for profiles API.  The backend endpoints must reply with a
+200 status code on success.  See the [external HTTP API requirements
+section][ext-profiles].
+
+**Default:** No default value, the variable is **required.**
+
+[ext-profiles]: externalhttp.md#profiles-backend
+
+
+
 ##  <a href="#QUERYLOG_PATH" id="QUERYLOG_PATH" name="QUERYLOG_PATH">`QUERYLOG_PATH`</a>
 
 The path to the file into which the query log is going to be written.
@@ -229,6 +268,16 @@ If `1`, enable collection of a set of special prometheus metrics (prefix is
 
 
 
+##  <a href="#RESEARCH_LOGS" id="RESEARCH_LOGS" name="RESEARCH_LOGS">`RESEARCH_LOGS`</a>
+
+If `1`, enable logging of additional info that may be required for research
+purposes (prefix `research:`).  The log will only be written when
+`RESEARCH_METRICS` is also set to `1`.  If `0`, disable logging of this info.
+
+**Default:** `0`.
+
+
+
 ##  <a href="#RULESTAT_URL" id="RULESTAT_URL" name="RULESTAT_URL">`RULESTAT_URL`</a>
 
 The URL to send filtering rule list statistics to.  If empty or unset, the
@@ -243,6 +292,14 @@ requirements section][ext-rulestat] on the expected format of the response.
 
 
 
+##  <a href="#SAFE_BROWSING_URL" id="SAFE_BROWSING_URL" name="SAFE_BROWSING_URL">`SAFE_BROWSING_URL`</a>
+
+The URL of source list of rules for dangerous domains safe browsing filter.
+
+**Default:** No default value, the variable is **required.**
+
+
+
 ##  <a href="#SENTRY_DSN" id="SENTRY_DSN" name="SENTRY_DSN">`SENTRY_DSN`</a>
 
 Sentry error collector address.  The special value `stderr` makes AdGuard DNS

doc/externalhttp.md

@@ -14,30 +14,50 @@ document should set the `Server` header in their replies.
 
 ##  Contents
 
- *  [Backend And Linked IP Service](#backend)
+ *  [Backend Billing Statistics](#backend-billstat)
-     *  [`GET /dns_api/v1/settings`](#backend-get-v1-settings)
+ *  [Backend Profiles Service](#backend-profiles)
-     *  [`POST /dns_api/v1/settings`](#backend-post-v1-devices_activity)
-     *  [Proxied Linked IP and Dynamic DNS (DDNS) Endpoints](#backend-linkip)
  *  [Consul Key-Value Storage](#consul)
  *  [Filtering](#filters)
      *  [Blocked Services](#filters-blocked-services)
      *  [Filtering Rule Lists](#filters-lists)
      *  [Safe Search](#filters-safe-search)
+ *  [Proxied Linked IP and Dynamic DNS (DDNS) Endpoints](#backend-linkip)
  *  [Rule Statistics Service](#rulestat)
 
 
 
-##  <a href="#backend" id="backend" name="backend">Backend And Linked IP Service</a>
+##  <a href="#backend-billstat" id="backend-billstat" name="backend-billstat">Backend Billing Statistics</a>
 
-This is the service to which the [`BACKEND_ENDPOINT`][env-backend] environment
+This is the service to which the [`BILLSTAT_URL`][env-billstat_url] environment
-variable points.  This service must provide two endpoints:
+variable points.  This service must provide one endpoint:
+`POST /dns_api/v1/devices_activity`, it must respond with a `200 OK` response
+code and accept a JSON document in the following format:
+
+```json
+{
+  "devices": [
+    {
+      "client_country": "AU",
+      "device_id": "abcd1234",
+      "time_ms": 1624443079309,
+      "asn": 1234,
+      "queries": 1000,
+      "proto": 1
+    }
+  ]
+}
+```
+
+[env-billstat_url]: environment.md#BILLSTAT_URL
 
 
 
-   ###  <a href="#backend-get-v1-settings" id="backend-get-v1-settings" name="backend-get-v1-settings">`GET /dns_api/v1/settings`</a>
+##  <a href="#backend-profiles" id="backend-profiles" name="backend-profiles">Backend Profiles Service</a>
 
-This endpoint must respond with a `200 OK` response code and a JSON document in
+This is the service to which the [`PROFILES_URL`][env-profiles_url] environment
-the following format:
+variable points.  This service must provide one endpoint:
+`GET /dns_api/v1/settings`, it must respond with a `200 OK` response code and
+accept a JSON document in the following format:
 
 ```json
 {
@@ -108,43 +128,7 @@ the following format:
 }
 ```
 
-
+[env-profiles_url]: environment.md#PROFILES_URL
-
-   ###  <a href="#backend-post-v1-devices_activity" id="backend-post-v1-devices_activity" name="backend-post-v1-devices_activity">`POST /dns_api/v1/devices_activity`</a>
-
-This endpoint must respond with a `200 OK` response code and accept a JSON
-document in the following format:
-
-```json
-{
-  "devices": [
-    {
-      "client_country": "AU",
-      "device_id": "abcd1234",
-      "time_ms": 1624443079309,
-      "asn": 1234,
-      "queries": 1000,
-      "proto": 1
-    }
-  ]
-}
-```
-
-
-   ###  <a href="#backend-linkip" id="backend-linkip" name="backend-linkip">Proxied Linked IP and Dynamic DNS (DDNS) Endpoints</a>
-
-The same service defined by the [`BACKEND_ENDPOINT`][env-backend] environment
-variable should define the following endpoints:
-
- *  `GET  /linkip/{device_id}/{encrypted}/status`;
- *  `GET  /linkip/{device_id}/{encrypted}`;
- *  `POST /ddns/{device_id}/{encrypted}/{domain}`;
- *  `POST /linkip/{device_id}/{encrypted}`.
-
-The AdGuard DNS proxy will add the `CF-Connecting-IP` header with the IP address
-of the original client as well as set the `User-Agent` header to its own value.
-
-[env-backend]: environment.md#BACKEND_ENDPOINT
 
 
 
@@ -192,7 +176,7 @@ format:
       "id": "my_filter",
       "rules": [
         "||example.com^",
-        "||example.net^",
+        "||example.net^"
       ]
     }
   ]
@@ -252,6 +236,23 @@ code and filtering rule lists with [`$dnsrewrite`][rules-dnsrewrite] rules for
 
 
 
+##  <a href="#backend-linkip" id="backend-linkip" name="backend-linkip">Proxied Linked IP and Dynamic DNS (DDNS) Endpoints</a>
+
+The service defined by the [`LINKED_IP_TARGET_URL`][env-linked_ip_target_url]
+environment variable should define the following endpoints:
+
+ *  `GET  /linkip/{device_id}/{encrypted}/status`;
+ *  `GET  /linkip/{device_id}/{encrypted}`;
+ *  `POST /ddns/{device_id}/{encrypted}/{domain}`;
+ *  `POST /linkip/{device_id}/{encrypted}`.
+
+The AdGuard DNS proxy will add the `CF-Connecting-IP` header with the IP address
+of the original client as well as set the `User-Agent` header to its own value.
+
+[env-linked_ip_target_url]: environment.md#LINKED_IP_TARGET_URL
+
+
+
 ##  <a href="#rulestat" id="rulestat" name="rulestat">Rule Statistics Service</a>
 
 This endpoint, defined by [`RULESTAT_URL`][env-rulestat], must respond with a
@@ -263,7 +264,7 @@ This endpoint, defined by [`RULESTAT_URL`][env-rulestat], must respond with a
     {
       "15": {
         "||example.com^": 1234,
-        "||example.net^": 5678,
+        "||example.net^": 5678
       }
     }
   ]

doc/http.md

@@ -110,7 +110,7 @@ The `protocol` field can have one of the following values:
 
 The linked IP and Dynamic DNS (DDNS, DynDNS) HTTP proxy.  If the [linked
 IP configuration][conf-web-linked_ip] is not empty, the following queries are
-either processed or proxied to [`BACKEND_ENDPOINT`][env-backend].
+either processed or proxied to [`LINKED_IP_TARGET_URL`][env-linked_ip_target_url].
 
  *  `GET  /robots.txt`: a special response is served, see below;
  *  `GET  /linkip/{device_id}/{encrypted}/status`: proxied;
@@ -128,7 +128,7 @@ Disallow: /
 The [static content](#static-content) is not served on the linked IP addresses.
 
 [conf-web-linked_ip]: configuration.md#web-linked_ip
-[env-backend]:        environment.md#BACKEND_ENDPOINT
+[env-linked_ip_target_url]: environment.md#LINKED_IP_TARGET_URL
 
 
 

go.mod

@@ -4,27 +4,26 @@ go 1.20
 
 require (
 	github.com/AdguardTeam/AdGuardDNS/internal/dnsserver v0.100.0
-	github.com/AdguardTeam/golibs v0.13.2
+	github.com/AdguardTeam/golibs v0.13.6
-	github.com/AdguardTeam/urlfilter v0.16.1
+	github.com/AdguardTeam/urlfilter v0.16.2
-	github.com/ameshkov/dnscrypt/v2 v2.2.5
+	github.com/ameshkov/dnscrypt/v2 v2.2.7
 	github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc
 	github.com/bluele/gcache v0.0.2
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
 	github.com/caarlos0/env/v7 v7.1.0
-	github.com/getsentry/sentry-go v0.19.0
+	github.com/getsentry/sentry-go v0.21.0
-	github.com/google/renameio v1.0.1
+	github.com/google/renameio/v2 v2.0.0
-	github.com/miekg/dns v1.1.52
+	github.com/miekg/dns v1.1.55
 	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.15.1
-	github.com/prometheus/client_model v0.3.0
+	github.com/prometheus/client_model v0.4.0
-	github.com/prometheus/common v0.41.0
+	github.com/prometheus/common v0.44.0
 	github.com/quic-go/quic-go v0.35.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
-	go.etcd.io/bbolt v1.3.7
+	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
+	golang.org/x/net v0.12.0
-	golang.org/x/net v0.8.0
+	golang.org/x/sys v0.10.0
-	golang.org/x/sys v0.6.0
 	golang.org/x/time v0.3.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -38,22 +37,22 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 // indirect
+	github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/onsi/ginkgo/v2 v2.9.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.10.0 // indirect
-	github.com/panjf2000/ants/v2 v2.7.1 // indirect
+	github.com/panjf2000/ants/v2 v2.7.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
 	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.10.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

go.sum

@@ -1,18 +1,18 @@
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
-github.com/AdguardTeam/golibs v0.13.2 h1:BPASsyQKmb+b8VnvsNOHp7bKfcZl9Z+Z2UhPjOiupSc=
+github.com/AdguardTeam/golibs v0.13.6 h1:z/0Q25pRLdaQxtoxvfSaooz5mdv8wj0R8KREj54q8yQ=
-github.com/AdguardTeam/golibs v0.13.2/go.mod h1:7ylQLv2Lqsc3UW3jHoITynYk6Y1tYtgEMkR09ppfsN8=
+github.com/AdguardTeam/golibs v0.13.6/go.mod h1:hOtcb8dPfKcFjWTPA904hTA4dl1aWvzeebdJpE72IPk=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
-github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
+github.com/AdguardTeam/urlfilter v0.16.2 h1:k9m9dUYVJ3sTswYa2/ukVNjicfGcz0oqFDO13hPmfHE=
-github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=
+github.com/AdguardTeam/urlfilter v0.16.2/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=
 github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
 github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw=
 github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us=
-github.com/ameshkov/dnscrypt/v2 v2.2.5 h1:Ju1gQeez+6XLtk/b/k3RoJ2t+Ls+BSItLTZjZeedneY=
+github.com/ameshkov/dnscrypt/v2 v2.2.7 h1:aEitLIR8HcxVodZ79mgRcCiC0A0I5kZPBuWGFwwulAw=
-github.com/ameshkov/dnscrypt/v2 v2.2.5/go.mod h1:Cu5GgMvCR10BeXgACiGDwXyOpfMktsSIidml1XBp6uM=
+github.com/ameshkov/dnscrypt/v2 v2.2.7/go.mod h1:qPWhwz6FdSmuK7W4sMyvogrez4MWdtzosdqlr0Rg3ow=
 github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo=
 github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
 github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODUaHcEi7ltENhbAK2VgZjfat6mLy03tQzo=
@@ -34,29 +34,28 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
 github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 h1:y7y0Oa6UawqTFPCDw9JG6pdKt4F9pAhHv0B7FMGaGD0=
 github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
-github.com/getsentry/sentry-go v0.19.0 h1:BcCH3CN5tXt5aML+gwmbFwVptLLQA+eT866fCO9wVOM=
+github.com/getsentry/sentry-go v0.21.0 h1:c9l5F1nPF30JIppulk4veau90PK6Smu3abgVtVQWon4=
-github.com/getsentry/sentry-go v0.19.0/go.mod h1:y3+lGEFEFexZtpbG1GUE2WD/f9zGyKYwpEqryTOC/nE=
+github.com/getsentry/sentry-go v0.21.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 h1:CqYfpuYIjnlNxM3msdyPRKabhXZWbKjf3Q8BWROFBso=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
-github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
-github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU=
+github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
-github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
+github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -64,16 +63,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
-github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c=
+github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
-github.com/miekg/dns v1.1.52/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
+github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.9.0 h1:Tugw2BKlNHTMfG+CheOITkYvk4LAh6MFOvikhGVnhE8=
+github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs=
-github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
+github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE=
-github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
+github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
 github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg=
 github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0=
-github.com/panjf2000/ants/v2 v2.7.1 h1:qBy5lfSdbxvrR0yUnZfaEDjf0FlCw4ufsbcsxmE7r+M=
+github.com/panjf2000/ants/v2 v2.7.5 h1:/vhh0Hza9G1vP1PdCj9hl6MUzCRbmtcTJL0OsnmytuU=
-github.com/panjf2000/ants/v2 v2.7.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
+github.com/panjf2000/ants/v2 v2.7.5/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
 github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible h1:IWzUvJ72xMjmrjR9q3H1PF+jwdN0uNQiR2t1BLNalyo=
 github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -81,14 +80,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.41.0 h1:npo01n6vUlRViIj5fgwiK8vlNIh8bnoxqh3gypKsyAw=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.41.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
 github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
@@ -97,46 +96,46 @@ github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8G
 github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
 github.com/quic-go/quic-go v0.35.1 h1:b0kzj6b/cQAf05cT0CkQubHM31wiA+xH3IBkxP62poo=
 github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA=
 github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
+golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
-golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -147,23 +146,23 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
+golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go.work.sum

@@ -4,6 +4,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
 cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
 cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA=
+cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU=
 cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
@@ -23,17 +24,36 @@ github.com/AdguardTeam/gomitmproxy v0.2.0 h1:rvCOf17pd1/CnMyMQW891zrEiIQBpQ8cIGj
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c=
+github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet/v6 v6.1.0 h1:hvO96X345XagdH1fAoBjpBYG4a1ghhL/QzalkduPuXk=
+github.com/CloudyKit/jet/v6 v6.1.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4=
+github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME=
+github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4=
+github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk=
+github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP3TjA/dfr4NAY8bghDwnXiU7cTKxQqo0=
+github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM=
+github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
+github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU=
+github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625 h1:ckJgFhFWywOx+YLEMIJsTb+NV6NexWICk5+AMSuz3ss=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
@@ -49,6 +69,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI=
 github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
 github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -58,16 +79,20 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
+github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw=
+github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
@@ -79,23 +104,35 @@ github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
+github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I=
 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw=
+github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
+github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk=
+github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
+github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
+github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
+github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
@@ -107,17 +144,19 @@ github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200j
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
@@ -125,6 +164,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99 h1:Ak8CrdlwwXwAZxzS66vgPt4U8yUZX7JwLvVR58FN5jM=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
@@ -132,6 +172,7 @@ github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI=
@@ -144,47 +185,81 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g=
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/ianlancetaylor/demangle v0.0.0-20220517205856-0058ec4f073c/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab h1:BA4a7pe6ZTd9F8kXETBoijjFJ/ntaa//1wiH9BZu4zU=
+github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/influxdata/influxdb v1.7.6 h1:8mQ7A/V+3noMGCt/P9pD09ISaiz9XvgCk303UYA3gcs=
 github.com/iris-contrib/jade v1.1.4 h1:WoYdfyJFfZIUgqNAeOyRfTNQZOksSlZ6+FnXR3AEpX0=
+github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5UybogqrXvJTBE=
 github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw=
+github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1 h1:ujPKutqRlJtcfWk6toYVYagwra7HQHbXOaS171b4Tg8=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kataras/blocks v0.0.7 h1:cF3RDY/vxnSRezc7vLFlQFTYXG/yAr1o7WImJuZbzC4=
+github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I=
 github.com/kataras/golog v0.1.7 h1:0TY5tHn5L5DlRIikepcaRR/6oInIr9AiWsxzt0vvlBE=
+github.com/kataras/golog v0.1.7/go.mod h1:jOSQ+C5fUqsNSwurB/oAHq1IFSb0KI3l6GMa7xB6dZA=
+github.com/kataras/golog v0.1.8 h1:isP8th4PJH2SrbkciKnylaND9xoTtfxv++NB+DF0l9g=
+github.com/kataras/golog v0.1.8/go.mod h1:rGPAin4hYROfk1qT9wZP6VY2rsb4zzc37QpdPjdkqVw=
 github.com/kataras/iris/v12 v12.2.0-beta5 h1:grB/oCf5baZhmYIeDMfgN3LYrtEcmK8pbxlRvEZ2pgw=
+github.com/kataras/iris/v12 v12.2.0-beta5/go.mod h1:q26aoWJ0Knx/00iPKg5iizDK7oQQSPjbD8np0XDh6dc=
+github.com/kataras/iris/v12 v12.2.0 h1:WzDY5nGuW/LgVaFS5BtTkW3crdSKJ/FEgWnxPnIVVLI=
+github.com/kataras/iris/v12 v12.2.0/go.mod h1:BLzBpEunc41GbE68OUaQlqX4jzi791mx5HU04uPb90Y=
 github.com/kataras/pio v0.0.11 h1:kqreJ5KOEXGMwHAWHDwIl+mjfNCPhAwZPa8gK7MKlyw=
+github.com/kataras/pio v0.0.11/go.mod h1:38hH6SWH6m4DKSYmRhlrCJ5WItwWgCVrTNU62XZyUvI=
 github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY=
+github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4=
 github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA=
+github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
+github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
+github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
 github.com/kr/pty v1.1.3 h1:/Um6a/ZmD5tF7peoOJ5oN5KMQ0DrGVQSXLNwyckutPk=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/labstack/echo/v4 v4.9.0 h1:wPOF1CE6gvt/kmbMR4dGzWvHMPT+sAEUJOwOTtvITVY=
+github.com/labstack/echo/v4 v4.9.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks=
+github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280ZKA=
+github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ=
 github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o=
+github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
+github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
+github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
 github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
 github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/mailgun/raymond/v2 v2.0.46 h1:aOYHhvTpF5USySJ0o7cpPno/Uh2I5qg2115K25A+Ft4=
+github.com/mailgun/raymond/v2 v2.0.46/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18=
+github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw=
+github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe h1:W/GaMY0y69G4cFlmsC6B9sbuo2fP8OFP1ABjt4kPz+w=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
 github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
 github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
@@ -193,40 +268,60 @@ github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4
 github.com/marten-seemann/qtls-go1-18 v0.1.0/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
 github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1 h1:SIYunPjnlXcW+gVfvm0IlSeR5U3WZUOLfVmqg85Go44=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
+github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
+github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
+github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
 github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86 h1:D6paGObi5Wud7xg83MaEFyjxQB1W5bz5d0IFppr+ymk=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab h1:eFXv9Nu1lGbrNbj619aWwZfVF5HBrm9Plte8aNptuTI=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
 github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc=
 github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
 github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
+github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
 github.com/openzipkin/zipkin-go v0.1.1 h1:A/ADD6HaPnAKj3yS7HjGHRK77qi41Hi0DirOOIQAeIw=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=
+github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk=
+github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4 h1:Fth6mevc5rX7glNLpbAMJnqKlfIkcTjZCSHEeqvKbcI=
@@ -275,6 +370,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133 h1:JtcyT0rk/9PKO
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9Ev6lojP2XaIshpT4ymkqhMeSghO5Ps00E=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e h1:qpG93cPwA5f7s/ZPBJnGOYQNK/vKsaDaseuKT5Asee8=
@@ -284,25 +380,43 @@ github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07 h1:UyzmZLoiDWMRywV4DUYb9Fbt8uiOSooupjTq10vpvnU=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tdewolff/minify/v2 v2.12.4 h1:kejsHQMM17n6/gwdw53qsi6lg0TGddZADVyQOz1KMdE=
+github.com/tdewolff/minify/v2 v2.12.4/go.mod h1:h+SRvSIX3kwgwTFOpSckvSxgax3uy8kZTSF1Ojrr3bk=
 github.com/tdewolff/parse/v2 v2.6.4 h1:KCkDvNUMof10e3QExio9OPZJT8SbdKojLBumw8YZycQ=
+github.com/tdewolff/parse/v2 v2.6.4/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
+github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc=
+github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
+github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/viant/assertly v0.4.8 h1:5x1GzBaRteIwTr5RAGFVG14uNeRFxVNbXPWrK2qAgpc=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0 h1:6TteTDQ68CjgcCe8wH3D3ZhUQQOJXMTbj/D9rkk2a1k=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
+github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
+github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA=
+github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0=
 github.com/yuin/goldmark v1.4.1 h1:/vn0k+RBvwlxEmP5E7SZMqNxPhfMVFEJiykr15/0XKM=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go4.org v0.0.0-20180809161055-417644f6feb5 h1:+hE86LblG4AyDgwMCLTE6FOlM9+qjHSYS+rKqxUVdsM=
@@ -326,8 +440,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
-golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -336,42 +449,49 @@ golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220516155154-20f960328961/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg=
+golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
+golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
+golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852 h1:xYq6+9AtI+xP3M4r0N1hCkHrInHDBohhquRgx9Kk6gI=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
 golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -379,9 +499,7 @@ golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
@@ -393,6 +511,8 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -414,6 +534,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919 h1:tmXTu+dfa+d9Evp8NpJdgOy6+rt8/x4yG7qPBrtNfLY=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

internal/agd/agd.go

@@ -2,39 +2,11 @@
 package agd
 
 import (
-	"crypto/rand"
-	"encoding/base64"
 	"fmt"
 )
 
 // Common Constants, Types, And Utilities
 
-// RequestID is the ID of a request.  It is an opaque, randomly generated
-// string.  API users should not rely on it being pseudorandom or
-// cryptographically random.
-type RequestID string
-
-// NewRequestID returns a new pseudorandom RequestID.  Prefer this to manual
-// conversion from other string types.
-func NewRequestID() (id RequestID) {
-	// Generate a random 16-byte (128-bit) number, encode it into a URL-safe
-	// Base64 string, and return it.
-	const N = 16
-
-	var idData [N]byte
-	_, err := rand.Read(idData[:])
-	if err != nil {
-		panic(fmt.Errorf("generating random request id: %w", err))
-	}
-
-	enc := base64.URLEncoding.WithPadding(base64.NoPadding)
-	n := enc.EncodedLen(N)
-	idData64 := make([]byte, n)
-	enc.Encode(idData64, idData[:])
-
-	return RequestID(idData64)
-}
-
 // unit is a convenient alias for struct{}.
 type unit = struct{}
 

internal/agd/context.go

@@ -50,7 +50,7 @@ func RequestIDFromContext(ctx context.Context) (id RequestID, ok bool) {
 	const key = ctxKeyReqID
 	v := ctx.Value(key)
 	if v == nil {
-		return "", false
+		return RequestID{}, false
 	}
 
 	id, ok = v.(RequestID)
@@ -95,14 +95,14 @@ type RequestInfo struct {
 	// Server is the name of the server which handles this request.
 	Server ServerName
 
-	// ID is the unique ID of the request.  It is resurfaced here to optimize
-	// context lookups.
-	ID RequestID
-
 	// Host is the lowercased, non-FQDN version of the hostname from the
 	// question of the request.
 	Host string
 
+	// ID is the unique ID of the request.  It is resurfaced here to optimize
+	// context lookups.
+	ID RequestID
+
 	// QType is the type of question for this request.
 	QType dnsmsg.RRType
 

internal/agd/country.go

@@ -1065,5 +1065,14 @@ func isUserAssigned(s string) (ok bool) {
 		return false
 	}
 
-	return s == "AA" || s == "OO" || s == "ZZ" || s[0] == 'X' || (s[0] == 'Q' && s[1] >= 'M')
+	if s[0] == 'X' || (s[0] == 'Q' && s[1] >= 'M') {
+		return true
+	}
+
+	switch s {
+	case "AA", "OO", "ZZ":
+		return true
+	default:
+		return false
+	}
 }

internal/agd/country_generate.go

@@ -137,7 +137,16 @@ func isUserAssigned(s string) (ok bool) {
 		return false
 	}
 
-	return s == "AA" || s == "OO" || s == "ZZ" || s[0] == 'X' || (s[0] == 'Q' && s[1] >= 'M')
+	if s[0] == 'X' || (s[0] == 'Q' && s[1] >= 'M') {
+		return true
+	}
+
+	switch s {
+	case "AA", "OO", "ZZ":
+		return true
+	default:
+		return false
+	}
 }
 `
 

internal/agd/filterlist.go

@@ -2,29 +2,11 @@ package agd
 
 import (
 	"fmt"
-	"net/url"
-	"time"
 	"unicode/utf8"
 
 	"github.com/AdguardTeam/golibs/errors"
 )
 
-// Filter Lists
-
-// FilterList is a list of filter rules.
-type FilterList struct {
-	// URL is the URL used to refresh the filter.
-	URL *url.URL
-
-	// ID is the unique ID of this filter.  It will also be used to create the
-	// cache file.
-	ID FilterListID
-
-	// RefreshIvl is the interval that defines how often a filter should be
-	// refreshed.  It is also used to check if the cached file is fresh enough.
-	RefreshIvl time.Duration
-}
-
 // FilterListID is the ID of a filter list.  It is an opaque string.
 type FilterListID string
 
@@ -51,6 +33,10 @@ const (
 	// a request was filtered by the safe browsing filter.
 	FilterListIDSafeBrowsing FilterListID = "safe_browsing"
 
+	// FilterListIDNewRegDomains is the special shared filter list ID used when
+	// a request was filtered by the newly registered domains filter.
+	FilterListIDNewRegDomains FilterListID = "newly_registered_domains"
+
 	// FilterListIDGeneralSafeSearch is the shared filter list ID used when
 	// a request was modified by the general safe search filter.
 	FilterListIDGeneralSafeSearch FilterListID = "general_safe_search"
@@ -135,6 +121,14 @@ type FilteringGroup struct {
 	// should be enforced.
 	SafeBrowsingEnabled bool
 
+	// BlockDangerousDomains shows whether the dangerous domains safe browsing
+	// filtering should be enforced.
+	BlockDangerousDomains bool
+
+	// BlockNewlyRegisteredDomains shows whether the newly registered domains
+	// safe browsing filtering should be enforced.
+	BlockNewlyRegisteredDomains bool
+
 	// GeneralSafeSearch shows whether the general safe search filtering should
 	// be enforced.
 	GeneralSafeSearch bool

internal/agd/profile.go

@@ -30,6 +30,13 @@ type Profile struct {
 	// [internal/profiledb/internal.FileCacheVersion].
 	Parental *ParentalProtectionSettings
 
+	// SafeBrowsing are the safe browsing settings for this profile.  They are
+	// ignored if FilteringEnabled is set to false.
+	//
+	// NOTE: Do not change fields of this structure without incrementing
+	// [internal/profiledb/internal.FileCacheVersion].
+	SafeBrowsing *SafeBrowsingSettings
+
 	// BlockingMode defines the way blocked responses are constructed.
 	//
 	// NOTE: Do not change fields of this structure without incrementing
@@ -85,14 +92,6 @@ type Profile struct {
 	// [internal/profiledb/internal.FileCacheVersion].
 	FilteringEnabled bool
 
-	// SafeBrowsingEnabled defines whether queries from devices of this profile
-	// should be filtered using the safe browsing filter.  Requires
-	// FilteringEnabled to be set to true.
-	//
-	// NOTE: Do not change fields of this structure without incrementing
-	// [internal/profiledb/internal.FileCacheVersion].
-	SafeBrowsingEnabled bool
-
 	// RuleListsEnabled defines whether queries from devices of this profile
 	// should be filtered using the filtering rule lists in RuleListIDs.
 	// Requires FilteringEnabled to be set to true.
@@ -158,9 +157,12 @@ func NewProfileID(s string) (id ProfileID, err error) {
 // DayRange is a range within a single day.  Start and End are minutes from the
 // start of day, with 0 being 00:00:00.(0) and 1439, 23:59:59.(9).
 //
-// Additionally, if both Start and End are set to math.MaxUint16, the range is
+// Additionally, if both Start and End are set to [math.MaxUint16], the range is
-// a special zero-length range.  This is done to reduce the amount of pointers
+// a special zero-length range.  This is needed, because when both Start and End
-// and thus GC time.
+// are zero, such DayRange indicates one minute after midnight; as well as to
+// reduce the amount of pointers and thus GC time.
+//
+// TODO(a.garipov):  Refactor.  See AGDNS-1516.
 type DayRange struct {
 	Start uint16
 	End   uint16
@@ -261,6 +263,25 @@ type ParentalProtectionSettings struct {
 	YoutubeSafeSearch bool
 }
 
+// SafeBrowsingSettings are the safe browsing settings of a profile.
+//
+// NOTE: Do not change fields of this structure without incrementing
+// [internal/profiledb/internal.FileCacheVersion].
+type SafeBrowsingSettings struct {
+	// Enabled defines whether queries from devices of this profile should be
+	// filtered using the safe browsing filter.  This must be true in order for
+	// all parameters below to work.
+	Enabled bool
+
+	// BlockDangerousDomains shows whether the dangerous domains safe browsing
+	// filtering should be enforced.
+	BlockDangerousDomains bool
+
+	// BlockNewlyRegisteredDomains shows whether the newly registered domains
+	// safe browsing filtering should be enforced.
+	BlockNewlyRegisteredDomains bool
+}
+
 // BlockedServiceID is the ID of a blocked service.  While these are usually
 // human-readable, clients should treat them as opaque strings.
 //

internal/agd/requestid.go

@@ -0,0 +1,52 @@
+package agd
+
+import (
+	"encoding/base64"
+	"fmt"
+	"time"
+
+	"golang.org/x/exp/rand"
+)
+
+// RequestIDLen is the length of a [RequestID] in bytes.  A RequestID is
+// currently a random 16-byte (128-bit) number.
+const RequestIDLen = 16
+
+// RequestID is the ID of a request.  It is an opaque, randomly generated
+// string.  API users should not rely on it being pseudorandom or
+// cryptographically random.
+type RequestID [RequestIDLen]byte
+
+// requestIDRand is used to create [RequestID]s.
+//
+// TODO(a.garipov): Consider making a struct instead of using one global source.
+var requestIDRand = rand.New(&rand.LockedSource{})
+
+// InitRequestID initializes the [RequestID] generator.
+func InitRequestID() {
+	requestIDRand.Seed(uint64(time.Now().UnixNano()))
+}
+
+// NewRequestID returns a new pseudorandom RequestID.  Prefer this to manual
+// conversion from other string types.
+func NewRequestID() (id RequestID) {
+	_, err := requestIDRand.Read(id[:])
+	if err != nil {
+		panic(fmt.Errorf("generating random request id: %w", err))
+	}
+
+	return id
+}
+
+// type check
+var _ fmt.Stringer = RequestID{}
+
+// String implements the [fmt.Stringer] interface for RequestID.
+func (id RequestID) String() (s string) {
+	enc := base64.URLEncoding.WithPadding(base64.NoPadding)
+	n := enc.EncodedLen(RequestIDLen)
+	idData64 := make([]byte, n)
+	enc.Encode(idData64, id[:])
+
+	return string(idData64)
+}

internal/agd/requestid_test.go

@@ -0,0 +1,29 @@
+package agd_test
+
+import (
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
+	"github.com/stretchr/testify/require"
+)
+
+var reqIDSink agd.RequestID
+
+func BenchmarkNewRequestID(b *testing.B) {
+	agd.InitRequestID()
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		reqIDSink = agd.NewRequestID()
+	}
+
+	require.NotEmpty(b, reqIDSink)
+
+	// Most recent result, on a ThinkPad X13 with a Ryzen Pro 7 CPU:
+	//	goos: linux
+	//	goarch: amd64
+	//	pkg: github.com/AdguardTeam/AdGuardDNS/internal/agd
+	//	cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics
+	//	BenchmarkNewRequestID-16        50985721                24.91 ns/op            0 B/op          0 allocs/op
+}

internal/agdhttp/client.go

@@ -91,7 +91,7 @@ func (c *Client) do(
 
 	reqID, ok := agd.RequestIDFromContext(ctx)
 	if ok {
-		req.Header.Set(httphdr.XRequestID, string(reqID))
+		req.Header.Set(httphdr.XRequestID, reqID.String())
 	}
 
 	req.Header.Set(httphdr.UserAgent, c.userAgent)

internal/agdmaps/agdmaps.go

@@ -1,40 +0,0 @@
-// Package agdmaps contains utilities for map handling.
-package agdmaps
-
-import (
-	"golang.org/x/exp/constraints"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
-)
-
-// OrderedRange is like the usual Go range but sorts the keys before iterating
-// ensuring a predictable order.  If cont is false, OrderedRange stops the
-// iteration.
-func OrderedRange[K constraints.Ordered, V any, M ~map[K]V](m M, f func(k K, v V) (cont bool)) {
-	keys := maps.Keys(m)
-	slices.Sort(keys)
-	for _, k := range keys {
-		if !f(k, m[k]) {
-			break
-		}
-	}
-}
-
-// OrderedRangeError is like [OrderedRange] but uses an error to signal that the
-// iteration must be stopped.  err is the same error as the one returned from f,
-// or nil if no errors are returned.
-func OrderedRangeError[K constraints.Ordered, V any, M ~map[K]V](
-	m M,
-	f func(k K, v V) (err error),
-) (err error) {
-	keys := maps.Keys(m)
-	slices.Sort(keys)
-	for _, k := range keys {
-		err = f(k, m[k])
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

internal/agdnet/resolver.go

@@ -124,18 +124,9 @@ func (c *CachingResolver) resolve(
 	refrTime := time.Now()
 
 	// Don't resolve IP addresses.
-	ip := net.ParseIP(host)
+	ip := ipFromHost(host, fam)
 	if ip != nil {
-		ip4 := ip.To4()
-		if fam == netutil.AddrFamilyIPv4 && ip4 != nil {
-			ips = []net.IP{ip4}
-		} else if fam == netutil.AddrFamilyIPv6 && ip4 == nil {
 		ips = []net.IP{ip}
-		} else {
-			// Not the right kind of IP address.  Cache absence of IP addresses
-			// for this network forever.
-			ips = []net.IP{}
-		}
 
 		// Set the refresh time to the maximum date that time.Duration allows to
 		// prevent this item from refreshing.
@@ -168,6 +159,24 @@ func (c *CachingResolver) resolve(
 	return item, nil
 }
 
+// ipFromHost returns a normalized IP address if host contains an IP address of
+// the given address family.
+func ipFromHost(host string, fam netutil.AddrFamily) (ip net.IP) {
+	ip = net.ParseIP(host)
+	if ip == nil {
+		return nil
+	}
+
+	ip4 := ip.To4()
+	if fam == netutil.AddrFamilyIPv4 && ip4 != nil {
+		return ip4
+	} else if fam == netutil.AddrFamilyIPv6 && ip4 == nil {
+		return ip
+	}
+
+	return nil
+}
+
 // isExpectedLookupError returns true if the error is an expected lookup error.
 func isExpectedLookupError(fam netutil.AddrFamily, err error) (ok bool) {
 	var dnsErr *net.DNSError

internal/agdtest/interface.go

@@ -26,141 +26,6 @@ import (
 //
 // Keep entities within a module/package in alphabetic order.
 
-// Module std
-
-// Package net
-//
-// TODO(a.garipov): Move these to golibs?
-
-// type check
-var _ net.Conn = (*Conn)(nil)
-
-// Conn is the [net.Conn] for tests.
-type Conn struct {
-	OnClose            func() (err error)
-	OnLocalAddr        func() (laddr net.Addr)
-	OnRead             func(b []byte) (n int, err error)
-	OnRemoteAddr       func() (raddr net.Addr)
-	OnSetDeadline      func(t time.Time) (err error)
-	OnSetReadDeadline  func(t time.Time) (err error)
-	OnSetWriteDeadline func(t time.Time) (err error)
-	OnWrite            func(b []byte) (n int, err error)
-}
-
-// Close implements the [net.Conn] interface for *Conn.
-func (c *Conn) Close() (err error) {
-	return c.OnClose()
-}
-
-// LocalAddr implements the [net.Conn] interface for *Conn.
-func (c *Conn) LocalAddr() (laddr net.Addr) {
-	return c.OnLocalAddr()
-}
-
-// Read implements the [net.Conn] interface for *Conn.
-func (c *Conn) Read(b []byte) (n int, err error) {
-	return c.OnRead(b)
-}
-
-// RemoteAddr implements the [net.Conn] interface for *Conn.
-func (c *Conn) RemoteAddr() (raddr net.Addr) {
-	return c.OnRemoteAddr()
-}
-
-// SetDeadline implements the [net.Conn] interface for *Conn.
-func (c *Conn) SetDeadline(t time.Time) (err error) {
-	return c.OnSetDeadline(t)
-}
-
-// SetReadDeadline implements the [net.Conn] interface for *Conn.
-func (c *Conn) SetReadDeadline(t time.Time) (err error) {
-	return c.OnSetReadDeadline(t)
-}
-
-// SetWriteDeadline implements the [net.Conn] interface for *Conn.
-func (c *Conn) SetWriteDeadline(t time.Time) (err error) {
-	return c.OnSetWriteDeadline(t)
-}
-
-// Write implements the [net.Conn] interface for *Conn.
-func (c *Conn) Write(b []byte) (n int, err error) {
-	return c.OnWrite(b)
-}
-
-// type check
-var _ net.Listener = (*Listener)(nil)
-
-// Listener is a [net.Listener] for tests.
-type Listener struct {
-	OnAccept func() (c net.Conn, err error)
-	OnAddr   func() (addr net.Addr)
-	OnClose  func() (err error)
-}
-
-// Accept implements the [net.Listener] interface for *Listener.
-func (l *Listener) Accept() (c net.Conn, err error) {
-	return l.OnAccept()
-}
-
-// Addr implements the [net.Listener] interface for *Listener.
-func (l *Listener) Addr() (addr net.Addr) {
-	return l.OnAddr()
-}
-
-// Close implements the [net.Listener] interface for *Listener.
-func (l *Listener) Close() (err error) {
-	return l.OnClose()
-}
-
-// type check
-var _ net.PacketConn = (*PacketConn)(nil)
-
-// PacketConn is the [net.PacketConn] for tests.
-type PacketConn struct {
-	OnClose            func() (err error)
-	OnLocalAddr        func() (laddr net.Addr)
-	OnReadFrom         func(b []byte) (n int, addr net.Addr, err error)
-	OnSetDeadline      func(t time.Time) (err error)
-	OnSetReadDeadline  func(t time.Time) (err error)
-	OnSetWriteDeadline func(t time.Time) (err error)
-	OnWriteTo          func(b []byte, addr net.Addr) (n int, err error)
-}
-
-// Close implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) Close() (err error) {
-	return c.OnClose()
-}
-
-// LocalAddr implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) LocalAddr() (laddr net.Addr) {
-	return c.OnLocalAddr()
-}
-
-// ReadFrom implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
-	return c.OnReadFrom(b)
-}
-
-// SetDeadline implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) SetDeadline(t time.Time) (err error) {
-	return c.OnSetDeadline(t)
-}
-
-// SetReadDeadline implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) SetReadDeadline(t time.Time) (err error) {
-	return c.OnSetReadDeadline(t)
-}
-
-// SetWriteDeadline implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) SetWriteDeadline(t time.Time) (err error) {
-	return c.OnSetWriteDeadline(t)
-}
-
-// WriteTo implements the [net.PacketConn] interface for *PacketConn.
-func (c *PacketConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
-	return c.OnWriteTo(b, addr)
-}
-
 // Module AdGuardDNS
 
 // type check

internal/backend/billstat.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdmaps"
 	"github.com/AdguardTeam/AdGuardDNS/internal/billstat"
 	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/mapsutil"
 )
 
 // Billing Statistics Uploader
@@ -114,7 +114,7 @@ type v1DevicesActivityReqDevice struct {
 // devices activity HTTP API.
 func billStatRecsToReq(records billstat.Records) (devices []*v1DevicesActivityReqDevice) {
 	devices = make([]*v1DevicesActivityReqDevice, 0, len(records))
-	agdmaps.OrderedRange(records, func(id agd.DeviceID, rec *billstat.Record) (cont bool) {
+	mapsutil.OrderedRange(records, func(id agd.DeviceID, rec *billstat.Record) (cont bool) {
 		devices = append(devices, &v1DevicesActivityReqDevice{
 			ClientCountry: rec.Country,
 			DeviceID:      id,

internal/backend/profiledb.go

@@ -216,9 +216,30 @@ type v1SettingsRespRuleLists struct {
 // v1SettingsRespSafeBrowsing is the structure for decoding the general safe
 // browsing filtering settings from the backend.
 type v1SettingsRespSafeBrowsing struct {
+	BlockDangerousDomains       *bool `json:"block_dangerous_domains"`
+	BlockNewlyRegisteredDomains bool  `json:"block_nrd"`
 	Enabled                     bool  `json:"enabled"`
 }
 
+// toInternal converts s to an [agd.SafeBrowsingSettings] instance.
+func (s *v1SettingsRespSafeBrowsing) toInternal() (res *agd.SafeBrowsingSettings) {
+	if s == nil {
+		return nil
+	}
+
+	// TODO(d.kolyshev): Don't make this migration after AGDNS-1537.
+	blockDangerDomains := s.Enabled
+	if s.BlockDangerousDomains != nil {
+		blockDangerDomains = *s.BlockDangerousDomains
+	}
+
+	return &agd.SafeBrowsingSettings{
+		Enabled:                     s.Enabled,
+		BlockDangerousDomains:       blockDangerDomains,
+		BlockNewlyRegisteredDomains: s.BlockNewlyRegisteredDomains,
+	}
+}
+
 // v1SettingsResp is the structure for decoding the response from the backend.
 type v1SettingsResp struct {
 	Settings []*v1SettingsRespSettings `json:"settings"`
@@ -510,8 +531,6 @@ func (r *v1SettingsResp) toInternal(
 			// other validation errors.
 		}
 
-		sbEnabled := s.SafeBrowsing != nil && s.SafeBrowsing.Enabled
-
 		pr.Devices = append(pr.Devices, devices...)
 
 		pr.Profiles = append(pr.Profiles, &agd.Profile{
@@ -523,7 +542,7 @@ func (r *v1SettingsResp) toInternal(
 			RuleListIDs:         ruleLists,
 			CustomRules:         rules,
 			FilteredResponseTTL: fltRespTTL,
-			SafeBrowsingEnabled: sbEnabled,
+			SafeBrowsing:        s.SafeBrowsing.toInternal(),
 			RuleListsEnabled:    rlEnabled,
 			FilteringEnabled:    s.FilteringEnabled,
 			QueryLogEnabled:     s.QueryLogEnabled,

internal/backend/profiledb_test.go

@@ -114,6 +114,12 @@ func testProfileResp(t *testing.T) (resp *profiledb.StorageResponse) {
 		YoutubeSafeSearch: false,
 	}
 
+	wantSafeBrowsing := &agd.SafeBrowsingSettings{
+		Enabled:                     true,
+		BlockDangerousDomains:       true,
+		BlockNewlyRegisteredDomains: false,
+	}
+
 	wantLinkedIP := netip.AddrFrom4([4]byte{1, 2, 3, 4})
 
 	wantBlockingMode := dnsmsg.BlockingModeCodec{
@@ -139,7 +145,7 @@ func testProfileResp(t *testing.T) (resp *profiledb.StorageResponse) {
 			RuleListIDs:         []agd.FilterListID{"1"},
 			CustomRules:         nil,
 			FilteredResponseTTL: 10 * time.Second,
-			SafeBrowsingEnabled: true,
+			SafeBrowsing:        wantSafeBrowsing,
 			RuleListsEnabled:    true,
 			FilteringEnabled:    true,
 			QueryLogEnabled:     true,
@@ -160,7 +166,7 @@ func testProfileResp(t *testing.T) (resp *profiledb.StorageResponse) {
 			RuleListIDs:         []agd.FilterListID{"1"},
 			CustomRules:         []agd.FilterRuleText{"||example.org^"},
 			FilteredResponseTTL: 3600 * time.Second,
-			SafeBrowsingEnabled: true,
+			SafeBrowsing:        wantSafeBrowsing,
 			RuleListsEnabled:    true,
 			FilteringEnabled:    true,
 			QueryLogEnabled:     true,

internal/bindtodevice/manager_linux.go

@@ -10,10 +10,10 @@ import (
 	"sync"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdmaps"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/netext"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/mapsutil"
 )
 
 // Manager creates individual listeners and dispatches connections to them.
@@ -80,7 +80,7 @@ func (m *Manager) Add(id ID, ifaceName string, port uint16, conf *ControlConfig)
 		return nil
 	}
 
-	err = agdmaps.OrderedRangeError(m.ifaceListeners, validateDup)
+	err = mapsutil.OrderedRangeError(m.ifaceListeners, validateDup)
 	if err != nil {
 		// Don't wrap the error, because it's informative enough as is.
 		return err

internal/cmd/additional.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdmaps"
+	"github.com/AdguardTeam/golibs/mapsutil"
 	"github.com/prometheus/common/model"
 )
 
@@ -14,7 +14,7 @@ type additionalInfo map[string]string
 
 // validateAdditionalInfo return an error is the section is invalid.
 func (c additionalInfo) validate() (err error) {
-	return agdmaps.OrderedRangeError(c, func(k, _ string) (keyErr error) {
+	return mapsutil.OrderedRangeError(c, func(k, _ string) (keyErr error) {
 		if model.LabelName(k).IsValid() {
 			return nil
 		}

internal/cmd/backend.go

@@ -16,6 +16,9 @@ import (
 // Business Logic Backend Configuration
 
 // backendConfig is the backend module configuration.
+//
+// TODO(a.garipov): Reorganize this object as there is no longer the only one
+// backend environment variable anymore.
 type backendConfig struct {
 	// Timeout is the timeout for all outgoing HTTP requests.  Zero means no
 	// timeout.
@@ -34,23 +37,6 @@ type backendConfig struct {
 	BillStatIvl timeutil.Duration `yaml:"bill_stat_interval"`
 }
 
-// toInternal converts c to the data storage configuration for the DNS server.
-// c is assumed to be valid.
-func (c *backendConfig) toInternal(
-	envs *environments,
-	errColl agd.ErrorCollector,
-) (profStrg *backend.ProfileStorageConfig, billStat *backend.BillStatConfig) {
-	backendEndpoint := &envs.BackendEndpoint.URL
-
-	return &backend.ProfileStorageConfig{
-			BaseEndpoint: netutil.CloneURL(backendEndpoint),
-			Now:          time.Now,
-			ErrColl:      errColl,
-		}, &backend.BillStatConfig{
-			BaseEndpoint: netutil.CloneURL(backendEndpoint),
-		}
-}
-
 // validate returns an error if the backend configuration is invalid.
 func (c *backendConfig) validate() (err error) {
 	switch {
@@ -78,13 +64,40 @@ func setupBackend(
 	sigHdlr signalHandler,
 	errColl agd.ErrorCollector,
 ) (profDB *profiledb.Default, rec *billstat.RuntimeRecorder, err error) {
-	profStrgConf, billStatConf := conf.toInternal(envs, errColl)
+	rec, err = setupBillStat(conf, envs, sigHdlr, errColl)
+	if err != nil {
+		// Don't wrap the error, because it's informative enough as is.
+		return nil, nil, err
+	}
+
+	profDB, err = setupProfDB(conf, envs, sigHdlr, errColl)
+	if err != nil {
+		// Don't wrap the error, because it's informative enough as is.
+		return nil, nil, err
+	}
+
+	return profDB, rec, nil
+}
+
+// setupBillStat creates and returns a billing-statistics recorder as well as
+// starts and registers its refresher in the signal handler.
+func setupBillStat(
+	conf *backendConfig,
+	envs *environments,
+	sigHdlr signalHandler,
+	errColl agd.ErrorCollector,
+) (rec *billstat.RuntimeRecorder, err error) {
+	billStatConf := &backend.BillStatConfig{
+		BaseEndpoint: netutil.CloneURL(&envs.BillStatURL.URL),
+	}
+
 	rec = billstat.NewRuntimeRecorder(&billstat.RuntimeRecorderConfig{
 		Uploader: backend.NewBillStat(billStatConf),
 	})
 
 	refrIvl := conf.RefreshIvl.Duration
 	timeout := conf.Timeout.Duration
+
 	billStatRefr := agd.NewRefreshWorker(&agd.RefreshWorkerConfig{
 		Context: func() (ctx context.Context, cancel context.CancelFunc) {
 			return context.WithTimeout(context.Background(), timeout)
@@ -98,21 +111,37 @@ func setupBackend(
 	})
 	err = billStatRefr.Start()
 	if err != nil {
-		return nil, nil, fmt.Errorf("starting bill stat recorder refresher: %w", err)
+		return nil, fmt.Errorf("starting bill stat recorder refresher: %w", err)
 	}
 
 	sigHdlr.add(billStatRefr)
 
-	profStrg := backend.NewProfileStorage(profStrgConf)
+	return rec, nil
-	profDB, err = profiledb.New(
-		profStrg,
-		conf.FullRefreshIvl.Duration,
-		envs.ProfilesCachePath,
-	)
-	if err != nil {
-		return nil, nil, fmt.Errorf("creating default profile database: %w", err)
 }
 
+// setupProfDB creates and returns a profile database as well as starts and
+// registers its refresher in the signal handler.
+func setupProfDB(
+	conf *backendConfig,
+	envs *environments,
+	sigHdlr signalHandler,
+	errColl agd.ErrorCollector,
+) (profDB *profiledb.Default, err error) {
+	profStrgConf := &backend.ProfileStorageConfig{
+		BaseEndpoint: netutil.CloneURL(&envs.ProfilesURL.URL),
+		Now:          time.Now,
+		ErrColl:      errColl,
+	}
+
+	profStrg := backend.NewProfileStorage(profStrgConf)
+	profDB, err = profiledb.New(profStrg, conf.FullRefreshIvl.Duration, envs.ProfilesCachePath)
+	if err != nil {
+		return nil, fmt.Errorf("creating default profile database: %w", err)
+	}
+
+	refrIvl := conf.RefreshIvl.Duration
+	timeout := conf.Timeout.Duration
+
 	profDBRefr := agd.NewRefreshWorker(&agd.RefreshWorkerConfig{
 		Context: func() (ctx context.Context, cancel context.CancelFunc) {
 			return context.WithTimeout(context.Background(), timeout)
@@ -126,10 +155,10 @@ func setupBackend(
 	})
 	err = profDBRefr.Start()
 	if err != nil {
-		return nil, nil, fmt.Errorf("starting default profile database refresher: %w", err)
+		return nil, fmt.Errorf("starting default profile database refresher: %w", err)
 	}
 
 	sigHdlr.add(profDBRefr)
 
-	return profDB, rec, nil
+	return profDB, nil
 }

internal/cmd/cache.go

@@ -1,6 +1,10 @@
 package cmd
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/AdguardTeam/golibs/timeutil"
+)
 
 // Cache Configuration
 
@@ -9,6 +13,9 @@ import "fmt"
 // TODO(a.garipov): Consider adding parameter Enabled or a new Type instead of
 // relying on Size == 0 to disable cache.
 type cacheConfig struct {
+	// TTLOverride is a section with the settings for cache item TTL overrides.
+	TTLOverride *ttlOverride `yaml:"ttl_override"`
+
 	// Type of cache to use.  See cacheType* constants.
 	Type string `yaml:"type"`
 
@@ -21,6 +28,16 @@ type cacheConfig struct {
 	ECSSize int `yaml:"ecs_size"`
 }
 
+// ttlOverride represents TTL override configuration.
+type ttlOverride struct {
+	// Min describes the minimum duration for cache item TTL.
+	Min timeutil.Duration `yaml:"min"`
+
+	// Enabled returns true if the cache item TTL could be overwritten with Min
+	// value.
+	Enabled bool `yaml:"enabled"`
+}
+
 // Cache types.
 const (
 	cacheTypeECS    = "ecs"
@@ -42,6 +59,25 @@ func (c *cacheConfig) validate() (err error) {
 		return newMustBeNonNegativeError("size", c.Size)
 	case c.Type == cacheTypeECS && c.ECSSize < 0:
 		return newMustBeNonNegativeError("ecs_size", c.ECSSize)
+	default:
+		// Go on.
+	}
+
+	err = c.TTLOverride.validate()
+	if err != nil {
+		return fmt.Errorf("ttl_override: %w", err)
+	}
+
+	return nil
+}
+
+// validate returns an error if the TTL override configuration is invalid.
+func (c *ttlOverride) validate() (err error) {
+	switch {
+	case c == nil:
+		return errNilConfig
+	case c.Min.Duration <= 0:
+		return newMustBePositiveError("min", c.Min)
 	default:
 		return nil
 	}

internal/cmd/cmd.go

@@ -5,10 +5,8 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"os"
 	"runtime"
-	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdnet"
@@ -32,9 +30,7 @@ import (
 func Main() {
 	// Initial Configuration
 
-	//lint:ignore SA1019 According to ameshkov, using a non-cryptographically
+	agd.InitRequestID()
-	//secure RNG is fine for things such as random upstream selection.
-	rand.Seed(time.Now().UnixNano())
 
 	// Log only to stdout and let users decide how to process it.
 	log.SetOutput(os.Stdout)
@@ -93,11 +89,17 @@ func Main() {
 	err = os.MkdirAll(envs.FilterCachePath, agd.DefaultDirPerm)
 	check(err)
 
+	// TODO(ameshkov): Consider making a separated max_size config for
+	// safe-browsing and adult-blocking filters.
+	maxFilterSize := int64(c.Filters.MaxSize.Bytes())
+
 	safeBrowsingHashes, safeBrowsingFilter, err := setupHashPrefixFilter(
 		c.SafeBrowsing,
 		filteringResolver,
 		agd.FilterListIDSafeBrowsing,
+		envs.SafeBrowsingURL,
 		envs.FilterCachePath,
+		maxFilterSize,
 		sigHdlr,
 		errColl,
 	)
@@ -107,7 +109,22 @@ func Main() {
 		c.AdultBlocking,
 		filteringResolver,
 		agd.FilterListIDAdultBlocking,
+		envs.AdultBlockingURL,
 		envs.FilterCachePath,
+		maxFilterSize,
+		sigHdlr,
+		errColl,
+	)
+	check(err)
+
+	_, newRegDomainsFilter, err := setupHashPrefixFilter(
+		// Reuse general safe browsing filter configuration.
+		c.SafeBrowsing,
+		filteringResolver,
+		agd.FilterListIDNewRegDomains,
+		envs.NewRegDomainsURL,
+		envs.FilterCachePath,
+		maxFilterSize,
 		sigHdlr,
 		errColl,
 	)
@@ -121,6 +138,7 @@ func Main() {
 		envs,
 		safeBrowsingFilter,
 		adultBlockingFilter,
+		newRegDomainsFilter,
 	)
 
 	fltRefrTimeout := c.Filters.RefreshTimeout.Duration
@@ -174,8 +192,7 @@ func Main() {
 
 	// DNSDB
 
-	dnsDB, err := envs.buildDNSDB(sigHdlr, errColl)
+	dnsDB := c.DNSDB.toInternal(errColl)
-	check(err)
 
 	// Filtering-rule statistics
 
@@ -241,8 +258,11 @@ func Main() {
 		ServerGroups:        srvGrps,
 		CacheSize:           c.Cache.Size,
 		ECSCacheSize:        c.Cache.ECSSize,
+		CacheMinTTL:         c.Cache.TTLOverride.Min.Duration,
+		UseCacheTTLOverride: c.Cache.TTLOverride.Enabled,
 		UseECSCache:         c.Cache.Type == cacheTypeECS,
 		ResearchMetrics:     bool(envs.ResearchMetrics),
+		ResearchLogs:        bool(envs.ResearchLogs),
 		ControlConf:         ctrlConf,
 	}
 

internal/cmd/config.go

@@ -30,6 +30,9 @@ type configuration struct {
 	// Upstream is the configuration of upstream servers for the DNS servers.
 	Upstream *upstreamConfig `yaml:"upstream"`
 
+	// DNSDB is the configuration of DNSDB buffer.
+	DNSDB *dnsDBConfig `yaml:"dnsdb"`
+
 	// Backend is the AdGuard HTTP backend service configuration.  See the
 	// environments type for more backend parameters.
 	Backend *backendConfig `yaml:"backend"`
@@ -114,6 +117,9 @@ func (c *configuration) validate() (err error) {
 	}, {
 		validate: c.Cache.validate,
 		name:     "cache",
+	}, {
+		validate: c.DNSDB.validate,
+		name:     "dnsdb",
 	}, {
 		validate: c.Backend.validate,
 		name:     "backend",

internal/cmd/dnsdb.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
+	"github.com/AdguardTeam/AdGuardDNS/internal/dnsdb"
+)
+
+// DNSDB Configuration.
+
+// dnsDBConfig is the configuration of the DNSDB module.
+type dnsDBConfig struct {
+	// MaxSize is the maximum amount of records in the memory buffer.
+	MaxSize int `yaml:"max_size"`
+
+	// Enabled describes if the DNSDB memory buffer is enabled.
+	Enabled bool `yaml:"enabled"`
+}
+
+// validate returns an error if the configuration is invalid.
+func (c *dnsDBConfig) validate() (err error) {
+	switch {
+	case c == nil:
+		return errNilConfig
+	case c.MaxSize <= 0:
+		return newMustBePositiveError("size", c.MaxSize)
+	default:
+		return nil
+	}
+}
+
+// toInternal builds and returns an anonymous statistics collector.
+func (c *dnsDBConfig) toInternal(errColl agd.ErrorCollector) (d dnsdb.Interface) {
+	if !c.Enabled {
+		return dnsdb.Empty{}
+	}
+
+	db := dnsdb.New(&dnsdb.DefaultConfig{
+		ErrColl: errColl,
+		MaxSize: c.MaxSize,
+	})
+
+	return db
+}

internal/cmd/env.go

@@ -24,18 +24,22 @@ import (
 
 // environments represents the configuration that is kept in the environment.
 type environments struct {
-	BackendEndpoint          *agdhttp.URL `env:"BACKEND_ENDPOINT,notEmpty"`
+	AdultBlockingURL         *agdhttp.URL `env:"ADULT_BLOCKING_URL,notEmpty"`
+	BillStatURL              *agdhttp.URL `env:"BILLSTAT_URL,notEmpty"`
 	BlockedServiceIndexURL   *agdhttp.URL `env:"BLOCKED_SERVICE_INDEX_URL,notEmpty"`
 	ConsulAllowlistURL       *agdhttp.URL `env:"CONSUL_ALLOWLIST_URL,notEmpty"`
 	ConsulDNSCheckKVURL      *agdhttp.URL `env:"CONSUL_DNSCHECK_KV_URL"`
 	ConsulDNSCheckSessionURL *agdhttp.URL `env:"CONSUL_DNSCHECK_SESSION_URL"`
 	FilterIndexURL           *agdhttp.URL `env:"FILTER_INDEX_URL,notEmpty"`
 	GeneralSafeSearchURL     *agdhttp.URL `env:"GENERAL_SAFE_SEARCH_URL,notEmpty"`
-	YoutubeSafeSearchURL     *agdhttp.URL `env:"YOUTUBE_SAFE_SEARCH_URL,notEmpty"`
+	LinkedIPTargetURL        *agdhttp.URL `env:"LINKED_IP_TARGET_URL"`
+	NewRegDomainsURL         *agdhttp.URL `env:"NEW_REG_DOMAINS_URL,notEmpty"`
+	ProfilesURL              *agdhttp.URL `env:"PROFILES_URL,notEmpty"`
 	RuleStatURL              *agdhttp.URL `env:"RULESTAT_URL"`
+	SafeBrowsingURL          *agdhttp.URL `env:"SAFE_BROWSING_URL,notEmpty"`
+	YoutubeSafeSearchURL     *agdhttp.URL `env:"YOUTUBE_SAFE_SEARCH_URL,notEmpty"`
 
 	ConfPath          string `env:"CONFIG_PATH" envDefault:"./config.yaml"`
-	DNSDBPath         string `env:"DNSDB_PATH"`
 	FilterCachePath   string `env:"FILTER_CACHE_PATH" envDefault:"./filters/"`
 	ProfilesCachePath string `env:"PROFILES_CACHE_PATH" envDefault:"./profilecache.json"`
 	GeoIPASNPath      string `env:"GEOIP_ASN_PATH" envDefault:"./asn.mmdb"`
@@ -51,6 +55,7 @@ type environments struct {
 	LogTimestamp    strictBool `env:"LOG_TIMESTAMP" envDefault:"1"`
 	LogVerbose      strictBool `env:"VERBOSE" envDefault:"0"`
 	ResearchMetrics strictBool `env:"RESEARCH_METRICS" envDefault:"0"`
+	ResearchLogs    strictBool `env:"RESEARCH_LOGS" envDefault:"0"`
 }
 
 // readEnvs reads the configuration.
@@ -97,41 +102,6 @@ func (envs *environments) buildErrColl() (errColl agd.ErrorCollector, err error)
 	return errcoll.NewSentryErrorCollector(cli), nil
 }
 
-// buildDNSDB builds and returns an anonymous statistics collector and register
-// its refresher in sigHdlr, if needed.
-func (envs *environments) buildDNSDB(
-	sigHdlr signalHandler,
-	errColl agd.ErrorCollector,
-) (d dnsdb.Interface, err error) {
-	if envs.DNSDBPath == "" {
-		return dnsdb.Empty{}, nil
-	}
-
-	b := dnsdb.NewBolt(&dnsdb.BoltConfig{
-		Path:    envs.DNSDBPath,
-		ErrColl: errColl,
-	})
-
-	refr := agd.NewRefreshWorker(&agd.RefreshWorkerConfig{
-		Context:   ctxWithDefaultTimeout,
-		Refresher: b,
-		ErrColl:   errColl,
-		Name:      "dnsdb",
-		// TODO(ameshkov): Consider making configurable.
-		Interval:            15 * time.Minute,
-		RefreshOnShutdown:   true,
-		RoutineLogsAreDebug: false,
-	})
-	err = refr.Start()
-	if err != nil {
-		return nil, fmt.Errorf("starting dnsdb refresher: %w", err)
-	}
-
-	sigHdlr.add(refr)
-
-	return b, nil
-}
-
 // geoIP returns an GeoIP database implementation from environment.
 func (envs *environments) geoIP(
 	c *geoIPConfig,
@@ -143,6 +113,8 @@ func (envs *environments) geoIP(
 		CountryPath:    envs.GeoIPCountryPath,
 		HostCacheSize:  c.HostCacheSize,
 		IPCacheSize:    c.IPCacheSize,
+		AllTopASNs:     geoip.DefaultTopASNs,
+		CountryTopASNs: geoip.DefaultCountryTopASNs,
 	})
 	if err != nil {
 		return nil, err

internal/cmd/filter.go

@@ -11,6 +11,7 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/hashprefix"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/timeutil"
+	"github.com/c2h5oh/datasize"
 )
 
 // Filters configuration
@@ -18,6 +19,9 @@ import (
 // filtersConfig contains the configuration for the filter lists and filtering
 // storage to be used.
 type filtersConfig struct {
+	// RuleListCache is the cache settings for the filtering rule-list.
+	RuleListCache *fltRuleListCache `yaml:"rule_list_cache"`
+
 	// CustomFilterCacheSize is the size of the LRU cache of compiled filtering
 	// engines for profiles with custom filtering rules.
 	CustomFilterCacheSize int `yaml:"custom_filter_cache_size"`
@@ -25,10 +29,6 @@ type filtersConfig struct {
 	// SafeSearchCacheSize is the size of the LRU cache of safe-search results.
 	SafeSearchCacheSize int `yaml:"safe_search_cache_size"`
 
-	// RuleListCacheSize defines the size of the LRU cache of rule-list
-	// filtering results.
-	RuleListCacheSize int `yaml:"rule_list_cache_size"`
-
 	// ResponseTTL is the TTL to set for DNS responses to requests for filtered
 	// domains.
 	ResponseTTL timeutil.Duration `yaml:"response_ttl"`
@@ -42,8 +42,8 @@ type filtersConfig struct {
 	// 30s timeout.
 	RefreshTimeout timeutil.Duration `yaml:"refresh_timeout"`
 
-	// UseRuleListCache, if true, enables rule list cache.
+	// MaxSize is the maximum size of the downloadable filtering rule-list.
-	UseRuleListCache bool `yaml:"use_rule_list_cache"`
+	MaxSize datasize.ByteSize `yaml:"max_size"`
 }
 
 // toInternal converts c to the filter storage configuration for the DNS server.
@@ -54,6 +54,7 @@ func (c *filtersConfig) toInternal(
 	envs *environments,
 	safeBrowsing *hashprefix.Filter,
 	adultBlocking *hashprefix.Filter,
+	newRegDomains *hashprefix.Filter,
 ) (conf *filter.DefaultStorageConfig) {
 	return &filter.DefaultStorageConfig{
 		FilterIndexURL:            netutil.CloneURL(&envs.FilterIndexURL.URL),
@@ -62,6 +63,7 @@ func (c *filtersConfig) toInternal(
 		YoutubeSafeSearchRulesURL: netutil.CloneURL(&envs.YoutubeSafeSearchURL.URL),
 		SafeBrowsing:              safeBrowsing,
 		AdultBlocking:             adultBlocking,
+		NewRegDomains:             newRegDomains,
 		Now:                       time.Now,
 		ErrColl:                   errColl,
 		Resolver:                  resolver,
@@ -70,9 +72,10 @@ func (c *filtersConfig) toInternal(
 		SafeSearchCacheSize:       c.SafeSearchCacheSize,
 		// TODO(a.garipov): Consider making this configurable.
 		SafeSearchCacheTTL: 1 * time.Hour,
-		RuleListCacheSize:  c.RuleListCacheSize,
+		RuleListCacheSize:  c.RuleListCache.Size,
 		RefreshIvl:         c.RefreshIvl.Duration,
-		UseRuleListCache:   c.UseRuleListCache,
+		UseRuleListCache:   c.RuleListCache.Enabled,
+		MaxRuleListSize:    int64(c.MaxSize.Bytes()),
 	}
 }
 
@@ -83,14 +86,43 @@ func (c *filtersConfig) validate() (err error) {
 		return errNilConfig
 	case c.SafeSearchCacheSize <= 0:
 		return newMustBePositiveError("safe_search_cache_size", c.SafeSearchCacheSize)
-	case c.RuleListCacheSize <= 0:
-		return newMustBePositiveError("rule_list_cache_size", c.RuleListCacheSize)
 	case c.ResponseTTL.Duration <= 0:
 		return newMustBePositiveError("response_ttl", c.ResponseTTL)
 	case c.RefreshIvl.Duration <= 0:
 		return newMustBePositiveError("refresh_interval", c.RefreshIvl)
 	case c.RefreshTimeout.Duration <= 0:
 		return newMustBePositiveError("refresh_timeout", c.RefreshTimeout)
+	case c.MaxSize <= 0:
+		return newMustBePositiveError("max_size", c.MaxSize)
+	default:
+		// Go on.
+	}
+
+	err = c.RuleListCache.validate()
+	if err != nil {
+		return fmt.Errorf("rule_list_cache: %w", err)
+	}
+
+	return nil
+}
+
+// fltRuleListCache contains filtering rule-list cache configuration.
+type fltRuleListCache struct {
+	// Size defines the size of the LRU cache of rule-list filtering results.
+	Size int `yaml:"size"`
+
+	// Enabled shows if the rule-list cache is enabled.  If it is false, the
+	// rest of the settings are ignored.
+	Enabled bool `yaml:"enabled"`
+}
+
+// validate returns an error if the rule-list cache configuration is invalid.
+func (c *fltRuleListCache) validate() (err error) {
+	switch {
+	case c == nil:
+		return errNilConfig
+	case c.Size <= 0:
+		return newMustBePositiveError("size", c.Size)
 	default:
 		return nil
 	}

internal/cmd/filteringgroup.go

@@ -71,6 +71,14 @@ type fltGrpParental struct {
 type fltGrpSafeBrowsing struct {
 	// Enabled shows if the general safe browsing filtering should be enforced.
 	Enabled bool `yaml:"enabled"`
+
+	// BlockDangerousDomains shows whether the dangerous domains safe browsing
+	// filtering should be enforced.
+	BlockDangerousDomains bool `yaml:"block_dangerous_domains"`
+
+	// BlockNewlyRegisteredDomains shows whether the newly registered domains
+	// safe browsing filtering should be enforced.
+	BlockNewlyRegisteredDomains bool `yaml:"block_newly_registered_domains"`
 }
 
 // validate returns an error if the filtering group is invalid.
@@ -134,6 +142,8 @@ func (groups filteringGroups) toInternal(
 			ParentalEnabled:             g.Parental.Enabled,
 			BlockAdult:                  g.Parental.BlockAdult,
 			SafeBrowsingEnabled:         g.SafeBrowsing.Enabled,
+			BlockDangerousDomains:       g.SafeBrowsing.BlockDangerousDomains,
+			BlockNewlyRegisteredDomains: g.SafeBrowsing.BlockNewlyRegisteredDomains,
 			GeneralSafeSearch:           g.Parental.GeneralSafeSearch,
 			YoutubeSafeSearch:           g.Parental.YoutubeSafeSearch,
 			BlockPrivateRelay:           g.BlockPrivateRelay,

internal/cmd/ifacelistener.go

@@ -2,9 +2,9 @@ package cmd
 
 import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdmaps"
 	"github.com/AdguardTeam/AdGuardDNS/internal/bindtodevice"
 	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/mapsutil"
 )
 
 // Network interface listener configuration
@@ -35,7 +35,7 @@ func (c *interfaceListenersConfig) toInternal(
 		ChannelBufferSize: c.ChannelBufferSize,
 	})
 
-	err = agdmaps.OrderedRangeError(
+	err = mapsutil.OrderedRangeError(
 		c.List,
 		func(id bindtodevice.ID, l *interfaceListener) (addErr error) {
 			return errors.Annotate(m.Add(id, l.Interface, l.Port, ctrlConf), "adding listener %q: %w", id)
@@ -67,7 +67,7 @@ func (c *interfaceListenersConfig) validate() (err error) {
 		// Go on.
 	}
 
-	err = agdmaps.OrderedRangeError(
+	err = mapsutil.OrderedRangeError(
 		c.List,
 		func(id bindtodevice.ID, l *interfaceListener) (lsnrErr error) {
 			return errors.Annotate(l.validate(), "interface %q: %w", id)

internal/cmd/network.go

@@ -3,19 +3,18 @@ package cmd
 import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/bindtodevice"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/netext"
+	"github.com/c2h5oh/datasize"
 )
 
 // network defines the network settings.
-//
-// TODO(a.garipov): Use [datasize.ByteSize] for sizes.
 type network struct {
-	// SndBufSize defines the size of socket send buffer in bytes.  Default is
+	// SndBufSize defines the size of socket send buffer.  Default is zero (uses
-	// zero (uses system settings).
+	// system settings).
-	SndBufSize int `yaml:"so_sndbuf"`
+	SndBufSize datasize.ByteSize `yaml:"so_sndbuf"`
 
-	// RcvBufSize defines the size of socket receive buffer in bytes.  Default
+	// RcvBufSize defines the size of socket receive buffer.  Default is zero
-	// is zero (uses system settings).
+	// (uses system settings).
-	RcvBufSize int `yaml:"so_rcvbuf"`
+	RcvBufSize datasize.ByteSize `yaml:"so_rcvbuf"`
 }
 
 // validate returns an error if the network configuration is invalid.
@@ -24,14 +23,6 @@ func (n *network) validate() (err error) {
 		return errNilConfig
 	}
 
-	if n.SndBufSize < 0 {
-		return newMustBeNonNegativeError("so_sndbuf", n.SndBufSize)
-	}
-
-	if n.RcvBufSize < 0 {
-		return newMustBeNonNegativeError("so_rcvbuf", n.RcvBufSize)
-	}
-
 	return nil
 }
 
@@ -39,12 +30,12 @@ func (n *network) validate() (err error) {
 // extension control configuration.
 func (n *network) toInternal() (bc *bindtodevice.ControlConfig, nc *netext.ControlConfig) {
 	bc = &bindtodevice.ControlConfig{
-		SndBufSize: n.SndBufSize,
+		SndBufSize: int(n.SndBufSize.Bytes()),
-		RcvBufSize: n.RcvBufSize,
+		RcvBufSize: int(n.RcvBufSize.Bytes()),
 	}
 	nc = &netext.ControlConfig{
-		SndBufSize: n.SndBufSize,
+		SndBufSize: int(n.SndBufSize.Bytes()),
-		RcvBufSize: n.RcvBufSize,
+		RcvBufSize: int(n.RcvBufSize.Bytes()),
 	}
 
 	return bc, nc

internal/cmd/safebrowsing.go

@@ -17,9 +17,6 @@ import (
 
 // safeBrowsingConfig is the configuration for one of the safe browsing filters.
 type safeBrowsingConfig struct {
-	// URL is the URL used to update the filter.
-	URL *agdhttp.URL `yaml:"url"`
-
 	// BlockHost is the hostname with which to respond to any requests that
 	// match the filter.
 	//
@@ -43,7 +40,9 @@ func (c *safeBrowsingConfig) toInternal(
 	errColl agd.ErrorCollector,
 	resolver agdnet.Resolver,
 	id agd.FilterListID,
+	url *agdhttp.URL,
 	cacheDir string,
+	maxSize int64,
 ) (fltConf *hashprefix.FilterConfig, err error) {
 	hashes, err := hashprefix.NewStorage("")
 	if err != nil {
@@ -52,7 +51,7 @@ func (c *safeBrowsingConfig) toInternal(
 
 	return &hashprefix.FilterConfig{
 		Hashes:          hashes,
-		URL:             netutil.CloneURL(&c.URL.URL),
+		URL:             netutil.CloneURL(&url.URL),
 		ErrColl:         errColl,
 		Resolver:        resolver,
 		ID:              id,
@@ -61,6 +60,7 @@ func (c *safeBrowsingConfig) toInternal(
 		Staleness:       c.RefreshIvl.Duration,
 		CacheTTL:        c.CacheTTL.Duration,
 		CacheSize:       c.CacheSize,
+		MaxSize:         maxSize,
 	}, nil
 }
 
@@ -70,8 +70,6 @@ func (c *safeBrowsingConfig) validate() (err error) {
 	switch {
 	case c == nil:
 		return errNilConfig
-	case c.URL == nil:
-		return errors.Error("no url")
 	case c.BlockHost == "":
 		return errors.Error("no block_host")
 	case c.CacheSize <= 0:
@@ -91,11 +89,13 @@ func setupHashPrefixFilter(
 	conf *safeBrowsingConfig,
 	resolver *agdnet.CachingResolver,
 	id agd.FilterListID,
+	url *agdhttp.URL,
 	cachePath string,
+	maxSize int64,
 	sigHdlr signalHandler,
 	errColl agd.ErrorCollector,
 ) (strg *hashprefix.Storage, flt *hashprefix.Filter, err error) {
-	fltConf, err := conf.toInternal(errColl, resolver, id, cachePath)
+	fltConf, err := conf.toInternal(errColl, resolver, id, url, cachePath, maxSize)
 	if err != nil {
 		return nil, nil, fmt.Errorf("configuring hash prefix filter %s: %w", id, err)
 	}

internal/cmd/server.go

@@ -204,18 +204,24 @@ func (s *server) bindData(
 
 	ifaces := s.BindInterfaces
 	bindData = make([]*agd.ServerBindData, 0, len(ifaces))
-	for i, iface := range s.BindInterfaces {
+	for i, iface := range ifaces {
+		address := string(iface.ID)
+
+		for j, subnet := range iface.Subnets {
 			var lc netext.ListenConfig
-		lc, err = btdMgr.ListenConfig(iface.ID, iface.Subnet)
+			lc, err = btdMgr.ListenConfig(iface.ID, subnet)
 			if err != nil {
-			return nil, fmt.Errorf("bind_interface at index %d: %w", i, err)
+				const errStr = "bind_interface at index %d: subnet at index %d: %w"
+
+				return nil, fmt.Errorf(errStr, i, j, err)
 			}
 
 			bindData = append(bindData, &agd.ServerBindData{
 				ListenConfig: lc,
-			Address:      string(iface.ID),
+				Address:      address,
 			})
 		}
+	}
 
 	return bindData, nil
 }
@@ -289,7 +295,7 @@ func (s *server) validateBindData() (err error) {
 // serverBindInterface contains the data for a network interface binding.
 type serverBindInterface struct {
 	ID      bindtodevice.ID `yaml:"id"`
-	Subnet netip.Prefix    `yaml:"subnet"`
+	Subnets []netip.Prefix  `yaml:"subnets"`
 }
 
 // validate returns an error if the network interface binding configuration is
@@ -300,9 +306,26 @@ func (c *serverBindInterface) validate() (err error) {
 		return errNilConfig
 	case c.ID == "":
 		return errors.Error("no id")
-	case !c.Subnet.IsValid():
+	case len(c.Subnets) == 0:
-		return errors.Error("bad subnet")
+		return errors.Error("no subnets")
 	default:
+		// Go on.
+	}
+
+	set := map[netip.Prefix]struct{}{}
+
+	for i, subnet := range c.Subnets {
+		if !subnet.IsValid() {
+			return fmt.Errorf("bad subnet at index %d", i)
+		}
+
+		_, ok := set[subnet]
+		if ok {
+			return fmt.Errorf("duplicate subnet %s at index %d", subnet, i)
+		}
+
+		set[subnet] = struct{}{}
+	}
+
 	return nil
 }
-}

internal/cmd/websvc.go

@@ -67,7 +67,6 @@ func (c *webConfig) toInternal(
 	}
 
 	conf = &websvc.Config{
-		LinkedIPBackendURL: netutil.CloneURL(&envs.BackendEndpoint.URL),
 		DNSCheck: dnsCk,
 		ErrColl:  errColl,
 		Timeout:  c.Timeout.Duration,
@@ -77,7 +76,7 @@ func (c *webConfig) toInternal(
 		conf.RootRedirectURL = netutil.CloneURL(&c.RootRedirectURL.URL)
 	}
 
-	conf.LinkedIP, err = c.LinkedIP.toInternal()
+	conf.LinkedIP, err = c.LinkedIP.toInternal(envs.LinkedIPTargetURL)
 	if err != nil {
 		return nil, fmt.Errorf("converting linked_ip: %w", err)
 	}
@@ -179,7 +178,9 @@ type linkedIPServer struct {
 
 // toInternal converts s to a linkedIP server configuration.  s is assumed to be
 // valid.
-func (s *linkedIPServer) toInternal() (srv *websvc.LinkedIPServer, err error) {
+func (s *linkedIPServer) toInternal(
+	targetURL *agdhttp.URL,
+) (srv *websvc.LinkedIPServer, err error) {
 	if s == nil {
 		return nil, nil
 	}
@@ -190,6 +191,12 @@ func (s *linkedIPServer) toInternal() (srv *websvc.LinkedIPServer, err error) {
 		return nil, fmt.Errorf("converting bind: %w", err)
 	}
 
+	if targetURL == nil {
+		return nil, fmt.Errorf("env variable LINKED_IP_TARGET_URL must be set for using linked_ip")
+	}
+
+	srv.TargetURL = netutil.CloneURL(&targetURL.URL)
+
 	return srv, nil
 }
 

internal/connlimiter/limiter_test.go

@@ -6,11 +6,11 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdtest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/connlimiter"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/AdguardTeam/golibs/testutil/fakenet"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -36,7 +36,7 @@ func TestLimiter(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	conn := &agdtest.Conn{
+	conn := &fakenet.Conn{
 		OnClose:     func() (err error) { return nil },
 		OnLocalAddr: func() (laddr net.Addr) { panic("not implemented") },
 		OnRead:      func(b []byte) (n int, err error) { panic("not implemented") },
@@ -52,7 +52,7 @@ func TestLimiter(t *testing.T) {
 		OnWrite:            func(b []byte) (n int, err error) { panic("not implemented") },
 	}
 
-	lsnr := &agdtest.Listener{
+	lsnr := &fakenet.Listener{
 		OnAccept: func() (c net.Conn, err error) { return conn, nil },
 		OnAddr: func() (addr net.Addr) {
 			return &net.TCPAddr{

internal/connlimiter/listenconfig_test.go

@@ -9,12 +9,13 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdtest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/connlimiter"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
+	"github.com/AdguardTeam/golibs/testutil/fakenet"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestListenConfig(t *testing.T) {
-	pc := &agdtest.PacketConn{
+	pc := &fakenet.PacketConn{
 		OnClose:            func() (err error) { panic("not implemented") },
 		OnLocalAddr:        func() (laddr net.Addr) { panic("not implemented") },
 		OnReadFrom:         func(b []byte) (n int, addr net.Addr, err error) { panic("not implemented") },
@@ -24,7 +25,7 @@ func TestListenConfig(t *testing.T) {
 		OnWriteTo:          func(b []byte, addr net.Addr) (n int, err error) { panic("not implemented") },
 	}
 
-	lsnr := &agdtest.Listener{
+	lsnr := &fakenet.Listener{
 		OnAccept: func() (c net.Conn, err error) { panic("not implemented") },
 		OnAddr:   func() (addr net.Addr) { panic("not implemented") },
 		OnClose:  func() (err error) { return nil },

internal/debugsvc/debugsvc.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/http/pprof"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/pprofutil"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -143,7 +143,7 @@ func (svc *Service) addHandler(serviceName string, mux *http.ServeMux) {
 	case "health-check":
 		healthMux(mux)
 	case "pprof":
-		pprofMux(mux)
+		pprofutil.RoutePprof(mux)
 	case "prometheus":
 		promMux(mux)
 	default:
@@ -166,23 +166,6 @@ func healthMux(mux *http.ServeMux) {
 	)
 }
 
-// pprofMux adds handler func to the mux from args for the pprof service.
-func pprofMux(mux *http.ServeMux) {
-	mux.HandleFunc("/debug/pprof/", pprof.Index)
-
-	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-
-	mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
-	mux.Handle("/debug/pprof/block", pprof.Handler("block"))
-	mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
-	mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
-	mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
-	mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
-}
-
 // promMux adds handler func to the mux from args for the prometheus service.
 func promMux(mux *http.ServeMux) {
 	mux.Handle("/metrics", promhttp.Handler())

internal/dnscheck/dnscheck.go

@@ -98,8 +98,14 @@ func isInvalidRandomIDRune(r rune) (ok bool) {
 
 // isValidRandomIDRune returns true if r is a valid random ID rune.
 func isValidRandomIDRune(r rune) (ok bool) {
-	return (r >= 'a' && r <= 'z') ||
+	switch {
-		(r >= 'A' && r <= 'Z') ||
+	case
-		(r >= '0' && r <= '9') ||
+		r >= 'a' && r <= 'z',
-		r == '-'
+		r >= 'A' && r <= 'Z',
+		r >= '0' && r <= '9',
+		r == '-':
+		return true
+	default:
+		return false
+	}
 }

internal/dnsdb/bolt.go

@@ -1,286 +0,0 @@
-package dnsdb
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
-	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/log"
-	"github.com/miekg/dns"
-	"go.etcd.io/bbolt"
-)
-
-// BoldDB-Based DNSDB
-
-// recordBuffer is an in-memory buffer of DNSDB records.
-type recordBuffer map[string][]*record
-
-// Bolt is BoltDB-based DNSDB implementation.
-type Bolt struct {
-	// dumpMu is used to make sure that the CSV endpoint is only used by one
-	// client at a time, as well as that the periodic flushing doesn't interfere
-	// with it.
-	//
-	// TODO(a.garipov): Consider adding rate limiting.
-	dumpMu *sync.Mutex
-
-	bufferMu *sync.Mutex
-	buffer   recordBuffer
-
-	errColl agd.ErrorCollector
-
-	path string
-}
-
-// BoltConfig is the BoltDB-based DNSDB configuration structure.
-type BoltConfig struct {
-	// ErrColl is used to collect HTTP errors as well as
-	ErrColl agd.ErrorCollector
-
-	// Path is the path to the BoltDB file.
-	Path string
-}
-
-// NewBolt creates a new BoltDB-based DNSDB.  c must not be nil.
-func NewBolt(c *BoltConfig) (db *Bolt) {
-	db = &Bolt{
-		dumpMu: &sync.Mutex{},
-
-		bufferMu: &sync.Mutex{},
-		buffer:   recordBuffer{},
-
-		errColl: c.ErrColl,
-
-		path: c.Path,
-	}
-
-	return db
-}
-
-// type check
-var _ Interface = (*Bolt)(nil)
-
-// Record implements the Interface interface for *Bolt.  It saves a DNS response
-// to its in-memory buffer.
-func (db *Bolt) Record(ctx context.Context, m *dns.Msg, ri *agd.RequestInfo) {
-	if isIgnoredMessage(m) {
-		return
-	}
-
-	q := m.Question[0]
-	if isIgnoredQuestion(q) {
-		return
-	}
-
-	key := requestKey(ri.Host, q.Qtype)
-	recs := toDBRecords(q.Qtype, ri.Host, m.Answer, dnsmsg.RCode(m.Rcode))
-	db.saveToBuffer(key, recs)
-}
-
-// saveToBuffer saves recs to the in-memory buffer.
-func (db *Bolt) saveToBuffer(key string, recs []*record) {
-	db.bufferMu.Lock()
-	defer db.bufferMu.Unlock()
-
-	prevRecs, ok := db.buffer[key]
-	if !ok {
-		db.buffer[key] = recs
-		metrics.DNSDBBufferSize.Inc()
-
-		return
-	}
-
-	// Consider that new answers either don't appear between rotations of the
-	// database or don't matter, and so don't merge new records with the old
-	// ones.  Just bump the hit counters.
-	for _, r := range prevRecs {
-		r.Hits++
-	}
-}
-
-// type check
-var _ agd.Refresher = (*Bolt)(nil)
-
-// Refresh implements the agd.Refresher interface for *Bolt.  It flushes the
-// current in-memory buffer data to disk.
-func (db *Bolt) Refresh(ctx context.Context) (err error) {
-	db.dumpMu.Lock()
-	defer db.dumpMu.Unlock()
-
-	_, err = db.flush(ctx, false)
-	if err != nil {
-		// Don't wrap the error, because it's informative enough as is.
-		return err
-	}
-
-	return nil
-}
-
-// flush saves the buffered records to the local BoltDB database.  If rotate is
-// true, it also rotates the database file and returns the path to the previous
-// database file.
-func (db *Bolt) flush(ctx context.Context, rotate bool) (prev string, err error) {
-	start := time.Now()
-	// TODO(a.garipov): Consider only replacing the buffer with an empty one if
-	// the write was successful.
-	buffer := db.replaceBuffer()
-	bdb, err := openBolt(db.path)
-	if err != nil {
-		return "", fmt.Errorf("opening boltdb: %w", err)
-	}
-	defer func() { err = errors.WithDeferred(err, bdb.Close()) }()
-
-	err = bdb.Batch(boltFlushFunc(ctx, buffer, db.errColl))
-	if err != nil {
-		return "", fmt.Errorf("saving: %w", err)
-	}
-
-	if rotate {
-		prev, err = db.rotate()
-		if err != nil {
-			return prev, fmt.Errorf("rotating: %w", err)
-		}
-	}
-
-	metrics.DNSDBSaveDuration.Observe(time.Since(start).Seconds())
-
-	return prev, nil
-}
-
-// replaceBuffer replaced db's current buffer with a new one and returns the
-// previous one.
-func (db *Bolt) replaceBuffer() (prev recordBuffer) {
-	db.bufferMu.Lock()
-	defer db.bufferMu.Unlock()
-
-	prev, db.buffer = db.buffer, recordBuffer{}
-	metrics.DNSDBBufferSize.Set(0)
-
-	return prev
-}
-
-// recordsBucket is the name of the bucket with the DNSDB records.
-const recordsBucket = "records"
-
-// openBolt opens and initializes the BoltDB file.
-func openBolt(dbPath string) (bdb *bbolt.DB, err error) {
-	bdb, err = bbolt.Open(dbPath, agd.DefaultPerm, nil)
-	if err != nil {
-		return nil, fmt.Errorf("opening file: %w", err)
-	}
-
-	err = bdb.Update(func(tx *bbolt.Tx) (ferr error) {
-		_, ferr = tx.CreateBucketIfNotExists([]byte(recordsBucket))
-
-		return ferr
-	})
-	if err != nil {
-		return nil, fmt.Errorf("initializing: %w", err)
-	}
-
-	return bdb, nil
-}
-
-// boltTxFunc is an alias for the type consumed by bbolt.(*DB).Batch.
-type boltTxFunc = func(tx *bbolt.Tx) (err error)
-
-// boltFlushFunc returns a function that reads data from the existing DNSDB
-// file, updates the current buffer, and writes it back.
-//
-// f always returns nil; all errors are reported using errColl.
-func boltFlushFunc(
-	ctx context.Context,
-	buffer recordBuffer,
-	errColl agd.ErrorCollector,
-) (f boltTxFunc) {
-	return func(tx *bbolt.Tx) (err error) {
-		b := tx.Bucket([]byte(recordsBucket))
-
-		for rk, recs := range buffer {
-			k := []byte(rk)
-			err = addExisting(b, k, recs)
-			if err != nil {
-				agd.Collectf(ctx, errColl, "dnsdb: adding existing data for %s: %w", rk, err)
-
-				// Consider errors from reading the previous database
-				// non-critical.
-			}
-
-			var dbData []byte
-			dbData, err = encode(recs)
-			if err != nil {
-				agd.Collectf(ctx, errColl, "dnsdb: encoding data for %s: %w", rk, err)
-
-				continue
-			}
-
-			err = b.Put(k, dbData)
-			if err != nil {
-				agd.Collectf(ctx, errColl, "dnsdb: writing data for %s: %w", rk, err)
-
-				// Consider errors from writing a single key non-critical.
-			}
-		}
-
-		metrics.DNSDBSize.Set(float64(b.Stats().KeyN))
-
-		return nil
-	}
-}
-
-// addExisting looks up previous data for key in b and updates recs with those
-// data if there are any.
-func addExisting(b *bbolt.Bucket, key []byte, recs []*record) (err error) {
-	prevData := b.Get(key)
-	if len(prevData) == 0 {
-		return nil
-	}
-
-	var prevRecs []*record
-	prevRecs, err = decode(prevData)
-	if err != nil {
-		return fmt.Errorf("decoding previous value: %w", err)
-	}
-
-	if len(prevRecs) == 0 {
-		return nil
-	}
-
-	// Use only the first Hits value, because all records share it.
-	prevHits := prevRecs[0].Hits
-	for _, r := range recs {
-		r.Hits += prevHits
-	}
-
-	return nil
-}
-
-// requestKey returns a key for identifying a request.
-func requestKey(name string, qt dnsmsg.RRType) (key string) {
-	return name + "_" + dns.TypeToString[qt]
-}
-
-// rotate moves the current DB file to a temporary file and returns the path to
-// that temporary file.
-func (db *Bolt) rotate() (prev string, err error) {
-	prevBase := fmt.Sprintf("%s.%d", filepath.Base(db.path), time.Now().Unix())
-	prev = filepath.Join(os.TempDir(), prevBase)
-	err = os.Rename(db.path, prev)
-	if err != nil {
-		return "", fmt.Errorf("renaming prev db: %w", err)
-	}
-
-	log.Info("dnsdb: renamed %q to %q", db.path, prev)
-
-	metrics.DNSDBSize.Set(0)
-	metrics.DNSDBRotateTime.SetToCurrentTime()
-
-	return prev, nil
-}

internal/dnsdb/bolthttp.go

@@ -1,139 +0,0 @@
-package dnsdb
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"encoding/csv"
-	"encoding/gob"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"strings"
-
-	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/httphdr"
-	"go.etcd.io/bbolt"
-)
-
-// BoltDB-Based DNSDB HTTP Handler
-
-// type check
-var _ http.Handler = (*Bolt)(nil)
-
-// ServeHTTP implements the http.Handler interface for *Bolt.
-func (db *Bolt) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	db.dumpMu.Lock()
-	defer db.dumpMu.Unlock()
-
-	dbPath, err := db.flush(ctx, true)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-
-		return
-	}
-
-	h := w.Header()
-	h.Add(httphdr.ContentType, agdhttp.HdrValTextCSV)
-
-	if dbPath == "" {
-		// No data.
-		w.WriteHeader(http.StatusOK)
-
-		return
-	}
-
-	h.Set(httphdr.Trailer, httphdr.XError)
-	defer func() {
-		if err != nil {
-			h.Set(httphdr.XError, err.Error())
-			agd.Collectf(ctx, db.errColl, "dnsdb: http handler error: %w", err)
-		}
-	}()
-
-	defer func() { err = errors.WithDeferred(err, os.Remove(dbPath)) }()
-
-	var rw io.Writer = w
-	// TODO(a.garipov): Consider parsing the quality value.
-	if strings.Contains(r.Header.Get(httphdr.AcceptEncoding), "gzip") {
-		h.Set(httphdr.ContentEncoding, "gzip")
-		gw := gzip.NewWriter(w)
-		defer func() { err = errors.WithDeferred(err, gw.Close()) }()
-
-		rw = gw
-	}
-
-	w.WriteHeader(http.StatusOK)
-
-	err = db.dumpToCSV(ctx, rw, dbPath)
-}
-
-// dumpToCSV converts the DNSDB at dbPath to CSV and writes it into w.  It
-// writes every record as it processes it.
-func (db *Bolt) dumpToCSV(ctx context.Context, w io.Writer, dbPath string) (err error) {
-	bdb, err := openBolt(dbPath)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("opening boltdb: %w", err)
-	}
-	defer func() { err = errors.WithDeferred(err, bdb.Close()) }()
-
-	err = bdb.Batch(boltDumpFunc(ctx, w, db.errColl))
-	if err != nil {
-		return fmt.Errorf("dumping db: %w", err)
-	}
-
-	return nil
-}
-
-// boltDumpFunc returns a function that reads data from the existing DNSDB file
-// and writes it into w as CSV records, one at a time.
-//
-// Decoding errors are reported using errColl.
-func boltDumpFunc(ctx context.Context, w io.Writer, errColl agd.ErrorCollector) (f boltTxFunc) {
-	return func(tx *bbolt.Tx) (err error) {
-		b := tx.Bucket([]byte(recordsBucket))
-		if b == nil {
-			return errors.Error("records bucket not found")
-		}
-
-		csvw := csv.NewWriter(w)
-		defer csvw.Flush()
-
-		c := b.Cursor()
-		for rk, v := c.First(); rk != nil; rk, v = c.Next() {
-			var recs []*record
-			err = gob.NewDecoder(bytes.NewReader(v)).Decode(&recs)
-			if err != nil {
-				agd.Collectf(ctx, errColl, "dnsdb: decoding data for %s: %w", rk, err)
-
-				continue
-			}
-
-			err = writeCSVRecs(csvw, recs)
-			if err != nil {
-				return fmt.Errorf("writing record for key %s: %w", rk, err)
-			}
-		}
-
-		return nil
-	}
-}
-
-// writeCSVRecs writes the CSV representation of recs into w.
-func writeCSVRecs(w *csv.Writer, recs []*record) (err error) {
-	for i, r := range recs {
-		err = w.Write(r.csv())
-		if err != nil {
-			return fmt.Errorf("record at index %d: %w", i, err)
-		}
-	}
-
-	return nil
-}

internal/dnsdb/buffer.go

@@ -0,0 +1,106 @@
+package dnsdb
+
+import (
+	"sync"
+
+	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
+	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
+	"github.com/miekg/dns"
+)
+
+// buffer contains the approximate statistics for DNS answers.  It saves data
+// until it reaches maxSize, upon which it can only increase the hits of the
+// previous records.
+type buffer struct {
+	// mu protects entries.
+	mu *sync.Mutex
+
+	// entries is the data of the statistics.
+	entries map[recordKey]*recordValue
+
+	// maxSize is the maximum length of entries.
+	maxSize int
+}
+
+// add increments the records for all answers.
+func (b *buffer) add(target string, answers []dns.RR, qt dnsmsg.RRType, rc dnsmsg.RCode) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Do nothing if the buffer is already full.
+	l := len(b.entries)
+	if l >= b.maxSize {
+		return
+	}
+
+	key := recordKey{
+		target: target,
+		qt:     qt,
+	}
+
+	prev, ok := b.entries[key]
+	if ok {
+		prev.hits++
+
+		// Note, that only the first set of answers is stored in the buffer.
+		// If a more detailed response is needed, maps.Copy can be used to
+		// achieve that.
+
+		return
+	}
+
+	b.entries[key] = &recordValue{
+		answers: toAnswerSet(answers, rc),
+		hits:    1,
+	}
+
+	metrics.DNSDBBufferSize.Set(float64(l + 1))
+}
+
+// all returns buffered records.
+func (b *buffer) all() (records []*record) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	for key, val := range b.entries {
+		if len(val.answers) == 0 {
+			records = append(records, &record{
+				target: key.target,
+				hits:   val.hits,
+				rrType: key.qt,
+			})
+
+			continue
+		}
+
+		for a := range val.answers {
+			records = append(records, &record{
+				target: key.target,
+				answer: a.value,
+				hits:   val.hits,
+				rrType: a.rrType,
+				rcode:  a.rcode,
+			})
+		}
+	}
+
+	return records
+}
+
+// toAnswerSet converts a slice of [dns.RR] to a map that can easier be
+// serialized to a csv.
+func toAnswerSet(answers []dns.RR, rc dnsmsg.RCode) (answerSet map[recordAnswer]unit) {
+	answerSet = map[recordAnswer]unit{}
+	for _, a := range answers {
+		ansStr := answerString(a)
+		if ansStr != "" {
+			answerSet[recordAnswer{
+				value:  ansStr,
+				rrType: a.Header().Rrtype,
+				rcode:  rc,
+			}] = unit{}
+		}
+	}
+
+	return answerSet
+}

internal/dnsdb/dnsdb.go

@@ -1,12 +1,19 @@
 // Package dnsdb contains types and utilities for collecting anonymous
 // statistics about the Internet.
+//
+// TODO(a.garipov): This needs way more tests.
 package dnsdb
 
 import (
 	"context"
+	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdnet"
+	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
+	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
 	"github.com/miekg/dns"
 )
 
@@ -27,7 +34,77 @@ var _ Interface = Empty{}
 // Record implements the Interface interface for Empty.
 func (Empty) Record(_ context.Context, _ *dns.Msg, _ *agd.RequestInfo) {}
 
-// isIgnoredMessage returns true if m must be ignored by DNSDB.
+// Default is the default DNSDB implementation.
+type Default struct {
+	buffer  *atomic.Pointer[buffer]
+	errColl agd.ErrorCollector
+	maxSize int
+}
+
+// DefaultConfig is the default DNS database configuration structure.
+type DefaultConfig struct {
+	// ErrColl is used to collect HTTP errors.
+	ErrColl agd.ErrorCollector
+
+	// MaxSize is the maximum amount of records in the memory buffer.
+	MaxSize int
+}
+
+// New creates a new default DNS database.  c must not be nil.
+func New(c *DefaultConfig) (db *Default) {
+	db = &Default{
+		buffer:  &atomic.Pointer[buffer]{},
+		errColl: c.ErrColl,
+		maxSize: c.MaxSize,
+	}
+
+	db.buffer.Store(&buffer{
+		mu:      &sync.Mutex{},
+		entries: map[recordKey]*recordValue{},
+		maxSize: db.maxSize,
+	})
+
+	return db
+}
+
+// type check
+var _ Interface = (*Default)(nil)
+
+// Record implements the Interface interface for *Default.  It saves a DNS
+// response to its in-memory buffer.
+func (db *Default) Record(ctx context.Context, m *dns.Msg, ri *agd.RequestInfo) {
+	if isIgnoredMessage(m) {
+		return
+	}
+
+	q := m.Question[0]
+	if isIgnoredQuestion(q) {
+		return
+	}
+
+	db.buffer.Load().add(ri.Host, m.Answer, q.Qtype, dnsmsg.RCode(m.Rcode))
+}
+
+// reset returns buffered records and resets the database.
+func (db *Default) reset() (records []*record) {
+	start := time.Now()
+
+	prevBuf := db.buffer.Swap(&buffer{
+		mu:      &sync.Mutex{},
+		entries: map[recordKey]*recordValue{},
+		maxSize: db.maxSize,
+	})
+
+	records = prevBuf.all()
+
+	metrics.DNSDBBufferSize.Set(0)
+	metrics.DNSDBRotateTime.SetToCurrentTime()
+	metrics.DNSDBSaveDuration.Observe(time.Since(start).Seconds())
+
+	return records
+}
+
+// isIgnoredMessage returns true if m must be ignored by DNS database.
 func isIgnoredMessage(m *dns.Msg) (ok bool) {
 	return m == nil ||
 		!m.Response ||
@@ -35,7 +112,7 @@ func isIgnoredMessage(m *dns.Msg) (ok bool) {
 		m.Rcode != dns.RcodeSuccess
 }
 
-// isIgnoredQuestion returns true if q must be ignored by DNSDB.
+// isIgnoredQuestion returns true if q must be ignored by DNS database.
 func isIgnoredQuestion(q dns.Question) (ok bool) {
 	return (q.Qtype != dns.TypeA && q.Qtype != dns.TypeAAAA) ||
 		// Android metric domain must be ignored by DNSDB to avoid filling it

internal/dnsdb/http.go

@@ -0,0 +1,68 @@
+package dnsdb
+
+import (
+	"compress/gzip"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
+	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/httphdr"
+)
+
+// Default DNS database HTTP Handler
+
+// type check
+var _ http.Handler = (*Default)(nil)
+
+// ServeHTTP implements the http.Handler interface for *Default.
+func (db *Default) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var err error
+	ctx := r.Context()
+
+	records := db.reset()
+
+	h := w.Header()
+	h.Add(httphdr.ContentType, agdhttp.HdrValTextCSV)
+
+	h.Set(httphdr.Trailer, httphdr.XError)
+	defer func() {
+		if err != nil {
+			h.Set(httphdr.XError, err.Error())
+			agd.Collectf(ctx, db.errColl, "dnsdb: http handler error: %w", err)
+		}
+	}()
+
+	var rw io.Writer = w
+	// TODO(a.garipov): Consider parsing the quality value.
+	if strings.Contains(r.Header.Get(httphdr.AcceptEncoding), "gzip") {
+		h.Set(httphdr.ContentEncoding, "gzip")
+		gw := gzip.NewWriter(w)
+		defer func() { err = errors.WithDeferred(err, gw.Close()) }()
+
+		rw = gw
+	}
+
+	w.WriteHeader(http.StatusOK)
+
+	csvw := csv.NewWriter(rw)
+	defer csvw.Flush()
+
+	err = writeCSVRecs(csvw, records)
+}
+
+// writeCSVRecs writes the CSV representation of recs into w.
+func writeCSVRecs(w *csv.Writer, recs []*record) (err error) {
+	for i, r := range recs {
+		err = w.Write(r.csv())
+		if err != nil {
+			return fmt.Errorf("record at index %d: %w", i, err)
+		}
+	}
+
+	return nil
+}

internal/dnsdb/http_test.go

@@ -1,13 +1,14 @@
 package dnsdb_test
 
 import (
+	"bytes"
 	"compress/gzip"
 	"context"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"os"
 	"strings"
 	"testing"
 
@@ -22,23 +23,9 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// newTmpBolt creates a *dnsdb.Bolt with temporary DB file.
+func TestDefault_ServeHTTP(t *testing.T) {
-func newTmpBolt(t *testing.T) (db *dnsdb.Bolt) {
-	tmpFile, err := os.CreateTemp(t.TempDir(), "*")
-	require.NoError(t, err)
-
-	conf := &dnsdb.BoltConfig{
-		ErrColl: &agdtest.ErrorCollector{
-			OnCollect: func(_ context.Context, _ error) { panic("not implemented") },
-		},
-		Path: tmpFile.Name(),
-	}
-
-	return dnsdb.NewBolt(conf)
-}
-
-func TestBolt_ServeHTTP(t *testing.T) {
 	const dname = "some-domain.name"
+	testIP := net.IP{1, 2, 3, 4}
 
 	successHdr := http.Header{
 		httphdr.ContentType:     []string{agdhttp.HdrValTextCSV},
@@ -47,21 +34,27 @@ func TestBolt_ServeHTTP(t *testing.T) {
 	}
 
 	newMsg := func(rcode int, name string, qtype uint16) (m *dns.Msg) {
-		return dnsservertest.NewResp(rcode, dnsservertest.NewReq(name, qtype, dns.ClassINET))
+		return dnsservertest.NewResp(
+			rcode,
+			dnsservertest.NewReq(name, qtype, dns.ClassINET),
+			dnsservertest.SectionAnswer{
+				dnsservertest.NewA(dname, 0, testIP),
+			},
+		)
 	}
 
 	testCases := []struct {
 		name     string
 		msgs     []*dns.Msg
 		wantHdr  http.Header
-		wantResp []byte
+		wantResp [][]byte
 	}{{
 		name: "single",
 		msgs: []*dns.Msg{
 			newMsg(dns.RcodeSuccess, dname, dns.TypeA),
 		},
 		wantHdr:  successHdr,
-		wantResp: []byte(dname + `,A,NOERROR,,1` + "\n"),
+		wantResp: [][]byte{[]byte(dname + `,A,NOERROR,` + testIP.String() + `,1`)},
 	}, {
 		name: "existing",
 		msgs: []*dns.Msg{
@@ -69,7 +62,7 @@ func TestBolt_ServeHTTP(t *testing.T) {
 			newMsg(dns.RcodeSuccess, dname, dns.TypeA),
 		},
 		wantHdr:  successHdr,
-		wantResp: []byte(dname + `,A,NOERROR,,2` + "\n"),
+		wantResp: [][]byte{[]byte(dname + `,A,NOERROR,` + testIP.String() + `,2`)},
 	}, {
 		name: "different",
 		msgs: []*dns.Msg{
@@ -77,8 +70,10 @@ func TestBolt_ServeHTTP(t *testing.T) {
 			newMsg(dns.RcodeSuccess, "sub."+dname, dns.TypeA),
 		},
 		wantHdr: successHdr,
-		wantResp: []byte(dname + `,A,NOERROR,,1` + "\n" +
+		wantResp: [][]byte{
-			"sub." + dname + `,A,NOERROR,,1` + "\n"),
+			[]byte("sub." + dname + `,A,NOERROR,` + testIP.String() + `,1`),
+			[]byte(dname + `,A,NOERROR,` + testIP.String() + `,1`),
+		},
 	}, {
 		name: "non-recordable",
 		msgs: []*dns.Msg{
@@ -90,15 +85,12 @@ func TestBolt_ServeHTTP(t *testing.T) {
 			newMsg(dns.RcodeSuccess, dname+"-dnsotls-ds.metric.gstatic.com.", dns.TypeA),
 		},
 		wantHdr:  successHdr,
-		wantResp: []byte{},
+		wantResp: [][]byte{},
 	}}
 
-	recordAndRefresh := func(
+	record := func(
 		t *testing.T,
-		db interface {
+		db dnsdb.Interface,
-			dnsdb.Interface
-			agd.Refresher
-		},
 		msgs []*dns.Msg,
 	) {
 		t.Helper()
@@ -111,9 +103,6 @@ func TestBolt_ServeHTTP(t *testing.T) {
 				// See [dnssvc.initMw.newRequestInfo].
 				Host: strings.TrimSuffix(m.Question[0].Name, "."),
 			})
-
-			err := db.Refresh(context.Background())
-			require.NoError(t, err)
 		}
 	}
 
@@ -125,11 +114,16 @@ func TestBolt_ServeHTTP(t *testing.T) {
 	r.Header.Add(httphdr.AcceptEncoding, "gzip")
 
 	for _, tc := range testCases {
-		db := newTmpBolt(t)
+		db := dnsdb.New(&dnsdb.DefaultConfig{
+			ErrColl: &agdtest.ErrorCollector{
+				OnCollect: func(_ context.Context, _ error) { panic("not implemented") },
+			},
+			MaxSize: 100,
+		})
 		rw := httptest.NewRecorder()
 
 		t.Run(tc.name, func(t *testing.T) {
-			recordAndRefresh(t, db, tc.msgs)
+			record(t, db, tc.msgs)
 
 			db.ServeHTTP(rw, r)
 			require.Equal(t, http.StatusOK, rw.Code)
@@ -143,25 +137,8 @@ func TestBolt_ServeHTTP(t *testing.T) {
 			decResp, err = io.ReadAll(gzipr)
 			require.NoError(t, err)
 
-			assert.Equal(t, tc.wantResp, decResp)
+			lines := bytes.Split(decResp, []byte("\n"))
+			assert.ElementsMatch(t, tc.wantResp, lines[:len(lines)-1])
 		})
 	}
-
-	t.Run("bad_db_path", func(t *testing.T) {
-		db := dnsdb.NewBolt(&dnsdb.BoltConfig{
-			Path: "bad/path",
-			ErrColl: &agdtest.ErrorCollector{
-				OnCollect: func(ctx context.Context, err error) { panic("not implemented") },
-			},
-		})
-
-		w := httptest.NewRecorder()
-
-		db.ServeHTTP(w, r)
-		assert.Equal(
-			t,
-			"opening boltdb: opening file: open bad/path: no such file or directory\n",
-			w.Body.String(),
-		)
-	})
 }

internal/dnsdb/record.go

@@ -1,8 +1,6 @@
 package dnsdb
 
 import (
-	"bytes"
-	"encoding/gob"
 	"strconv"
 	"strings"
 
@@ -10,35 +8,30 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DNSDB Records
+// record is a single DNSDB record as it is stored in the record's database.
-
-// record is a single DNSDB record as it is stored in the BoldDB database.
-//
-// DO NOT change the names of the fields, since gob is used to encode it.
 type record struct {
-	// DomainName is the question FQDN from the request.
+	// target is the question target from the request.
-	DomainName string
+	target string
 
-	// Answer is either the IP address (for A and AAAA responses) or the
+	// answer is either the IP address (for A and AAAA responses) or the
 	// hostname (for CNAME responses).
 	//
 	// If there are no answers, this field is empty.
-	Answer string
+	answer string
 
-	// Hits shows how many times this domain was requested.  All records with
+	// hits shows how many times this domain was requested.
-	// the same DomainName share this value.
+	hits uint64
-	Hits uint64
 
-	// RRType is the resource record type of the answer.  Currently, only A,
+	// rrType is the resource record type of the answer.  Currently, only A,
 	// AAAA, and CNAME responses are recorded.
 	//
-	// If there are no answers, RRType is the type of the resource record type
+	// If there are no answers, rrType is the type of the resource record type
 	// of the question instead.
-	RRType dnsmsg.RRType
+	rrType dnsmsg.RRType
 
-	// RCode is the response code.  Currently we only record successful queries,
+	// rcode is the response code.  Currently we only record successful queries,
 	// but that may change if the future.
-	RCode dnsmsg.RCode
+	rcode dnsmsg.RCode
 }
 
 // csv returns CSV fields containing the record's information in the predefined
@@ -47,86 +40,46 @@ func (r *record) csv() (fields []string) {
 	// DO NOT change the order of fields, since other parts of the system depend
 	// on it.
 	return []string{
-		r.DomainName,
+		r.target,
-		dns.TypeToString[r.RRType],
+		dns.TypeToString[r.rrType],
-		dns.RcodeToString[int(r.RCode)],
+		dns.RcodeToString[int(r.rcode)],
-		r.Answer,
+		r.answer,
-		strconv.FormatUint(r.Hits, 10),
+		strconv.FormatUint(r.hits, 10),
 	}
 }
 
-// encode encodes a slice of DNSDB records using gob.
+// answerString returns a string representation of an answer record.
-func encode(recs []*record) (b []byte, err error) {
+func answerString(rr dns.RR) (s string) {
-	buf := &bytes.Buffer{}
-	err = gob.NewEncoder(buf).Encode(recs)
-	if err != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}
-
-// decode decodes a slice of DNSDB records from gob data.
-func decode(b []byte) (recs []*record, err error) {
-	r := bytes.NewReader(b)
-	err = gob.NewDecoder(r).Decode(&recs)
-	if err != nil {
-		return nil, err
-	}
-
-	return recs, nil
-}
-
-// toDBRecords converts DNS query data into a slice of DNSDB records.
-func toDBRecords(qt dnsmsg.RRType, name string, ans []dns.RR, rc dnsmsg.RCode) (recs []*record) {
-	if len(ans) == 0 {
-		return []*record{newDBRecord(qt, name, nil, rc)}
-	}
-
-	for _, rr := range ans {
-		if isAcceptedRRType(rr) {
-			recs = append(recs, newDBRecord(qt, name, rr, rc))
-		}
-	}
-
-	return recs
-}
-
-// isAcceptedRRType returns true if rr has one of the accepted answer resource
-// record types.
-func isAcceptedRRType(rr dns.RR) (ok bool) {
-	switch rr.(type) {
-	case *dns.A, *dns.AAAA, *dns.CNAME:
-		return true
-	default:
-		return false
-	}
-}
-
-// newDBRecord converts the DNS message data to a DNSDB record.
-func newDBRecord(qt dnsmsg.RRType, name string, rr dns.RR, rc dnsmsg.RCode) (rec *record) {
-	rec = &record{
-		DomainName: name,
-		Hits:       1,
-		RCode:      rc,
-	}
-
-	if rr == nil {
-		rec.RRType = qt
-
-		return rec
-	}
-
-	rec.RRType = rr.Header().Rrtype
-
 	switch v := rr.(type) {
 	case *dns.A:
-		rec.Answer = v.A.String()
+		return v.A.String()
 	case *dns.AAAA:
-		rec.Answer = v.AAAA.String()
+		return v.AAAA.String()
 	case *dns.CNAME:
-		rec.Answer = strings.TrimSuffix(v.Target, ".")
+		return strings.TrimSuffix(v.Target, ".")
+	default:
+		return ""
+	}
 }
 
-	return rec
+// recordKey is the key a DNSDB entry.
+type recordKey struct {
+	target string
+	qt     dnsmsg.RRType
+}
+
+// unit is a convenient alias for struct{}.
+type unit = struct{}
+
+// recordValue contains the values for a single record key.
+type recordValue struct {
+	answers map[recordAnswer]unit
+	hits    uint64
+}
+
+// recordAnswer contains a single piece of the answer data.
+type recordAnswer struct {
+	value  string
+	rrType dnsmsg.RRType
+	rcode  dnsmsg.RCode
 }

internal/dnsmsg/dnsmsg.go

@@ -6,8 +6,10 @@ package dnsmsg
 
 import (
 	"fmt"
+	"math"
 	"net/netip"
 
+	"github.com/AdguardTeam/golibs/mathutil"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/miekg/dns"
 )
@@ -143,3 +145,66 @@ func ecsData(esn *dns.EDNS0_SUBNET) (subnet netip.Prefix, scope uint8, err error
 
 	return subnet, esn.SourceScope, nil
 }
+
+// SetMinTTL overrides TTL values of all answer records according to the min
+// TTL.
+func SetMinTTL(r *dns.Msg, minTTL uint32) {
+	for _, rr := range r.Answer {
+		h := rr.Header()
+
+		// TODO(d.kolyshev): Use built-in max in go 1.21.
+		h.Ttl = mathutil.Max(h.Ttl, minTTL)
+	}
+}
+
+// ServFailMaxCacheTTL is the maximum time-to-live value for caching
+// SERVFAIL responses in seconds.  It's consistent with the upper constraint
+// of 5 minutes given by RFC 2308.
+//
+// See https://datatracker.ietf.org/doc/html/rfc2308#section-7.1.
+const ServFailMaxCacheTTL = 30
+
+// FindLowestTTL gets the lowest TTL among all DNS message's RRs.
+func FindLowestTTL(msg *dns.Msg) (ttl uint32) {
+	// Use the maximum value as a guard value.  If the inner loop is entered,
+	// it's going to be rewritten with an actual TTL value that is lower than
+	// MaxUint32.  If the inner loop isn't entered, catch that and return zero.
+	ttl = math.MaxUint32
+	for _, rrs := range [][]dns.RR{msg.Answer, msg.Ns, msg.Extra} {
+		for _, rr := range rrs {
+			ttl = getTTLIfLower(rr, ttl)
+			if ttl == 0 {
+				return 0
+			}
+		}
+	}
+
+	switch {
+	case msg.Rcode == dns.RcodeServerFailure && ttl > ServFailMaxCacheTTL:
+		return ServFailMaxCacheTTL
+	case ttl == math.MaxUint32:
+		return 0
+	default:
+		return ttl
+	}
+}
+
+// getTTLIfLower is a helper function that checks the TTL of the specified RR
+// and returns it if it's lower than the one passed in the arguments.
+func getTTLIfLower(r dns.RR, ttl uint32) (res uint32) {
+	switch r := r.(type) {
+	case *dns.OPT:
+		// Don't even consider the OPT RRs TTL.
+		return ttl
+	case *dns.SOA:
+		if r.Minttl > 0 && r.Minttl < ttl {
+			// Per RFC 2308, the TTL of a SOA RR is the minimum of SOA.MINIMUM
+			// field and the header's value.
+			ttl = r.Minttl
+		}
+	default:
+		// Go on.
+	}
+
+	return mathutil.Min(r.Header().Ttl, ttl)
+}

internal/dnsserver/cache/cache.go

@@ -14,17 +14,26 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/golibs/mathutil"
 	"github.com/bluele/gcache"
 	"github.com/miekg/dns"
 )
 
 // Middleware is a simple DNS caching middleware with no ECS support.
+//
+// TODO(a.garipov): Extract cache logic to golibs.
 type Middleware struct {
 	// metrics is a listener for the middleware events.
 	metrics MetricsListener
 
 	// cache is the underlying LRU cache.
 	cache gcache.Cache
+
+	// cacheMinTTL is the minimum supported TTL for cache items.
+	cacheMinTTL time.Duration
+
+	// useTTLOverride shows if the TTL overrides logic should be used.
+	useTTLOverride bool
 }
 
 // MiddlewareConfig is the configuration structure for NewMiddleware.
@@ -37,6 +46,12 @@ type MiddlewareConfig struct {
 	// Size is the number of entities to hold in the cache.  It must be greater
 	// than zero.
 	Size int
+
+	// MinTTL is the minimum supported TTL for cache items.
+	MinTTL time.Duration
+
+	// UseTTLOverride shows if the TTL overrides logic should be used.
+	UseTTLOverride bool
 }
 
 // NewMiddleware initializes a new LRU caching middleware.  c must not be nil.
@@ -51,6 +66,8 @@ func NewMiddleware(c *MiddlewareConfig) (m *Middleware) {
 	return &Middleware{
 		metrics:        metrics,
 		cache:          gcache.New(c.Size).LRU().Build(),
+		cacheMinTTL:    c.MinTTL,
+		useTTLOverride: c.UseTTLOverride,
 	}
 }
 
@@ -128,15 +145,22 @@ func (m *Middleware) set(msg *dns.Msg) (err error) {
 		return nil
 	}
 
-	ttl := m.findLowestTTL(msg)
+	ttl := findLowestTTL(msg)
 	if ttl == 0 || !isCacheable(msg) {
 		return nil
 	}
 
+	exp := time.Duration(ttl) * time.Second
+	if m.useTTLOverride && msg.Rcode != dns.RcodeServerFailure {
+		// TODO(d.kolyshev): Use built-in max in go 1.21.
+		exp = mathutil.Max(exp, m.cacheMinTTL)
+		setMinTTL(msg, uint32(exp.Seconds()))
+	}
+
 	key := toCacheKey(msg)
 	i := m.toCacheItem(msg)
 
-	return m.cache.SetWithExpire(key, i, time.Duration(ttl)*time.Second)
+	return m.cache.SetWithExpire(key, i, exp)
 }
 
 // toCacheKey returns the cache key for msg.  msg must have one question record.
@@ -236,8 +260,19 @@ func isCacheableNOERROR(resp *dns.Msg) (ok bool) {
 	return false
 }
 
+// setMinTTL overrides TTL values of all answer records according to the min
+// TTL.
+func setMinTTL(r *dns.Msg, minTTL uint32) {
+	for _, rr := range r.Answer {
+		h := rr.Header()
+
+		// TODO(d.kolyshev): Use built-in max in go 1.21.
+		h.Ttl = mathutil.Max(h.Ttl, minTTL)
+	}
+}
+
 // findLowestTTL gets the lowest TTL among all DNS message's RRs.
-func (m *Middleware) findLowestTTL(msg *dns.Msg) (ttl uint32) {
+func findLowestTTL(msg *dns.Msg) (ttl uint32) {
 	// servFailMaxCacheTTL is the maximum time-to-live value for caching
 	// SERVFAIL responses in seconds.  It's consistent with the upper constraint
 	// of 5 minutes given by the RFC 2308.
@@ -321,7 +356,7 @@ func (m *Middleware) fromCacheItem(item cacheItem, req *dns.Msg) (msg *dns.Msg)
 
 	// Update all the TTL of all depending on when the item was cached.  If it's
 	// already expired, update TTL to 0.
-	newTTL := m.findLowestTTL(item.msg)
+	newTTL := findLowestTTL(item.msg)
 	if timeLeft := math.Round(float64(newTTL) - time.Since(item.when).Seconds()); timeLeft > 0 {
 		newTTL = uint32(timeLeft)
 	}

internal/dnsserver/cache/cache_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"testing"
+	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/cache"
@@ -15,75 +16,101 @@ import (
 
 func TestMiddleware_Wrap(t *testing.T) {
 	const (
+		servFailMaxCacheTTL = 30
+
 		reqHostname = "example.com"
 		reqCname    = "cname.example.com"
 		reqNs1      = "ns1.example.com"
 		reqNs2      = "ns2.example.com"
+
+		defaultTTL uint32 = 3600
 	)
 
+	testTTL := 60 * time.Second
+
 	aReq := dnsservertest.NewReq(reqHostname, dns.TypeA, dns.ClassINET)
 	cnameReq := dnsservertest.NewReq(reqHostname, dns.TypeCNAME, dns.ClassINET)
-	cnameAns := dnsservertest.SectionAnswer{dnsservertest.NewCNAME(reqHostname, 3600, reqCname)}
+	cnameAns := dnsservertest.SectionAnswer{dnsservertest.NewCNAME(reqHostname, defaultTTL, reqCname)}
-	soaNs := dnsservertest.SectionNs{dnsservertest.NewSOA(reqHostname, 3600, reqNs1, reqNs2)}
+	soaNs := dnsservertest.SectionNs{dnsservertest.NewSOA(reqHostname, defaultTTL, reqNs1, reqNs2)}
 
 	const N = 5
 	testCases := []struct {
 		req        *dns.Msg
 		resp       *dns.Msg
 		name       string
+		minTTL     *time.Duration
 		wantNumReq int
+		wantTTL    uint32
 	}{{
 		req: aReq,
 		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
-			dnsservertest.NewA(reqHostname, 3600, net.IP{1, 2, 3, 4}),
+			dnsservertest.NewA(reqHostname, defaultTTL, net.IP{1, 2, 3, 4}),
 		}),
 		name:       "simple_a",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq),
 		name:       "empty_answer",
 		wantNumReq: N,
+		minTTL:     nil,
+		wantTTL:    0,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, soaNs),
 		name:       "authoritative_nodata",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, cnameAns, soaNs),
 		name:       "nodata_with_cname",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, cnameAns),
 		name:       "nodata_with_cname_no_soa",
 		wantNumReq: N,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req: aReq,
 		resp: dnsservertest.NewResp(dns.RcodeNameError, aReq, dnsservertest.SectionNs{
-			dnsservertest.NewNS(reqHostname, 3600, reqNs1),
+			dnsservertest.NewNS(reqHostname, defaultTTL, reqNs1),
 		}),
 		name: "non_authoritative_nxdomain",
 		// TODO(ameshkov): Consider https://datatracker.ietf.org/doc/html/rfc2308#section-3.
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeNameError, aReq, soaNs),
 		name:       "authoritative_nxdomain",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeServerFailure, aReq),
 		name:       "simple_server_failure",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    servFailMaxCacheTTL,
 	}, {
 		req: cnameReq,
 		resp: dnsservertest.NewResp(dns.RcodeSuccess, cnameReq, dnsservertest.SectionAnswer{
-			dnsservertest.NewCNAME(reqHostname, 3600, reqCname),
+			dnsservertest.NewCNAME(reqHostname, defaultTTL, reqCname),
 		}),
 		name:       "simple_cname_ans",
 		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}, {
 		req: aReq,
 		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
@@ -91,11 +118,51 @@ func TestMiddleware_Wrap(t *testing.T) {
 		}),
 		name:       "expired_one",
 		wantNumReq: N,
+		minTTL:     nil,
+		wantTTL:    0,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 10, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_ok",
+		wantNumReq: 1,
+		minTTL:     &testTTL,
+		wantTTL:    uint32(testTTL.Seconds()),
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 1000, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_max",
+		wantNumReq: 1,
+		minTTL:     &testTTL,
+		wantTTL:    1000,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 0, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_zero",
+		wantNumReq: N,
+		minTTL:     &testTTL,
+		wantTTL:    0,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeServerFailure, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, servFailMaxCacheTTL, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_servfail",
+		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    servFailMaxCacheTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeNotImplemented, aReq, soaNs),
 		name:       "unexpected_response",
 		wantNumReq: N,
+		minTTL:     nil,
+		wantTTL:    defaultTTL,
 	}}
 
 	for _, tc := range testCases {
@@ -109,10 +176,17 @@ func TestMiddleware_Wrap(t *testing.T) {
 				},
 			)
 
+			var minTTL time.Duration
+			if tc.minTTL != nil {
+				minTTL = *tc.minTTL
+			}
+
 			withCache := dnsserver.WithMiddlewares(
 				handler,
 				cache.NewMiddleware(&cache.MiddlewareConfig{
 					Size:           100,
+					MinTTL:         minTTL,
+					UseTTLOverride: tc.minTTL != nil,
 				}),
 			)
 
@@ -126,8 +200,13 @@ func TestMiddleware_Wrap(t *testing.T) {
 
 			require.NoError(t, err)
 
-			assert.Equal(t, tc.resp, nrw.Msg())
+			m := nrw.Msg()
+			assert.Equal(t, tc.resp, m)
 			assert.Equal(t, tc.wantNumReq, numReq)
+
+			if len(m.Answer) > 0 {
+				assert.Equal(t, tc.wantTTL, m.Answer[0].Header().Ttl)
+			}
 		})
 	}
 }

internal/dnsserver/dnsservertest/msg.go

@@ -168,6 +168,30 @@ func NewAAAA(name string, ttl uint32, aaaa net.IP) (rr dns.RR) {
 	}
 }
 
+// NewHTTPS constructs the new resource record of type HTTPS with IPv4 and IPv6
+// hint records from provided v4Hint and v6Hint parameters.
+//
+// TODO(d.kolyshev): Add "alpn" and other SVCB key-value pairs.
+func NewHTTPS(name string, ttl uint32, v4Hint, v6Hint []net.IP) (rr dns.RR) {
+	svcb := dns.SVCB{
+		Hdr: dns.RR_Header{
+			Name:   dns.Fqdn(name),
+			Rrtype: dns.TypeHTTPS,
+			Class:  dns.ClassINET,
+			Ttl:    ttl,
+		},
+		Target: dns.Fqdn(name),
+		Value: []dns.SVCBKeyValue{
+			&dns.SVCBIPv4Hint{Hint: v4Hint},
+			&dns.SVCBIPv6Hint{Hint: v6Hint},
+		},
+	}
+
+	return &dns.HTTPS{
+		SVCB: svcb,
+	}
+}
+
 // NewSOA constructs the new resource record of type SOA.
 func NewSOA(name string, ttl uint32, ns, mbox string) (rr dns.RR) {
 	return &dns.SOA{

internal/dnsserver/forward/forward.go

@@ -23,7 +23,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
 	"net"
 	"net/netip"
 	"sync/atomic"
@@ -32,6 +31,7 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/miekg/dns"
+	"golang.org/x/exp/rand"
 )
 
 // Handler is a struct that implements [dnsserver.Handler] and forwards DNS
@@ -44,6 +44,11 @@ type Handler struct {
 	// upstream is the main upstream where this handler forwards DNS queries.
 	upstream Upstream
 
+	// rand is a random-number generator that is not cryptographically secure
+	// and is used for randomized upstream selection and other non-sensitive
+	// tasks.
+	rand *rand.Rand
+
 	// hcDomainTmpl is the template for domains used to perform healthcheck
 	// requests.
 	hcDomainTmpl string
@@ -123,11 +128,14 @@ func NewHandler(c *HandlerConfig) (h *Handler) {
 			Network: c.Network,
 			Address: c.Address,
 		}),
+		rand:         rand.New(&rand.LockedSource{}),
 		hcDomainTmpl: c.HealthcheckDomainTmpl,
 		timeout:      c.Timeout,
 		hcBackoff:    c.HealthcheckBackoffDuration,
 	}
 
+	h.rand.Seed(uint64(time.Now().UnixNano()))
+
 	if l := c.MetricsListener; l != nil {
 		h.metrics = l
 	} else {
@@ -196,9 +204,7 @@ func (h *Handler) ServeDNS(
 	}
 
 	if useFallbacks && len(h.fallbacks) > 0 {
-		// #nosec G404 -- We don't need a real random for a simple fallbacks
+		i := h.rand.Intn(len(h.fallbacks))
-		// rotation, we just need a simple fast pseudo-random.
-		i := rand.Intn(len(h.fallbacks))
 		f := h.fallbacks[i]
 		resp, err = h.exchange(ctx, f, req)
 	}

internal/dnsserver/forward/healthcheck.go

@@ -3,7 +3,6 @@ package forward
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"strconv"
 	"strings"
 	"time"
@@ -74,9 +73,7 @@ const randomPlaceholder = "${RANDOM}"
 func (h *Handler) healthcheck(ctx context.Context) (err error) {
 	domain := h.hcDomainTmpl
 	if strings.Contains(domain, randomPlaceholder) {
-		// #nosec G404 -- We don't need a real random for generating randomized
+		randStr := strconv.FormatUint(h.rand.Uint64(), 16)
-		// domain names here.
-		randStr := strconv.FormatUint(rand.Uint64(), 16)
 		domain = strings.ReplaceAll(domain, randomPlaceholder, randStr)
 	}
 

internal/dnsserver/forward/upstreamplain.go

@@ -159,7 +159,7 @@ func (u *UpstreamPlain) exchangeUDP(
 		//
 		// Thus, non-network errors are considered being related to the
 		// response.  It may also happen the received response is intended for
-		// another timeouted request sent from the same source port, but falling
+		// another timed out request sent from the same source port, but falling
 		// back to TCP in this case shouldn't hurt.
 		fallbackToTCP = !isExpectedConnErr(err)
 

internal/dnsserver/forward/upstreamplain_test.go

@@ -5,7 +5,6 @@ import (
 	"net/netip"
 	"testing"
 
-	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/dnsservertest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/forward"
@@ -188,16 +187,16 @@ func TestUpstreamPlain_Exchange_fallbackSuccess(t *testing.T) {
 
 	// Prepare malformed responses.
 
-	badIDResp := dnsmsg.Clone(resp)
+	badIDResp := resp.Copy()
 	badIDResp.Id = ^req.Id
 
-	badQNumResp := dnsmsg.Clone(resp)
+	badQNumResp := resp.Copy()
 	badQNumResp.Question = append(badQNumResp.Question, req.Question[0])
 
-	badQnameResp := dnsmsg.Clone(resp)
+	badQnameResp := resp.Copy()
 	badQnameResp.Question[0].Name = badDomain
 
-	badQtypeResp := dnsmsg.Clone(resp)
+	badQtypeResp := resp.Copy()
 	badQtypeResp.Question[0].Qtype = dns.TypeMX
 
 	testCases := []struct {
@@ -218,9 +217,9 @@ func TestUpstreamPlain_Exchange_fallbackSuccess(t *testing.T) {
 	}}
 
 	for _, tc := range testCases {
-		clonedReq := dnsmsg.Clone(req)
+		clonedReq := req.Copy()
-		badResp := dnsmsg.Clone(tc.udpResp)
+		badResp := tc.udpResp.Copy()
-		goodResp := dnsmsg.Clone(resp)
+		goodResp := resp.Copy()
 
 		// Use only unbuffered channels to block until received and validated.
 		netCh := make(chan string)

internal/dnsserver/go.mod

@@ -3,19 +3,19 @@ module github.com/AdguardTeam/AdGuardDNS/internal/dnsserver
 go 1.20
 
 require (
-	github.com/AdguardTeam/golibs v0.13.2
+	github.com/AdguardTeam/golibs v0.13.6
-	github.com/ameshkov/dnscrypt/v2 v2.2.5
+	github.com/ameshkov/dnscrypt/v2 v2.2.7
 	github.com/ameshkov/dnsstamps v1.0.3
 	github.com/bluele/gcache v0.0.2
-	github.com/miekg/dns v1.1.52
+	github.com/miekg/dns v1.1.55
-	github.com/panjf2000/ants/v2 v2.7.1
+	github.com/panjf2000/ants/v2 v2.7.5
 	github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.15.1
-	github.com/quic-go/quic-go v0.33.0
+	github.com/quic-go/quic-go v0.35.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
+	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
-	golang.org/x/net v0.8.0
+	golang.org/x/net v0.12.0
-	golang.org/x/sys v0.6.0
+	golang.org/x/sys v0.10.0
 )
 
 require (
@@ -24,25 +24,24 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 // indirect
+	github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
-	github.com/kr/pretty v0.2.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/onsi/ginkgo/v2 v2.9.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.10.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.41.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
-	github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.10.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/dnsserver/go.sum

@@ -1,10 +1,11 @@
-github.com/AdguardTeam/golibs v0.13.2 h1:BPASsyQKmb+b8VnvsNOHp7bKfcZl9Z+Z2UhPjOiupSc=
+github.com/AdguardTeam/golibs v0.13.6 h1:z/0Q25pRLdaQxtoxvfSaooz5mdv8wj0R8KREj54q8yQ=
+github.com/AdguardTeam/golibs v0.13.6/go.mod h1:hOtcb8dPfKcFjWTPA904hTA4dl1aWvzeebdJpE72IPk=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw=
 github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us=
-github.com/ameshkov/dnscrypt/v2 v2.2.5 h1:Ju1gQeez+6XLtk/b/k3RoJ2t+Ls+BSItLTZjZeedneY=
+github.com/ameshkov/dnscrypt/v2 v2.2.7 h1:aEitLIR8HcxVodZ79mgRcCiC0A0I5kZPBuWGFwwulAw=
-github.com/ameshkov/dnscrypt/v2 v2.2.5/go.mod h1:Cu5GgMvCR10BeXgACiGDwXyOpfMktsSIidml1XBp6uM=
+github.com/ameshkov/dnscrypt/v2 v2.2.7/go.mod h1:qPWhwz6FdSmuK7W4sMyvogrez4MWdtzosdqlr0Rg3ow=
 github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo=
 github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -17,110 +18,108 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 h1:CqYfpuYIjnlNxM3msdyPRKabhXZWbKjf3Q8BWROFBso=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
-github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c=
+github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
-github.com/onsi/ginkgo/v2 v2.9.0 h1:Tugw2BKlNHTMfG+CheOITkYvk4LAh6MFOvikhGVnhE8=
+github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
-github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
+github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs=
-github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
+github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE=
-github.com/panjf2000/ants/v2 v2.7.1 h1:qBy5lfSdbxvrR0yUnZfaEDjf0FlCw4ufsbcsxmE7r+M=
+github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
-github.com/panjf2000/ants/v2 v2.7.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
+github.com/panjf2000/ants/v2 v2.7.5 h1:/vhh0Hza9G1vP1PdCj9hl6MUzCRbmtcTJL0OsnmytuU=
+github.com/panjf2000/ants/v2 v2.7.5/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
 github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible h1:IWzUvJ72xMjmrjR9q3H1PF+jwdN0uNQiR2t1BLNalyo=
 github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.41.0 h1:npo01n6vUlRViIj5fgwiK8vlNIh8bnoxqh3gypKsyAw=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.41.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-19 v0.2.1 h1:aJcKNMkH5ASEJB9FXNeZCyTEIHU1J7MmHyz1Q1TSG1A=
+github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
-github.com/quic-go/qtls-go1-19 v0.2.1/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
+github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
-github.com/quic-go/qtls-go1-20 v0.1.1 h1:KbChDlg82d3IHqaj2bn6GfKRj84Per2VGf5XV3wSwQk=
+github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E=
-github.com/quic-go/qtls-go1-20 v0.1.1/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
+github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
-github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0=
+github.com/quic-go/quic-go v0.35.1 h1:b0kzj6b/cQAf05cT0CkQubHM31wiA+xH3IBkxP62poo=
-github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA=
+github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
+golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
+golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
+golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/dnssvc/debug.go

@@ -21,6 +21,7 @@ const (
 	hdrNameRuleListID  = "rule-list-id"
 	hdrNameRule        = "rule"
 	hdrNameClientIP    = "client-ip"
+	hdrNameServerIP    = "server-ip"
 	hdrNameDeviceID    = "device-id"
 	hdrNameProfileID   = "profile-id"
 	hdrNameCountry     = "country"
@@ -55,6 +56,15 @@ func (svc *Service) writeDebugResponse(
 		return fmt.Errorf("adding %s extra: %w", hdrNameClientIP, err)
 	}
 
+	lAddr := rw.LocalAddr()
+	localIP, _ := netutil.IPAndPortFromAddr(lAddr)
+
+	setQuestionName(debugReq, "", hdrNameServerIP)
+	err = svc.messages.AppendDebugExtra(debugReq, resp, localIP.String())
+	if err != nil {
+		return fmt.Errorf("adding %s extra: %w", hdrNameServerIP, err)
+	}
+
 	err = svc.appendDebugExtraFromContext(ctx, debugReq, resp)
 	if err != nil {
 		// Don't wrap the error, because it's informative enough as is.

internal/dnssvc/debug_internal_test.go

@@ -48,6 +48,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 	)
 
 	clientIPStr := testClientIP.String()
+	serverIPStr := testServerAddr.String()
 	testCases := []struct {
 		name      string
 		ri        *agd.RequestInfo
@@ -61,6 +62,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"resp.res-type.adguard-dns.com.", "normal"},
 		}),
 	}, {
@@ -70,6 +72,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"req.res-type.adguard-dns.com.", "blocked"},
 			{"req.rule.adguard-dns.com.", "||example.com^"},
 			{"req.rule-list-id.adguard-dns.com.", "fl1"},
@@ -81,6 +84,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: &filter.ResultBlocked{List: fltListID2, Rule: blockRule},
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"resp.res-type.adguard-dns.com.", "blocked"},
 			{"resp.rule.adguard-dns.com.", "||example.com^"},
 			{"resp.rule-list-id.adguard-dns.com.", "fl2"},
@@ -92,6 +96,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"req.res-type.adguard-dns.com.", "allowed"},
 			{"req.rule.adguard-dns.com.", ""},
 			{"req.rule-list-id.adguard-dns.com.", ""},
@@ -103,6 +108,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: &filter.ResultAllowed{},
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"resp.res-type.adguard-dns.com.", "allowed"},
 			{"resp.rule.adguard-dns.com.", ""},
 			{"resp.rule-list-id.adguard-dns.com.", ""},
@@ -116,6 +122,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"req.res-type.adguard-dns.com.", "modified"},
 			{"req.rule.adguard-dns.com.", "||example.com^$dnsrewrite=REFUSED"},
 			{"req.rule-list-id.adguard-dns.com.", ""},
@@ -127,6 +134,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"device-id.adguard-dns.com.", testDeviceID},
 			{"resp.res-type.adguard-dns.com.", "normal"},
 		}),
@@ -139,6 +147,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"profile-id.adguard-dns.com.", testProfileID},
 			{"resp.res-type.adguard-dns.com.", "normal"},
 		}),
@@ -149,6 +158,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"country.adguard-dns.com.", string(agd.CountryAD)},
 			{"asn.adguard-dns.com.", "0"},
 			{"resp.res-type.adguard-dns.com.", "normal"},
@@ -162,6 +172,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 		respRes: nil,
 		wantExtra: newTXTExtra([][2]string{
 			{"client-ip.adguard-dns.com.", clientIPStr},
+			{"server-ip.adguard-dns.com.", serverIPStr},
 			{"country.adguard-dns.com.", string(agd.CountryAD)},
 			{"asn.adguard-dns.com.", "0"},
 			{"subdivision.adguard-dns.com.", "CA"},
@@ -171,7 +182,7 @@ func TestService_writeDebugResponse(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			rw := dnsserver.NewNonWriterResponseWriter(nil, testRAddr)
+			rw := dnsserver.NewNonWriterResponseWriter(testLocalAddr, testRAddr)
 
 			ctx := agd.ContextWithRequestInfo(context.Background(), tc.ri)
 

internal/dnssvc/dnssvc.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/billstat"
@@ -110,12 +111,22 @@ type Config struct {
 
 	// CacheSize is the size of the DNS cache for domain names that don't
 	// support ECS.
+	//
+	// TODO(a.garipov): Extract this and following fields to cache configuration
+	// struct.
 	CacheSize int
 
 	// ECSCacheSize is the size of the DNS cache for domain names that support
 	// ECS.
 	ECSCacheSize int
 
+	// CacheMinTTL is the minimum supported TTL for cache items.  This setting
+	// is used when UseCacheTTLOverride set to true.
+	CacheMinTTL time.Duration
+
+	// UseCacheTTLOverride shows if the TTL overrides logic should be used.
+	UseCacheTTLOverride bool
+
 	// UseECSCache shows if the EDNS Client Subnet (ECS) aware cache should be
 	// used.
 	UseECSCache bool
@@ -124,6 +135,12 @@ type Config struct {
 	// This is a set of metrics that we may need temporary, so its collection is
 	// controlled by a separate setting.
 	ResearchMetrics bool
+
+	// ResearchLogs controls whether logging of additional info for research
+	// purposes is enabled.  These logs may be overly verbose and are only
+	// required temporary, that's why it's controlled by a separate setting.
+	// This setting will only be used when ResearchMetrics is also set to true.
+	ResearchLogs bool
 }
 
 // New returns a new DNS service.
@@ -148,6 +165,8 @@ func New(c *Config) (svc *Service, err error) {
 		cacheSize:           c.CacheSize,
 		ecsCacheSize:        c.ECSCacheSize,
 		useECSCache:         c.UseECSCache,
+		cacheMinTTL:         c.CacheMinTTL,
+		useCacheTTLOverride: c.UseCacheTTLOverride,
 	}
 	handler = preUps.Wrap(handler)
 
@@ -163,6 +182,7 @@ func New(c *Config) (svc *Service, err error) {
 		ruleStat:        c.RuleStat,
 		groups:          groups,
 		researchMetrics: c.ResearchMetrics,
+		researchLog:     c.ResearchLogs,
 	}
 
 	for i, srvGrp := range c.ServerGroups {
@@ -224,7 +244,15 @@ type Service struct {
 	queryLog querylog.Interface
 	ruleStat rulestat.Interface
 	groups   []*serverGroup
+
+	// researchMetrics enables reporting metrics that may be needed for research
+	// purposes.
 	researchMetrics bool
+
+	// researchLog enables logging of additional information that may be needed
+	// for research purposes.  It will only be used when researchMetrics is set
+	// to true.
+	researchLog bool
 }
 
 // mustStartListener starts l and panics on any error.

internal/dnssvc/dnssvc_internal_test.go

@@ -17,6 +17,10 @@ var (
 	testClientAddr     = testClientAddrPort.Addr()
 
 	testServerAddr = netip.MustParseAddr("5.6.7.8")
+	testLocalAddr  = &net.TCPAddr{
+		IP:   testServerAddr.AsSlice(),
+		Port: 54321,
+	}
 )
 
 // testDeviceID is the common device ID for tests

internal/dnssvc/initmw.go

@@ -174,7 +174,7 @@ func (mw *initMw) addProfile(
 		return err
 	}
 
-	optlog.Debug2("init mw: got device id %q and ip %s", id, ri.RemoteIP)
+	optlog.Debug3("init mw: got device id %q, raddr %s, and laddr %s", id, ri.RemoteIP, localIP)
 
 	prof, dev, byWhat, err := mw.profile(ctx, localIP, ri.RemoteIP, id)
 	if err != nil {

internal/dnssvc/middleware.go

@@ -93,7 +93,7 @@ func (mh *svcHandler) ServeDNS(
 	origResp := nwrw.Msg()
 	respRes, elapsedResp := mh.svc.filterResponse(ctx, req, origResp, flt, reqInfo, modReq)
 
-	mh.svc.reportMetrics(reqInfo, reqRes, respRes, elapsedReq+elapsedResp)
+	mh.svc.reportMetrics(reqInfo, reqRes, respRes, origResp, elapsedReq+elapsedResp)
 
 	if isDebug {
 		return mh.svc.writeDebugResponse(ctx, rw, req, origResp, reqRes, respRes)
@@ -196,6 +196,7 @@ func (svc *Service) reportMetrics(
 	ri *agd.RequestInfo,
 	reqRes filter.Result,
 	respRes filter.Result,
+	origResp *dns.Msg,
 	elapsedFiltering time.Duration,
 ) {
 	var ctry, cont string
@@ -221,7 +222,7 @@ func (svc *Service) reportMetrics(
 	metrics.DNSSvcUsersCountUpdate(ri.RemoteIP)
 
 	if svc.researchMetrics {
-		metrics.ReportResearchMetrics(ri, id, isBlocked)
+		metrics.ReportResearch(ri, origResp, id, isBlocked, svc.researchLog)
 	}
 }
 

internal/dnssvc/preupstreammw.go

@@ -3,6 +3,7 @@ package dnssvc
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdnet"
@@ -25,9 +26,11 @@ import (
 type preUpstreamMw struct {
 	db                  dnsdb.Interface
 	geoIP               geoip.Interface
+	cacheMinTTL         time.Duration
 	cacheSize           int
 	ecsCacheSize        int
 	useECSCache         bool
+	useCacheTTLOverride bool
 }
 
 // type check
@@ -44,11 +47,15 @@ func (mw *preUpstreamMw) Wrap(h dnsserver.Handler) (wrapped dnsserver.Handler) {
 				GeoIP:          mw.geoIP,
 				Size:           mw.cacheSize,
 				ECSSize:        mw.ecsCacheSize,
+				MinTTL:         mw.cacheMinTTL,
+				UseTTLOverride: mw.useCacheTTLOverride,
 			})
 		} else {
 			cacheMw = cache.NewMiddleware(&cache.MiddlewareConfig{
 				MetricsListener: &prometheus.CacheMetricsListener{},
 				Size:            mw.cacheSize,
+				MinTTL:          mw.cacheMinTTL,
+				UseTTLOverride:  mw.useCacheTTLOverride,
 			})
 		}
 

internal/ecscache/cache.go

@@ -140,7 +140,7 @@ func (mw *Middleware) toCacheKey(cr *cacheRequest, respIsECSDependent bool) (key
 // set saves resp to the cache if it's cacheable.  If msg cannot be cached, it
 // is ignored.
 func (mw *Middleware) set(resp *dns.Msg, cr *cacheRequest, respIsECSDependent bool) {
-	ttl := findLowestTTL(resp)
+	ttl := dnsmsg.FindLowestTTL(resp)
 	if ttl == 0 || !isCacheable(resp) {
 		return
 	}
@@ -150,10 +150,17 @@ func (mw *Middleware) set(resp *dns.Msg, cr *cacheRequest, respIsECSDependent bo
 		cache = mw.ecsCache
 	}
 
+	exp := time.Duration(ttl) * time.Second
+	if mw.useTTLOverride && resp.Rcode != dns.RcodeServerFailure {
+		// TODO(d.kolyshev): Use built-in max in go 1.21.
+		exp = mathutil.Max(exp, mw.cacheMinTTL)
+		dnsmsg.SetMinTTL(resp, uint32(exp.Seconds()))
+	}
+
 	key := mw.toCacheKey(cr, respIsECSDependent)
 	item := toCacheItem(resp, cr.host)
 
-	err := cache.SetWithExpire(key, item, time.Duration(ttl)*time.Second)
+	err := cache.SetWithExpire(key, item, exp)
 	if err != nil {
 		// Shouldn't happen, since we don't set a serialization function.
 		panic(fmt.Errorf("ecs-cache: setting cache item: %w", err))
@@ -187,7 +194,7 @@ func toCacheItem(msg *dns.Msg, host string) (item *cacheItem) {
 func fromCacheItem(item *cacheItem, req *dns.Msg, reqDO bool) (msg *dns.Msg) {
 	// Update the TTL depending on when the item was cached.  If it's already
 	// expired, update TTL to 0.
-	newTTL := findLowestTTL(item.msg)
+	newTTL := dnsmsg.FindLowestTTL(item.msg)
 	if timeLeft := time.Duration(newTTL)*time.Second - time.Since(item.when); timeLeft > 0 {
 		newTTL = uint32(roundDiv(timeLeft, time.Second))
 	} else {

internal/ecscache/ecscache.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
@@ -35,6 +36,12 @@ type Middleware struct {
 
 	// cacheReqPool is a pool of cache requests.
 	cacheReqPool *sync.Pool
+
+	// cacheMinTTL is the minimum supported TTL for cache items.
+	cacheMinTTL time.Duration
+
+	// useTTLOverride shows if the TTL overrides logic should be used.
+	useTTLOverride bool
 }
 
 // MiddlewareConfig is the configuration structure for NewMiddleware.
@@ -43,6 +50,9 @@ type MiddlewareConfig struct {
 	// not be nil.
 	GeoIP geoip.Interface
 
+	// MinTTL is the minimum supported TTL for cache items.
+	MinTTL time.Duration
+
 	// Size is the number of entities to hold in the cache for hosts that don't
 	// support ECS.  It must be greater than zero.
 	Size int
@@ -50,6 +60,9 @@ type MiddlewareConfig struct {
 	// ECSSize is the number of entities to hold in the cache for hosts that
 	// support ECS.  It must be greater than zero.
 	ECSSize int
+
+	// UseTTLOverride shows if the TTL overrides logic should be used.
+	UseTTLOverride bool
 }
 
 // NewMiddleware initializes a new ECS-aware LRU caching middleware.  c must not
@@ -58,6 +71,8 @@ func NewMiddleware(c *MiddlewareConfig) (m *Middleware) {
 	return &Middleware{
 		cache:          gcache.New(c.Size).LRU().Build(),
 		ecsCache:       gcache.New(c.ECSSize).LRU().Build(),
+		cacheMinTTL:    c.MinTTL,
+		useTTLOverride: c.UseTTLOverride,
 		geoIP:          c.GeoIP,
 		cacheReqPool: &sync.Pool{
 			New: func() (req any) {

internal/ecscache/ecscache_test.go

@@ -49,10 +49,13 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 		dnsservertest.NewSOA(reqHostname, defaultTTL, reqNS1, reqNS2),
 	}
 
+	testTTL := 60 * time.Second
+
 	const N = 5
 	testCases := []struct {
 		req        *dns.Msg
 		resp       *dns.Msg
+		minTTL     *time.Duration
 		name       string
 		wantNumReq int
 		wantTTL    uint32
@@ -64,30 +67,35 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 		name:       "simple_a",
 		wantNumReq: 1,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq),
 		name:       "empty_answer",
 		wantNumReq: N,
 		wantTTL:    0,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, soaNS),
 		name:       "authoritative_nodata",
 		wantNumReq: 1,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, cnameAns, soaNS),
 		name:       "nodata_with_cname",
 		wantNumReq: 1,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeSuccess, aReq, cnameAns),
 		name:       "nodata_with_cname_no_soa",
 		wantNumReq: N,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req: aReq,
 		resp: dnsservertest.NewResp(dns.RcodeNameError, aReq, dnsservertest.SectionNs{
@@ -97,18 +105,21 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 		// TODO(ameshkov): Consider https://datatracker.ietf.org/doc/html/rfc2308#section-3.
 		wantNumReq: 1,
 		wantTTL:    0,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeNameError, aReq, soaNS),
 		name:       "authoritative_nxdomain",
 		wantNumReq: 1,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeServerFailure, aReq),
 		name:       "simple_server_failure",
 		wantNumReq: 1,
-		wantTTL:    ecscache.ServFailMaxCacheTTL,
+		wantTTL:    dnsmsg.ServFailMaxCacheTTL,
+		minTTL:     nil,
 	}, {
 		req: cnameReq,
 		resp: dnsservertest.NewResp(dns.RcodeSuccess, cnameReq, dnsservertest.SectionAnswer{
@@ -117,6 +128,7 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 		name:       "simple_cname_ans",
 		wantNumReq: 1,
 		wantTTL:    defaultTTL,
+		minTTL:     nil,
 	}, {
 		req: aReq,
 		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
@@ -125,12 +137,50 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 		name:       "expired_one",
 		wantNumReq: N,
 		wantTTL:    0,
+		minTTL:     nil,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 10, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_ok",
+		wantNumReq: 1,
+		minTTL:     &testTTL,
+		wantTTL:    uint32(testTTL.Seconds()),
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 1000, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_max",
+		wantNumReq: 1,
+		minTTL:     &testTTL,
+		wantTTL:    1000,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeSuccess, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, 0, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_zero",
+		wantNumReq: N,
+		minTTL:     &testTTL,
+		wantTTL:    0,
+	}, {
+		req: aReq,
+		resp: dnsservertest.NewResp(dns.RcodeServerFailure, aReq, dnsservertest.SectionAnswer{
+			dnsservertest.NewA(reqHostname, dnsmsg.ServFailMaxCacheTTL, net.IP{1, 2, 3, 4}),
+		}),
+		name:       "override_ttl_servfail",
+		wantNumReq: 1,
+		minTTL:     nil,
+		wantTTL:    dnsmsg.ServFailMaxCacheTTL,
 	}, {
 		req:        aReq,
 		resp:       dnsservertest.NewResp(dns.RcodeNotImplemented, aReq, soaNS),
 		name:       "unexpected_response",
 		wantNumReq: N,
 		wantTTL:    0,
+		minTTL:     nil,
 	}}
 
 	for _, tc := range testCases {
@@ -144,11 +194,18 @@ func TestMiddleware_Wrap_noECS(t *testing.T) {
 				},
 			)
 
+			var minTTL time.Duration
+			if tc.minTTL != nil {
+				minTTL = *tc.minTTL
+			}
+
 			withCache := newWithCache(
 				t,
 				handler,
 				agd.CountryNone,
 				netutil.ZeroPrefix(netutil.AddrFamilyIPv4),
+				minTTL,
+				tc.minTTL != nil,
 			)
 			ri := &agd.RequestInfo{
 				Host:     tc.req.Question[0].Name,
@@ -284,7 +341,7 @@ func TestMiddleware_Wrap_ecs(t *testing.T) {
 				},
 			)
 
-			withCache := newWithCache(t, handler, ctry, tc.ctrySubnet)
+			withCache := newWithCache(t, handler, ctry, tc.ctrySubnet, 0, false)
 			ri := &agd.RequestInfo{
 				Location: &agd.Location{
 					Country: ctry,
@@ -493,7 +550,7 @@ func TestMiddleware_Wrap_ecsOrder(t *testing.T) {
 	}}
 
 	for _, tc := range testCases {
-		withCache := newWithCache(t, handler, ctry, ctrySubnet)
+		withCache := newWithCache(t, handler, ctry, ctrySubnet, 0, false)
 
 		t.Run(tc.name, func(t *testing.T) {
 			for i, req := range tc.sequence {
@@ -554,6 +611,8 @@ func newWithCache(
 	h dnsserver.Handler,
 	wantCtry agd.Country,
 	geoIPNet netip.Prefix,
+	minTTL time.Duration,
+	useTTLOverride bool,
 ) (wrapped dnsserver.Handler) {
 	t.Helper()
 
@@ -581,6 +640,8 @@ func newWithCache(
 			GeoIP:          geoIP,
 			Size:           100,
 			ECSSize:        100,
+			MinTTL:         minTTL,
+			UseTTLOverride: useTTLOverride,
 		}),
 	)
 }

internal/ecscache/msg.go

@@ -2,13 +2,11 @@ package ecscache
 
 import (
 	"fmt"
-	"math"
 	"net"
 	"net/netip"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
-	"github.com/AdguardTeam/golibs/mathutil"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/miekg/dns"
 )
@@ -220,55 +218,3 @@ func isCacheableNOERROR(resp *dns.Msg) (ok bool) {
 
 	return false
 }
-
-// ServFailMaxCacheTTL is the maximum time-to-live value for caching
-// SERVFAIL responses in seconds.  It's consistent with the upper constraint
-// of 5 minutes given by RFC 2308.
-//
-// See https://datatracker.ietf.org/doc/html/rfc2308#section-7.1.
-const ServFailMaxCacheTTL = 30
-
-// findLowestTTL gets the lowest TTL among all DNS message's RRs.
-func findLowestTTL(msg *dns.Msg) (ttl uint32) {
-	// Use the maximum value as a guard value.  If the inner loop is entered,
-	// it's going to be rewritten with an actual TTL value that is lower than
-	// MaxUint32.  If the inner loop isn't entered, catch that and return zero.
-	ttl = math.MaxUint32
-	for _, rrs := range [][]dns.RR{msg.Answer, msg.Ns, msg.Extra} {
-		for _, rr := range rrs {
-			ttl = getTTLIfLower(rr, ttl)
-			if ttl == 0 {
-				return 0
-			}
-		}
-	}
-
-	switch {
-	case msg.Rcode == dns.RcodeServerFailure && ttl > ServFailMaxCacheTTL:
-		return ServFailMaxCacheTTL
-	case ttl == math.MaxUint32:
-		return 0
-	default:
-		return ttl
-	}
-}
-
-// getTTLIfLower is a helper function that checks the TTL of the specified RR
-// and returns it if it's lower than the one passed in the arguments.
-func getTTLIfLower(r dns.RR, ttl uint32) (res uint32) {
-	switch r := r.(type) {
-	case *dns.OPT:
-		// Don't even consider the OPT RRs TTL.
-		return ttl
-	case *dns.SOA:
-		if r.Minttl > 0 && r.Minttl < ttl {
-			// Per RFC 2308, the TTL of a SOA RR is the minimum of SOA.MINIMUM
-			// field and the header's value.
-			ttl = r.Minttl
-		}
-	default:
-		// Go on.
-	}
-
-	return mathutil.Min(r.Header().Ttl, ttl)
-}

internal/errcoll/sentry.go

@@ -168,7 +168,7 @@ func tagsFromCtx(ctx context.Context) (tags sentryTags) {
 	var reqID agd.RequestID
 	if ri, ok := agd.RequestInfoFromContext(ctx); ok {
 		tags["filtering_group_id"] = string(ri.FilteringGroup.ID)
-		tags["request_id"] = string(ri.ID)
+		tags["request_id"] = ri.ID.String()
 
 		if p := ri.Profile; p != nil {
 			tags["profile_id"] = string(p.ID)
@@ -179,7 +179,7 @@ func tagsFromCtx(ctx context.Context) (tags sentryTags) {
 	} else if reqID, ok = agd.RequestIDFromContext(ctx); ok {
 		// This context could be from the part of the pipeline where the request
 		// ID hasn't yet been resurfaced.
-		tags["request_id"] = string(reqID)
+		tags["request_id"] = reqID.String()
 	}
 
 	if si, ok := dnsserver.ServerInfoFromContext(ctx); ok {

internal/errcoll/sentry_test.go

@@ -67,7 +67,8 @@ func TestSentryErrorCollector(t *testing.T) {
 	const devID = "dev1234"
 	const fltGrpID = "fg1234"
 	const profID = "prof1234"
-	const reqID = "req5678"
+
+	reqID := agd.NewRequestID()
 
 	ctx := context.Background()
 	ctx = agd.ContextWithRequestInfo(ctx, &agd.RequestInfo{
@@ -103,7 +104,7 @@ func TestSentryErrorCollector(t *testing.T) {
 		"device_id":          devID,
 		"filtering_group_id": fltGrpID,
 		"profile_id":         profID,
-		"request_id":         reqID,
+		"request_id":         reqID.String(),
 	}
 	assert.Equal(t, wantTags, gotTags)
 }

internal/filter/filter_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/hashprefix"
+	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/filtertest"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/stretchr/testify/require"
 )
@@ -36,9 +37,6 @@ const testDeviceName agd.DeviceName = "My Device"
 // testSvcID is the standard ID of the blocked service for tests.
 const testSvcID agd.BlockedServiceID = "service"
 
-// testRefreshIvl is the standard refresh interval for tests.
-const testRefreshIvl = 1 * time.Hour
-
 // Common constants.  Keep in sync with ./testdata/filter and
 // ./safesearchhost.csv.
 const (
@@ -185,16 +183,18 @@ func prepareConf(t testing.TB) (c *filter.DefaultStorageConfig) {
 		YoutubeSafeSearchRulesURL: ssURL,
 		SafeBrowsing:              &hashprefix.Filter{},
 		AdultBlocking:             &hashprefix.Filter{},
+		NewRegDomains:             &hashprefix.Filter{},
 		Now:                       time.Now,
 		ErrColl:                   nil,
 		Resolver:                  nil,
 		CacheDir:                  cacheDir,
 		CustomFilterCacheSize:     100,
 		SafeSearchCacheSize:       100,
-		SafeSearchCacheTTL:        1 * time.Hour,
+		SafeSearchCacheTTL:        filtertest.CacheTTL,
 		RuleListCacheSize:         100,
-		RefreshIvl:                testRefreshIvl,
+		RefreshIvl:                filtertest.Staleness,
 		UseRuleListCache:          false,
+		MaxRuleListSize:           filtertest.FilterMaxSize,
 	}
 }
 

internal/filter/hashprefix/filter.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdnet"
+	"github.com/AdguardTeam/AdGuardDNS/internal/dnsmsg"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/resultcache"
 	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
@@ -56,6 +57,9 @@ type FilterConfig struct {
 
 	// CacheSize is the size of the filter's result cache.
 	CacheSize int
+
+	// MaxSize is the maximum size in bytes of the downloadable rule-list.
+	MaxSize int64
 }
 
 // Filter is a filter that matches hosts by their hashes based on a
@@ -82,14 +86,15 @@ func NewFilter(c *FilterConfig) (f *Filter, err error) {
 		repHost:  c.ReplacementHost,
 	}
 
-	f.refr = internal.NewRefreshable(
+	f.refr = internal.NewRefreshable(&internal.RefreshableConfig{
-		&agd.FilterList{
-			ID:         id,
 		URL:       c.URL,
-			RefreshIvl: c.Staleness,
+		ID:        id,
-		},
+		CachePath: c.CachePath,
-		c.CachePath,
+		Staleness: c.Staleness,
-	)
+		// TODO(ameshkov): Consider making configurable.
+		Timeout: internal.DefaultFilterRefreshTimeout,
+		MaxSize: c.MaxSize,
+	})
 
 	err = f.refresh(context.Background(), true)
 	if err != nil {
@@ -124,8 +129,8 @@ func (f *Filter) FilterRequest(
 		return rm.CloneForReq(req), nil
 	}
 
-	fam := netutil.AddrFamilyFromRRType(qt)
+	fam, ok := isFilterable(qt)
-	if fam == netutil.AddrFamilyNone {
+	if !ok {
 		return nil, nil
 	}
 
@@ -148,17 +153,10 @@ func (f *Filter) FilterRequest(
 	ctx, cancel := context.WithTimeout(ctx, internal.DefaultResolveTimeout)
 	defer cancel()
 
-	var result *dns.Msg
+	result, err := f.filteredResponse(ctx, req, ri, fam)
-	ips, err := f.resolver.LookupIP(ctx, fam, f.repHost)
 	if err != nil {
-		agd.Collectf(ctx, f.errColl, "filter %s: resolving: %w", f.id, err)
+		// Don't wrap the error, because it's informative enough as is.
-
+		return nil, err
-		result = ri.Messages.NewMsgSERVFAIL(req)
-	} else {
-		result, err = ri.Messages.NewIPRespMsg(req, ips...)
-		if err != nil {
-			return nil, fmt.Errorf("filter %s: creating modified result: %w", f.id, err)
-		}
 	}
 
 	rm = &internal.ResultModified{
@@ -177,6 +175,58 @@ func (f *Filter) FilterRequest(
 	return rm, nil
 }
 
+// isFilterable returns true if the question type is filterable.  If the type is
+// filterable with a blocked page, fam is the address family for the IP
+// addresses of the blocked page; otherwise fam is [netutil.AddrFamilyNone].
+func isFilterable(qt dnsmsg.RRType) (fam netutil.AddrFamily, ok bool) {
+	if qt == dns.TypeHTTPS {
+		return netutil.AddrFamilyNone, true
+	}
+
+	fam = netutil.AddrFamilyFromRRType(qt)
+
+	return fam, fam != netutil.AddrFamilyNone
+}
+
+// filteredResponse returns a filtered response.
+func (f *Filter) filteredResponse(
+	ctx context.Context,
+	req *dns.Msg,
+	ri *agd.RequestInfo,
+	fam netutil.AddrFamily,
+) (resp *dns.Msg, err error) {
+	if fam == netutil.AddrFamilyNone {
+		// This is an HTTPS query.  For them, just return NODATA or other
+		// blocked response.  See AGDNS-1551.
+		//
+		// TODO(ameshkov): Consider putting the resolved IP addresses into hints
+		// to show the blocked page here as well.
+		resp, err = ri.Messages.NewBlockedRespMsg(req)
+		if err != nil {
+			return nil, fmt.Errorf("filter %s: creating blocked result: %w", f.id, err)
+		}
+
+		return resp, nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, internal.DefaultResolveTimeout)
+	defer cancel()
+
+	ips, err := f.resolver.LookupIP(ctx, fam, f.repHost)
+	if err != nil {
+		agd.Collectf(ctx, f.errColl, "filter %s: resolving: %w", f.id, err)
+
+		return ri.Messages.NewMsgSERVFAIL(req), nil
+	}
+
+	resp, err = ri.Messages.NewIPRespMsg(req, ips...)
+	if err != nil {
+		return nil, fmt.Errorf("filter %s: creating modified result: %w", f.id, err)
+	}
+
+	return resp, nil
+}
+
 // updateCacheSizeMetrics updates cache size metrics.
 func (f *Filter) updateCacheSizeMetrics(size int) {
 	switch id := f.id; id {
@@ -184,6 +234,8 @@ func (f *Filter) updateCacheSizeMetrics(size int) {
 		metrics.HashPrefixFilterSafeBrowsingCacheSize.Set(float64(size))
 	case agd.FilterListIDAdultBlocking:
 		metrics.HashPrefixFilterAdultBlockingCacheSize.Set(float64(size))
+	case agd.FilterListIDNewRegDomains:
+		metrics.HashPrefixFilterNewRegDomainsCacheSize.Set(float64(size))
 	default:
 		panic(fmt.Errorf("unsupported FilterListID %s", id))
 	}
@@ -199,6 +251,9 @@ func (f *Filter) updateCacheLookupsMetrics(hit bool) {
 	case agd.FilterListIDAdultBlocking:
 		hitsMetric = metrics.HashPrefixFilterCacheAdultBlockingHits
 		missesMetric = metrics.HashPrefixFilterCacheAdultBlockingMisses
+	case agd.FilterListIDNewRegDomains:
+		hitsMetric = metrics.HashPrefixFilterCacheNewRegDomainsHits
+		missesMetric = metrics.HashPrefixFilterCacheNewRegDomainsMisses
 	default:
 		panic(fmt.Errorf("unsupported filter list id %s", id))
 	}

internal/filter/hashprefix/filter_test.go

@@ -49,9 +49,10 @@ func TestFilter_FilterRequest(t *testing.T) {
 		ID:              agd.FilterListIDAdultBlocking,
 		CachePath:       cachePath,
 		ReplacementHost: "repl.example",
-		Staleness:       1 * time.Minute,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        1 * time.Minute,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       1,
+		MaxSize:         filtertest.FilterMaxSize,
 	})
 	require.NoError(t, err)
 
@@ -157,6 +158,30 @@ func TestFilter_FilterRequest(t *testing.T) {
 
 		assert.Nil(t, r)
 	})
+
+	t.Run("https", func(t *testing.T) {
+		req := dnsservertest.NewReq(dns.Fqdn(testHost), dns.TypeHTTPS, dns.ClassINET)
+		ri := &agd.RequestInfo{
+			Messages: messages,
+			Host:     testHost,
+			QType:    dns.TypeHTTPS,
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), filtertest.Timeout)
+		t.Cleanup(cancel)
+
+		var r internal.Result
+		r, err = f.FilterRequest(ctx, req, ri)
+		require.NoError(t, err)
+		require.NotNil(t, r)
+
+		m := testutil.RequireTypeAssert[*internal.ResultModified](t, r)
+		require.NotNil(t, m.Msg)
+		require.Len(t, m.Msg.Question, 1)
+
+		assert.Equal(t, m.Msg.Question[0].Qtype, dns.TypeHTTPS)
+		assert.Len(t, m.Msg.Answer, 0)
+	})
 }
 
 // newModifiedResult is a helper for creating modified results for tests.
@@ -203,9 +228,10 @@ func TestFilter_Refresh(t *testing.T) {
 		ID:              agd.FilterListIDAdultBlocking,
 		CachePath:       cachePath,
 		ReplacementHost: "",
-		Staleness:       1 * time.Minute,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        1 * time.Minute,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       1,
+		MaxSize:         filtertest.FilterMaxSize,
 	})
 	require.NoError(t, err)
 
@@ -257,9 +283,10 @@ func TestFilter_FilterRequest_staleCache(t *testing.T) {
 		ID:              agd.FilterListIDAdultBlocking,
 		CachePath:       cachePath,
 		ReplacementHost: "repl.example",
-		Staleness:       1 * time.Minute,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        1 * time.Minute,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       1,
+		MaxSize:         filtertest.FilterMaxSize,
 	}
 	f, err := hashprefix.NewFilter(fconf)
 	require.NoError(t, err)

internal/filter/index.go

@@ -0,0 +1,66 @@
+package filter
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
+	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
+)
+
+// filterIndexResp is the struct for the JSON response from a filter index API.
+type filterIndexResp struct {
+	Filters []*filterIndexRespFilter `json:"filters"`
+}
+
+// filterIndexRespFilter is the struct for a filter from the JSON response from
+// a filter index API.
+type filterIndexRespFilter struct {
+	DownloadURL string `json:"downloadUrl"`
+	ID          string `json:"filterId"`
+}
+
+// filterIndexFilterData is the data of a single item in the filtering-rule
+// index response.
+type filterIndexFilterData struct {
+	url *url.URL
+	id  agd.FilterListID
+}
+
+// toInternal converts the filters from the index to []*filterIndexFilterData.
+func (r *filterIndexResp) toInternal(
+	ctx context.Context,
+	errColl agd.ErrorCollector,
+) (fls []*filterIndexFilterData) {
+	fls = make([]*filterIndexFilterData, 0, len(r.Filters))
+	for _, rf := range r.Filters {
+		id, err := agd.NewFilterListID(rf.ID)
+		if err != nil {
+			agd.Collectf(ctx, errColl, "%s: validating id %q: %w", strgLogPrefix, rf.ID, err)
+
+			continue
+		}
+
+		var u *url.URL
+		u, err = agdhttp.ParseHTTPURL(rf.DownloadURL)
+		if err != nil {
+			agd.Collectf(
+				ctx,
+				errColl,
+				"%s: validating url %q: %w",
+				strgLogPrefix,
+				rf.DownloadURL,
+				err,
+			)
+
+			continue
+		}
+
+		fls = append(fls, &filterIndexFilterData{
+			url: u,
+			id:  id,
+		})
+	}
+
+	return fls
+}

internal/filter/internal/composite/composite.go

@@ -25,6 +25,7 @@ import (
 // result.
 type Filter struct {
 	safeBrowsing         *hashprefix.Filter
+	newRegisteredDomains *hashprefix.Filter
 	adultBlocking        *hashprefix.Filter
 
 	genSafeSearch *safesearch.Filter
@@ -50,6 +51,10 @@ type Config struct {
 	// AdultBlocking is the adult-content filter to apply, if any.
 	AdultBlocking *hashprefix.Filter
 
+	// NewRegisteredDomains is the newly registered domains filter to apply, if
+	// any.
+	NewRegisteredDomains *hashprefix.Filter
+
 	// GeneralSafeSearch is the general safe-search filter to apply, if any.
 	GeneralSafeSearch *safesearch.Filter
 
@@ -83,6 +88,7 @@ func New(c *Config) (f *Filter) {
 		custom:               c.Custom,
 		ruleLists:            c.RuleLists,
 		svcLists:             c.ServiceLists,
+		newRegisteredDomains: c.NewRegisteredDomains,
 	}
 }
 
@@ -143,6 +149,9 @@ func (f *Filter) FilterRequest(
 	}, {
 		filter: nullify(f.ytSafeSearch),
 		id:     agd.FilterListIDYoutubeSafeSearch,
+	}, {
+		filter: nullify(f.newRegisteredDomains),
+		id:     agd.FilterListIDNewRegDomains,
 	}}
 
 	for _, rf := range reqFilters {
@@ -179,7 +188,7 @@ func nullify[T *safesearch.Filter | *hashprefix.Filter](flt T) (fr internal.Requ
 // returns the action created from the filter list network rule with the highest
 // priority.  If f is empty, it returns nil with no error.
 func (f *Filter) FilterResponse(
-	ctx context.Context,
+	_ context.Context,
 	resp *dns.Msg,
 	ri *agd.RequestInfo,
 ) (r internal.Result, err error) {
@@ -188,6 +197,13 @@ func (f *Filter) FilterResponse(
 	}
 
 	for _, ans := range resp.Answer {
+		if rr, ok := ans.(*dns.HTTPS); ok {
+			r = f.filterHTTPSRecords(rr, ri, resp)
+			if r != nil {
+				return r, nil
+			}
+		}
+
 		host, rrType, ok := parseRespAnswer(ans)
 		if !ok {
 			continue
@@ -202,6 +218,45 @@ func (f *Filter) FilterResponse(
 	return r, nil
 }
 
+// filterHTTPSRecords filters HTTPS answers information through all rule list
+// filters of the composite filter.
+func (f *Filter) filterHTTPSRecords(
+	rr *dns.HTTPS,
+	ri *agd.RequestInfo,
+	resp *dns.Msg,
+) (r internal.Result) {
+	for _, kv := range rr.Value {
+		switch kv.Key() {
+		case dns.SVCB_IPV4HINT, dns.SVCB_IPV6HINT:
+			r = f.filterSVCBHint(kv.String(), ri, resp)
+			if r != nil {
+				return r
+			}
+		default:
+			// Go on.
+		}
+	}
+
+	return nil
+}
+
+// filterSVCBHint filters SVCB hint information through all rule list filters of
+// the composite filter.
+func (f *Filter) filterSVCBHint(
+	hint string,
+	ri *agd.RequestInfo,
+	resp *dns.Msg,
+) (r internal.Result) {
+	for _, s := range strings.Split(hint, ",") {
+		r = f.filterWithRuleLists(ri, s, dns.TypeHTTPS, resp, true)
+		if r != nil {
+			return r
+		}
+	}
+
+	return nil
+}
+
 // parseRespAnswer parses hostname and rrType from the answer if there are any.
 // If ans is of a type that doesn't have an IP address or a hostname in it, ok
 // is false.
@@ -226,6 +281,7 @@ func (f *Filter) isEmpty() (ok bool) {
 			f.genSafeSearch == nil &&
 			f.ytSafeSearch == nil &&
 			f.custom == nil &&
+			f.newRegisteredDomains == nil &&
 			len(f.ruleLists) == 0 &&
 			len(f.svcLists) == 0)
 }

internal/filter/internal/composite/composite_test.go

@@ -5,7 +5,6 @@ import (
 	"net"
 	"net/http"
 	"net/netip"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -371,9 +370,13 @@ func TestFilter_FilterRequest_safeSearch(t *testing.T) {
 	const fltListID = agd.FilterListIDGeneralSafeSearch
 
 	gen := safesearch.New(&safesearch.Config{
-		List: &agd.FilterList{
+		Refreshable: &internal.RefreshableConfig{
 			URL:       srvURL,
 			ID:        fltListID,
+			CachePath: cachePath,
+			Staleness: filtertest.Staleness,
+			Timeout:   filtertest.Timeout,
+			MaxSize:   filtertest.FilterMaxSize,
 		},
 		Resolver: &agdtest.Resolver{
 			OnLookupIP: func(
@@ -389,7 +392,6 @@ func TestFilter_FilterRequest_safeSearch(t *testing.T) {
 				panic("not implemented")
 			},
 		},
-		CacheDir:  filepath.Dir(cachePath),
 		CacheTTL:  1 * time.Minute,
 		CacheSize: 100,
 	})
@@ -451,12 +453,14 @@ func TestFilter_FilterResponse(t *testing.T) {
 
 	const (
 		blockedCNAME   = filtertest.ReqHost
+		passedIPv4Str  = "1.1.1.1"
 		blockedIPv4Str = "1.2.3.4"
 		blockedIPv6Str = "1234::cdef"
 		blockRules     = blockedCNAME + "\n" + blockedIPv4Str + "\n" + blockedIPv6Str + "\n"
 	)
 
 	var (
+		passedIPv4  net.IP = netip.MustParseAddr(passedIPv4Str).AsSlice()
 		blockedIPv4 net.IP = netip.MustParseAddr(blockedIPv4Str).AsSlice()
 		blockedIPv6 net.IP = netip.MustParseAddr(blockedIPv6Str).AsSlice()
 	)
@@ -475,6 +479,14 @@ func TestFilter_FilterResponse(t *testing.T) {
 		respAns  dnsservertest.SectionAnswer
 		qType    dnsmsg.RRType
 	}{{
+		name:     "pass",
+		reqFQDN:  filtertest.ReqFQDN,
+		wantRule: "",
+		respAns: dnsservertest.SectionAnswer{
+			dnsservertest.NewA(filtertest.ReqFQDN, ttl, passedIPv4),
+		},
+		qType: dns.TypeA,
+	}, {
 		name:     "cname",
 		reqFQDN:  cnameReqFQDN,
 		wantRule: filtertest.ReqHost,
@@ -499,6 +511,43 @@ func TestFilter_FilterResponse(t *testing.T) {
 			dnsservertest.NewAAAA(filtertest.ReqFQDN, ttl, blockedIPv6),
 		},
 		qType: dns.TypeAAAA,
+	}, {
+		name:     "ipv4hint",
+		reqFQDN:  filtertest.ReqFQDN,
+		wantRule: blockedIPv4Str,
+		respAns: dnsservertest.SectionAnswer{
+			dnsservertest.NewHTTPS(filtertest.ReqFQDN, ttl, []net.IP{blockedIPv4}, []net.IP{}),
+		},
+		qType: dns.TypeHTTPS,
+	}, {
+		name:     "ipv6hint",
+		reqFQDN:  filtertest.ReqFQDN,
+		wantRule: blockedIPv6Str,
+		respAns: dnsservertest.SectionAnswer{
+			dnsservertest.NewHTTPS(filtertest.ReqFQDN, ttl, []net.IP{}, []net.IP{blockedIPv6}),
+		},
+		qType: dns.TypeHTTPS,
+	}, {
+		name:     "ipv4_ipv6_hints",
+		reqFQDN:  filtertest.ReqFQDN,
+		wantRule: blockedIPv4Str,
+		respAns: dnsservertest.SectionAnswer{
+			dnsservertest.NewHTTPS(
+				filtertest.ReqFQDN,
+				ttl,
+				[]net.IP{blockedIPv4},
+				[]net.IP{blockedIPv6},
+			),
+		},
+		qType: dns.TypeHTTPS,
+	}, {
+		name:     "pass_hints",
+		reqFQDN:  filtertest.ReqFQDN,
+		wantRule: "",
+		respAns: dnsservertest.SectionAnswer{
+			dnsservertest.NewHTTPS(filtertest.ReqFQDN, ttl, []net.IP{passedIPv4}, []net.IP{}),
+		},
+		qType: dns.TypeHTTPS,
 	}}
 
 	for _, tc := range testCases {
@@ -511,6 +560,12 @@ func TestFilter_FilterResponse(t *testing.T) {
 			res, err := f.FilterResponse(ctx, resp, ri)
 			require.NoError(t, err)
 
+			if tc.wantRule == "" {
+				assert.Nil(t, res)
+
+				return
+			}
+
 			want := &internal.ResultBlocked{
 				List: testFltListID1,
 				Rule: tc.wantRule,

internal/filter/internal/filtertest/filtertest.go

@@ -16,6 +16,7 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
 	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/c2h5oh/datasize"
 	"github.com/stretchr/testify/require"
 )
 
@@ -35,9 +36,19 @@ const ReqFQDN = ReqHost + "."
 // ServerName is the common server name for filtering tests.
 const ServerName = "testServer/1.0"
 
+// CacheTTL is the common long cache-TTL for filtering tests.
+const CacheTTL = 1 * time.Hour
+
+// Staleness is the common long staleness for filtering tests.
+const Staleness = 1 * time.Hour
+
 // Timeout is the common timeout for filtering tests.
 const Timeout = 1 * time.Second
 
+// FilterMaxSize is the maximum size of the downloadable rule-list for filtering
+// tests.
+const FilterMaxSize = 640 * int64(datasize.KB)
+
 // PrepareRefreshable launches an HTTP server serving the given text and code,
 // as well as creates a cache file.  If code is zero, the server isn't started.
 // If reqCh not nil, a signal is sent every time the server is called.  The

internal/filter/internal/internal.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/c2h5oh/datasize"
 	"github.com/miekg/dns"
 )
 
@@ -29,13 +28,8 @@ type Interface interface {
 	FilterResponse(ctx context.Context, resp *dns.Msg, ri *agd.RequestInfo) (r Result, err error)
 }
 
-// maxFilterSize is the maximum size of downloaded filters.
-const maxFilterSize = 256 * int64(datasize.MB)
-
 // DefaultFilterRefreshTimeout is the default timeout to use when fetching
 // filter lists data.
-//
-// TODO(a.garipov): Consider making timeouts where they are used configurable.
 const DefaultFilterRefreshTimeout = 3 * time.Minute
 
 // DefaultResolveTimeout is the default timeout for resolving hosts for

internal/filter/internal/refreshable.go

@@ -18,45 +18,61 @@ import (
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
-	"github.com/google/renameio"
+	renameio "github.com/google/renameio/v2"
 )
 
 // Refreshable contains entities common to filters that can refresh themselves
 // from a file and a URL.
 type Refreshable struct {
-	// http is the HTTP client used to refresh the filter.
 	http      *agdhttp.Client
-
-	// url is the URL used to refresh the filter.
 	url       *url.URL
-
-	// id is the filter list ID, if any.
 	id        agd.FilterListID
-
-	// cachePath is the path to the file containing the cached filter rules.
 	cachePath string
-
-	// staleness is the time after which a file is considered stale.
 	staleness time.Duration
+	maxSize   int64
 }
 
-// NewRefreshable returns a new refreshable filter.  All parameters must be
+// RefreshableConfig is the configuration structure for a refreshable filter.
-// non-zero.
+type RefreshableConfig struct {
-func NewRefreshable(l *agd.FilterList, cachePath string) (f *Refreshable) {
+	// URL is the URL used to refresh the filter.
+	URL *url.URL
+
+	// ID is the filter list ID for this filter.
+	ID agd.FilterListID
+
+	// CachePath is the path to the file containing the cached filter rules.
+	CachePath string
+
+	// Staleness is the time after which a file is considered stale.
+	Staleness time.Duration
+
+	// Timeout is the timeout for the HTTP client used by this refreshable
+	// filter.
+	Timeout time.Duration
+
+	// MaxSize is the maximum size in bytes of the downloadable filter content.
+	MaxSize int64
+}
+
+// NewRefreshable returns a new refreshable filter.  c must not be nil.
+func NewRefreshable(c *RefreshableConfig) (f *Refreshable) {
 	return &Refreshable{
 		http: agdhttp.NewClient(&agdhttp.ClientConfig{
-			Timeout: DefaultFilterRefreshTimeout,
+			Timeout: c.Timeout,
 		}),
-		url:       l.URL,
+		url:       c.URL,
-		id:        l.ID,
+		id:        c.ID,
-		cachePath: cachePath,
+		cachePath: c.CachePath,
-		staleness: l.RefreshIvl,
+		staleness: c.Staleness,
+		maxSize:   c.MaxSize,
 	}
 }
 
 // Refresh reloads the filter data.  If acceptStale is true, refresh doesn't try
 // to load the filter data from its URL when there is already a file in the
 // cache directory, regardless of its staleness.
+//
+// TODO(a.garipov): Consider making refresh return a reader instead of a string.
 func (f *Refreshable) Refresh(
 	ctx context.Context,
 	acceptStale bool,
@@ -77,6 +93,8 @@ func (f *Refreshable) Refresh(
 		if err != nil {
 			return "", fmt.Errorf("refreshing from url %q: %w", f.url, err)
 		}
+	} else {
+		log.Info("%s: using cached data from file %q", f.id, f.cachePath)
 	}
 
 	return text, nil
@@ -163,7 +181,7 @@ func (f *Refreshable) refreshFromURL(
 
 	b := &strings.Builder{}
 	mw := io.MultiWriter(b, tmpFile)
-	_, err = io.Copy(mw, agdio.LimitReader(resp.Body, maxFilterSize))
+	_, err = io.Copy(mw, agdio.LimitReader(resp.Body, f.maxSize))
 	if err != nil {
 		return "", agdhttp.WrapServerError(fmt.Errorf("reading into file: %w", err), resp)
 	}

internal/filter/internal/refreshable_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/filtertest"
 	"github.com/AdguardTeam/golibs/testutil"
@@ -20,12 +19,13 @@ import (
 // refrID is the ID of a [agd.FilterList] used for testing.
 const refrID = "test_id"
 
-func TestRefreshable_Refresh(t *testing.T) {
+// Default texts for tests.
 const (
-		defaultFileText = "||filefilter.example\n"
+	testFileText = "||filefilter.example\n"
-		defaultURLText  = "||urlfilter.example\n"
+	testURLText  = "||urlfilter.example\n"
 )
 
+func TestRefreshable_Refresh(t *testing.T) {
 	testCases := []struct {
 		name         string
 		wantText     string
@@ -38,9 +38,9 @@ func TestRefreshable_Refresh(t *testing.T) {
 		useCacheFile bool
 	}{{
 		name:         "no_file",
-		wantText:     defaultURLText,
+		wantText:     testURLText,
 		wantErrMsg:   "",
-		srvText:      defaultURLText,
+		srvText:      testURLText,
 		staleness:    0,
 		srvCode:      http.StatusOK,
 		acceptStale:  true,
@@ -71,19 +71,19 @@ func TestRefreshable_Refresh(t *testing.T) {
 		useCacheFile: false,
 	}, {
 		name:         "file",
-		wantText:     defaultFileText,
+		wantText:     testFileText,
 		wantErrMsg:   "",
 		srvText:      "",
-		staleness:    1 * time.Hour,
+		staleness:    filtertest.Staleness,
 		srvCode:      0,
 		acceptStale:  true,
 		expectReq:    false,
 		useCacheFile: true,
 	}, {
 		name:         "file_stale",
-		wantText:     defaultURLText,
+		wantText:     testURLText,
 		wantErrMsg:   "",
-		srvText:      defaultURLText,
+		srvText:      testURLText,
 		staleness:    -1 * time.Hour,
 		srvCode:      http.StatusOK,
 		acceptStale:  false,
@@ -91,7 +91,7 @@ func TestRefreshable_Refresh(t *testing.T) {
 		useCacheFile: true,
 	}, {
 		name:         "file_stale_accept",
-		wantText:     defaultFileText,
+		wantText:     testFileText,
 		wantErrMsg:   "",
 		srvText:      "",
 		staleness:    -1 * time.Hour,
@@ -103,32 +103,25 @@ func TestRefreshable_Refresh(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			var err error
-
 			reqCh := make(chan struct{}, 1)
-			var cachePath string
 			realCachePath, srvURL := filtertest.PrepareRefreshable(t, reqCh, tc.srvText, tc.srvCode)
-			if tc.useCacheFile {
+			cachePath := prepareCachePath(t, realCachePath, tc.useCacheFile)
-				cachePath = realCachePath
 
-				err = os.WriteFile(cachePath, []byte(defaultFileText), 0o600)
+			c := &internal.RefreshableConfig{
-				require.NoError(t, err)
-			} else {
-				cachePath = filepath.Join(t.TempDir(), "does_not_exist")
-			}
-
-			fl := &agd.FilterList{
 				URL:       srvURL,
 				ID:        refrID,
-				RefreshIvl: tc.staleness,
+				CachePath: cachePath,
+				Staleness: tc.staleness,
+				Timeout:   filtertest.Timeout,
+				MaxSize:   filtertest.FilterMaxSize,
 			}
-			f := internal.NewRefreshable(fl, cachePath)
+
+			f := internal.NewRefreshable(c)
 
 			ctx, cancel := context.WithTimeout(context.Background(), filtertest.Timeout)
 			t.Cleanup(cancel)
 
-			var gotText string
+			gotText, err := f.Refresh(ctx, tc.acceptStale)
-			gotText, err = f.Refresh(ctx, tc.acceptStale)
 			if tc.expectReq {
 				testutil.RequireReceive(t, reqCh, filtertest.Timeout)
 			}
@@ -145,21 +138,38 @@ func TestRefreshable_Refresh(t *testing.T) {
 	}
 }
 
+// prepareCachePath is a helper that either returns a non-existing file (if
+// useCacheFile is false) or prepares a cache file using realCachePath and
+// [testFileText].
+func prepareCachePath(t *testing.T, realCachePath string, useCacheFile bool) (cachePath string) {
+	t.Helper()
+
+	if !useCacheFile {
+		return filepath.Join(t.TempDir(), "does_not_exist")
+	}
+
+	err := os.WriteFile(realCachePath, []byte(testFileText), 0o600)
+	require.NoError(t, err)
+
+	return realCachePath
+}
+
 func TestRefreshable_Refresh_properStaleness(t *testing.T) {
-	const (
+	const responseDur = time.Second / 5
-		responseDur = time.Second / 5
-		staleness   = time.Hour
-	)
 
 	reqCh := make(chan struct{})
 	cachePath, addr := filtertest.PrepareRefreshable(t, reqCh, filtertest.BlockRule, http.StatusOK)
 
-	fl := &agd.FilterList{
+	c := &internal.RefreshableConfig{
 		URL:       addr,
 		ID:        refrID,
-		RefreshIvl: staleness,
+		CachePath: cachePath,
+		Staleness: filtertest.Staleness,
+		Timeout:   filtertest.Timeout,
+		MaxSize:   filtertest.FilterMaxSize,
 	}
-	f := internal.NewRefreshable(fl, cachePath)
+
+	f := internal.NewRefreshable(c)
 
 	ctx, cancel := context.WithTimeout(context.Background(), filtertest.Timeout)
 	t.Cleanup(cancel)

internal/filter/internal/rulelist/refreshable.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net/netip"
-	"path/filepath"
 	"sync"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
@@ -32,21 +31,28 @@ type Refreshable struct {
 }
 
 // NewRefreshable returns a new refreshable DNS request and response filter
-// based on the provided rule list.  l must be non-nil.  The initial refresh
+// based on the provided rule list.  c must be non-nil.  The initial refresh
 // should be called explicitly if necessary.
 func NewRefreshable(
-	l *agd.FilterList,
+	c *internal.RefreshableConfig,
-	fileCacheDir string,
 	memCacheSize int,
 	useMemCache bool,
 ) (f *Refreshable) {
 	f = &Refreshable{
 		mu: &sync.RWMutex{},
-		refr: internal.NewRefreshable(l, filepath.Join(fileCacheDir, string(l.ID))),
+		refr: internal.NewRefreshable(&internal.RefreshableConfig{
+			URL:       c.URL,
+			ID:        c.ID,
+			CachePath: c.CachePath,
+			Staleness: c.Staleness,
+			// TODO(ameshkov): Consider making configurable.
+			Timeout: internal.DefaultFilterRefreshTimeout,
+			MaxSize: c.MaxSize,
+		}),
 	}
 
 	var err error
-	f.filter, err = newFilter("", l.ID, "", memCacheSize, useMemCache)
+	f.filter, err = newFilter("", c.ID, "", memCacheSize, useMemCache)
 	if err != nil {
 		// Should never happen, since text is empty.
 		panic(fmt.Errorf("unexpected filter error: %w", err))

internal/filter/internal/rulelist/refreshable_test.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"net/http"
 	"net/netip"
-	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
+	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/filtertest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/rulelist"
 	"github.com/AdguardTeam/golibs/testutil"
@@ -82,12 +81,13 @@ func TestRefreshable_ID(t *testing.T) {
 func TestRefreshable_Refresh(t *testing.T) {
 	cachePath, srvURL := filtertest.PrepareRefreshable(t, nil, testBlockRule, http.StatusOK)
 	rl := rulelist.NewRefreshable(
-		&agd.FilterList{
+		&internal.RefreshableConfig{
 			URL:       srvURL,
 			ID:        testFltListID,
-			RefreshIvl: 1 * time.Hour,
+			CachePath: cachePath,
+			Staleness: filtertest.Staleness,
+			MaxSize:   filtertest.FilterMaxSize,
 		},
-		filepath.Dir(cachePath),
 		100,
 		true,
 	)

internal/filter/internal/rulelist/rulelist.go

@@ -18,7 +18,7 @@ import (
 // newURLFilterID returns a new random ID for the urlfilter DNS engine to use.
 func newURLFilterID() (id int) {
 	// #nosec G404 -- Do not use cryptographically random ID generation, since
-	// these are only used in one place, internal/filter.compFilter.filterMsg,
+	// these are only used in ../composite/Filter.mustRuleListDataByURLFilterID
 	// and are not used in any security-sensitive context.
 	//
 	// Despite the fact that the type of integer filter list IDs in module

internal/filter/internal/safesearch/safesearch.go

@@ -32,8 +32,9 @@ type Filter struct {
 
 // Config contains configuration for the safe-search filter.
 type Config struct {
-	// List is the filtering-rule list used to filter requests.
+	// Refreshable is the configuration of the refreshable filter-list within
-	List *agd.FilterList
+	// the safe-search filter.
+	Refreshable *internal.RefreshableConfig
 
 	// Resolver is used to resolve the IP addresses of replacement hosts.
 	Resolver agdnet.Resolver
@@ -41,10 +42,6 @@ type Config struct {
 	// ErrColl is used to report errors of replacement-host resolving.
 	ErrColl agd.ErrorCollector
 
-	// CacheDir is the path to the directory where the cached filter files are
-	// put.  The directory must exist.
-	CacheDir string
-
 	// CacheTTL is the time to live of the result cache-items.
 	//
 	//lint:ignore U1000 TODO(a.garipov): Currently unused.  See AGDNS-398.
@@ -57,15 +54,13 @@ type Config struct {
 // New returns a new safe-search filter.  c must not be nil.  The initial
 // refresh should be called explicitly if necessary.
 func New(c *Config) (f *Filter) {
-	id := c.List.ID
-
 	return &Filter{
 		resCache: resultcache.New[*internal.ResultModified](c.CacheSize),
 		// Don't use the rule list cache, since safeSearch already has its own.
-		flt:      rulelist.NewRefreshable(c.List, c.CacheDir, 0, false),
+		flt:      rulelist.NewRefreshable(c.Refreshable, 0, false),
 		resolver: c.Resolver,
 		errColl:  c.ErrColl,
-		id:       id,
+		id:       c.Refreshable.ID,
 	}
 }
 

internal/filter/internal/safesearch/safesearch_test.go

@@ -60,9 +60,13 @@ func TestFilter(t *testing.T) {
 	require.NoError(t, err)
 
 	f := safesearch.New(&safesearch.Config{
-		List: &agd.FilterList{
+		Refreshable: &internal.RefreshableConfig{
 			ID:        id,
 			URL:       srvURL,
+			CachePath: cachePath,
+			Staleness: filtertest.Staleness,
+			Timeout:   filtertest.Timeout,
+			MaxSize:   filtertest.FilterMaxSize,
 		},
 		Resolver: &agdtest.Resolver{
 			OnLookupIP: func(
@@ -85,7 +89,6 @@ func TestFilter(t *testing.T) {
 				panic("not implemented")
 			},
 		},
-		CacheDir:  filepath.Dir(cachePath),
 		CacheTTL:  1 * time.Minute,
 		CacheSize: 100,
 	})

internal/filter/internal/serviceblock/serviceblock.go

@@ -7,27 +7,22 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
+	"strings"
-	"net/url"
 	"sync"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/rulelist"
 	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
-	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 )
 
 // Filter is a service-blocking filter that uses rule lists that it gets from an
 // index.
 type Filter struct {
-	// url is the URL from which the services are fetched.
+	// refr is the helper entity containing the refreshable part of the index
-	url *url.URL
+	// refresh and caching logic.
-
+	refr *internal.Refreshable
-	// http is the HTTP client used to refresh the filter.
-	http *agdhttp.Client
 
 	// mu protects services.
 	mu *sync.RWMutex
@@ -39,17 +34,15 @@ type Filter struct {
 	errColl agd.ErrorCollector
 }
 
-// serviceRuleLists is convenient alias for a ID to filter mapping.
+// serviceRuleLists is convenient alias for an ID to filter mapping.
 type serviceRuleLists = map[agd.BlockedServiceID]*rulelist.Immutable
 
 // New returns a fully initialized service blocker.
-func New(indexURL *url.URL, errColl agd.ErrorCollector) (f *Filter) {
+func New(refr *internal.Refreshable, errColl agd.ErrorCollector) (f *Filter) {
 	return &Filter{
-		url: indexURL,
+		refr:     refr,
-		http: agdhttp.NewClient(&agdhttp.ClientConfig{
-			Timeout: internal.DefaultFilterRefreshTimeout,
-		}),
 		mu:       &sync.RWMutex{},
+		services: serviceRuleLists{},
 		errColl:  errColl,
 	}
 }
@@ -69,22 +62,23 @@ func (f *Filter) RuleLists(
 
 	for _, id := range ids {
 		rl := f.services[id]
-		if rl != nil {
+		if rl == nil {
+			log.Info("service filter: warning: no service with id %s", id)
+		} else {
 			rls = append(rls, rl)
-
-			continue
 		}
-
-		reportErr := fmt.Errorf("service filter: no service with id %s", id)
-		f.errColl.Collect(ctx, reportErr)
-		log.Info("warning: %s", reportErr)
 	}
 
 	return rls
 }
 
 // Refresh loads new service data from the index URL.
-func (f *Filter) Refresh(ctx context.Context, cacheSize int, useCache bool) (err error) {
+func (f *Filter) Refresh(
+	ctx context.Context,
+	cacheSize int,
+	useCache bool,
+	acceptStale bool,
+) (err error) {
 	fltIDStr := string(agd.FilterListIDBlockedService)
 	defer func() {
 		if err != nil {
@@ -92,7 +86,7 @@ func (f *Filter) Refresh(ctx context.Context, cacheSize int, useCache bool) (err
 		}
 	}()
 
-	resp, err := f.loadIndex(ctx)
+	resp, err := f.loadIndex(ctx, acceptStale)
 	if err != nil {
 		// Don't wrap the error, because it's informative enough as is.
 		return err
@@ -122,25 +116,16 @@ func (f *Filter) Refresh(ctx context.Context, cacheSize int, useCache bool) (err
 }
 
 // loadIndex fetches, decodes, and returns the blocked service index data.
-func (f *Filter) loadIndex(ctx context.Context) (resp *indexResp, err error) {
+func (f *Filter) loadIndex(ctx context.Context, acceptStale bool) (resp *indexResp, err error) {
-	defer func() { err = errors.Annotate(err, "loading blocked service index from %q: %w", f.url) }()
+	text, err := f.refr.Refresh(ctx, acceptStale)
-
-	httpResp, err := f.http.Get(ctx, f.url)
 	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
+		return nil, fmt.Errorf("loading index: %w", err)
-	}
-	defer func() { err = errors.WithDeferred(err, httpResp.Body.Close()) }()
-
-	err = agdhttp.CheckStatus(httpResp, http.StatusOK)
-	if err != nil {
-		// Don't wrap the error, because it's informative enough as is.
-		return nil, err
 	}
 
 	resp = &indexResp{}
-	err = json.NewDecoder(httpResp.Body).Decode(resp)
+	err = json.NewDecoder(strings.NewReader(text)).Decode(resp)
 	if err != nil {
-		return nil, agdhttp.WrapServerError(fmt.Errorf("decoding: %w", err), httpResp)
+		return nil, fmt.Errorf("decoding index: %w", err)
 	}
 
 	log.Debug("service filter: loaded index with %d blocked services", len(resp.BlockedServices))

internal/filter/internal/serviceblock/serviceblock_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdtest"
+	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/filtertest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/serviceblock"
 	"github.com/AdguardTeam/golibs/testutil"
@@ -14,15 +15,20 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestMain(m *testing.M) {
+	testutil.DiscardLogOutput(m)
+}
+
 // Common blocked service IDs for tests.
 const (
 	testSvcID1          agd.BlockedServiceID = "svc_1"
 	testSvcID2          agd.BlockedServiceID = "svc_2"
+	testSvcIDNotPresent agd.BlockedServiceID = "svc_not_present"
 )
 
 // testData is a sample of a service index response.
 //
-// See https://github.com/atropnikov/HostlistsRegistry/blob/main/assets/services.json.
+// See https://github.com/AdguardTeam/HostlistsRegistry/blob/main/assets/services.json.
 const testData string = `{
   "blocked_services": [
     {
@@ -44,7 +50,7 @@ const testData string = `{
 
 func TestFilter(t *testing.T) {
 	reqCh := make(chan struct{}, 1)
-	_, srvURL := filtertest.PrepareRefreshable(t, reqCh, testData, http.StatusOK)
+	cachePath, srvURL := filtertest.PrepareRefreshable(t, reqCh, testData, http.StatusOK)
 
 	errColl := &agdtest.ErrorCollector{
 		OnCollect: func(ctx context.Context, err error) {
@@ -52,23 +58,40 @@ func TestFilter(t *testing.T) {
 		},
 	}
 
-	f := serviceblock.New(srvURL, errColl)
+	refr := internal.NewRefreshable(&internal.RefreshableConfig{
+		URL:       srvURL,
+		ID:        agd.FilterListIDBlockedService,
+		CachePath: cachePath,
+		Staleness: filtertest.Staleness,
+		Timeout:   filtertest.Timeout,
+		MaxSize:   filtertest.FilterMaxSize,
+	})
+
+	f := serviceblock.New(refr, errColl)
 
 	ctx := context.Background()
-	err := f.Refresh(ctx, 0, false)
+	err := f.Refresh(ctx, 0, false, false)
 	require.NoError(t, err)
 
 	testutil.RequireReceive(t, reqCh, filtertest.Timeout)
 
-	svcIDs := []agd.BlockedServiceID{testSvcID1, testSvcID2}
+	rls := f.RuleLists(ctx, []agd.BlockedServiceID{
-	rls := f.RuleLists(ctx, svcIDs)
+		testSvcID1,
+		testSvcID2,
+		testSvcIDNotPresent,
+	})
 	require.Len(t, rls, 2)
 
+	wantSvcIDs := []agd.BlockedServiceID{
+		testSvcID1,
+		testSvcID2,
+	}
+
 	gotFltIDs := make([]agd.FilterListID, 2)
 	gotSvcIDs := make([]agd.BlockedServiceID, 2)
 	gotFltIDs[0], gotSvcIDs[0] = rls[0].ID()
 	gotFltIDs[1], gotSvcIDs[1] = rls[1].ID()
 	assert.Equal(t, agd.FilterListIDBlockedService, gotFltIDs[0])
 	assert.Equal(t, agd.FilterListIDBlockedService, gotFltIDs[1])
-	assert.ElementsMatch(t, svcIDs, gotSvcIDs)
+	assert.ElementsMatch(t, wantSvcIDs, gotSvcIDs)
 }

internal/filter/storage.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"net/url"
+	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/AdguardTeam/AdGuardDNS/internal/agd"
-	"github.com/AdguardTeam/AdGuardDNS/internal/agdhttp"
 	"github.com/AdguardTeam/AdGuardDNS/internal/agdnet"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/hashprefix"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal"
@@ -20,13 +20,10 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/safesearch"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/serviceblock"
 	"github.com/AdguardTeam/AdGuardDNS/internal/metrics"
-	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/bluele/gcache"
 )
 
-// Filter storage
-
 // Storage is a storage for filters.
 type Storage interface {
 	// FilterFromContext returns a filter combining rules and types of filtering
@@ -42,19 +39,15 @@ type Storage interface {
 // based on rule lists, custom filters of profiles, safe browsing, and safe
 // search ones.
 type DefaultStorage struct {
+	// refr is the helper entity containing the refreshable part of the index
+	// refresh and caching logic.
+	refr *internal.Refreshable
+
 	// mu protects ruleLists.
 	mu *sync.RWMutex
 
-	// URL is the URL of the filtering rule index document.  See filterIndexResp
-	// and related types.
-	url *url.URL
-
-	// http is the HTTP client used to update the rule list filters from the
-	// index.
-	http *agdhttp.Client
-
 	// ruleLists are the filter list ID to a rule list filter map.
-	ruleLists map[agd.FilterListID]*rulelist.Refreshable
+	ruleLists filteringRuleLists
 
 	// services is the service blocking filter.
 	services *serviceblock.Filter
@@ -65,6 +58,9 @@ type DefaultStorage struct {
 	// adultBlocking is the adult content blocking safe browsing filter.
 	adultBlocking *hashprefix.Filter
 
+	// newRegDomains is the newly registered domains filter.
+	newRegDomains *hashprefix.Filter
+
 	// genSafeSearch is the general safe search filter.
 	genSafeSearch *safesearch.Filter
 
@@ -93,10 +89,25 @@ type DefaultStorage struct {
 	// filtering results.
 	ruleListCacheSize int
 
+	// maxRuleListSize is the maximum size in bytes of the downloadable
+	// rule-list content.
+	maxRuleListSize int64
+
 	// useRuleListCache, if true, enables rule list cache.
 	useRuleListCache bool
 }
 
+// filteringRuleLists is convenient alias for an ID to filter mapping.
+type filteringRuleLists = map[agd.FilterListID]*rulelist.Refreshable
+
+// Filenames for filter indexes.
+//
+// TODO(ameshkov): Consider making configurable.
+const (
+	ruleListIndexFilename = "filters.json"
+	serviceIndexFilename  = "services.json"
+)
+
 // DefaultStorageConfig contains configuration for a filter storage based on
 // rule lists.
 type DefaultStorageConfig struct {
@@ -122,6 +133,10 @@ type DefaultStorageConfig struct {
 	// browsing filter.  It must not be nil.
 	AdultBlocking *hashprefix.Filter
 
+	// NewRegDomains is the configuration for the newly registered domains safe
+	// browsing filter.  It must not be nil.
+	NewRegDomains *hashprefix.Filter
+
 	// Now is a function that returns current time.
 	Now func() (now time.Time)
 
@@ -153,49 +168,84 @@ type DefaultStorageConfig struct {
 
 	// RefreshIvl is the refresh interval for this storage.  It defines how
 	// often the filter rule lists are updated from the index.
+	//
+	// TODO(a.garipov): This value is used both for refreshes and for filter
+	// staleness, which can cause issues.  Consider splitting the two.
 	RefreshIvl time.Duration
 
 	// UseRuleListCache, if true, enables rule list cache.
 	UseRuleListCache bool
+
+	// MaxRuleListSize is the maximum size in bytes of the downloadable
+	// rule-list content.
+	MaxRuleListSize int64
 }
 
 // NewDefaultStorage returns a new filter storage.  c must not be nil.
 func NewDefaultStorage(c *DefaultStorageConfig) (s *DefaultStorage, err error) {
 	genSafeSearch := safesearch.New(&safesearch.Config{
-		List: &agd.FilterList{
+		Refreshable: &internal.RefreshableConfig{
 			URL:       c.GeneralSafeSearchRulesURL,
 			ID:        agd.FilterListIDGeneralSafeSearch,
-			RefreshIvl: c.RefreshIvl,
+			CachePath: filepath.Join(c.CacheDir, string(agd.FilterListIDGeneralSafeSearch)),
+			Staleness: c.RefreshIvl,
+			// TODO(ameshkov): Consider making configurable.
+			Timeout: internal.DefaultFilterRefreshTimeout,
+			MaxSize: c.MaxRuleListSize,
 		},
 		Resolver:  c.Resolver,
 		ErrColl:   c.ErrColl,
-		CacheDir:  c.CacheDir,
 		CacheTTL:  c.SafeSearchCacheTTL,
 		CacheSize: c.SafeSearchCacheSize,
 	})
 
 	ytSafeSearch := safesearch.New(&safesearch.Config{
-		List: &agd.FilterList{
+		Refreshable: &internal.RefreshableConfig{
 			URL:       c.YoutubeSafeSearchRulesURL,
 			ID:        agd.FilterListIDYoutubeSafeSearch,
-			RefreshIvl: c.RefreshIvl,
+			CachePath: filepath.Join(c.CacheDir, string(agd.FilterListIDYoutubeSafeSearch)),
+			Staleness: c.RefreshIvl,
+			// TODO(ameshkov): Consider making configurable.
+			Timeout: internal.DefaultFilterRefreshTimeout,
+			MaxSize: c.MaxRuleListSize,
 		},
 		Resolver:  c.Resolver,
 		ErrColl:   c.ErrColl,
-		CacheDir:  c.CacheDir,
 		CacheTTL:  c.SafeSearchCacheTTL,
 		CacheSize: c.SafeSearchCacheSize,
 	})
 
-	s = &DefaultStorage{
+	ruleListIdxRefr := internal.NewRefreshable(&internal.RefreshableConfig{
-		mu:  &sync.RWMutex{},
+		URL: c.FilterIndexURL,
-		url: c.FilterIndexURL,
+		// TODO(a.garipov): Consider adding special IDs for indexes.
-		http: agdhttp.NewClient(&agdhttp.ClientConfig{
+		ID:        "rule_list_index",
+		CachePath: filepath.Join(c.CacheDir, ruleListIndexFilename),
+		Staleness: c.RefreshIvl,
+		// TODO(ameshkov): Consider making configurable.
 		Timeout: internal.DefaultFilterRefreshTimeout,
-		}),
+		// TODO(a.garipov): Consider using a different limit here.
-		services:      serviceblock.New(c.BlockedServiceIndexURL, c.ErrColl),
+		MaxSize: c.MaxRuleListSize,
+	})
+
+	svcIdxRefr := internal.NewRefreshable(&internal.RefreshableConfig{
+		URL: c.BlockedServiceIndexURL,
+		// TODO(a.garipov): Consider adding special IDs for indexes.
+		ID:        "blocked_service_index",
+		CachePath: filepath.Join(c.CacheDir, serviceIndexFilename),
+		Staleness: c.RefreshIvl,
+		// TODO(ameshkov): Consider making configurable.
+		Timeout: internal.DefaultFilterRefreshTimeout,
+		// TODO(a.garipov): Consider using a different limit here.
+		MaxSize: c.MaxRuleListSize,
+	})
+
+	s = &DefaultStorage{
+		refr:          ruleListIdxRefr,
+		mu:            &sync.RWMutex{},
+		services:      serviceblock.New(svcIdxRefr, c.ErrColl),
 		safeBrowsing:  c.SafeBrowsing,
 		adultBlocking: c.AdultBlocking,
+		newRegDomains: c.NewRegDomains,
 		genSafeSearch: genSafeSearch,
 		ytSafeSearch:  ytSafeSearch,
 		now:           c.Now,
@@ -208,6 +258,7 @@ func NewDefaultStorage(c *DefaultStorageConfig) (s *DefaultStorage, err error) {
 		refreshIvl:        c.RefreshIvl,
 		ruleListCacheSize: c.RuleListCacheSize,
 		useRuleListCache:  c.UseRuleListCache,
+		maxRuleListSize:   c.MaxRuleListSize,
 	}
 
 	err = s.refresh(context.Background(), true)
@@ -234,7 +285,7 @@ func (s *DefaultStorage) FilterFromContext(ctx context.Context, ri *agd.RequestI
 		c.RuleLists = s.filters(g.RuleListIDs)
 	}
 
-	c.SafeBrowsing, c.AdultBlocking = s.safeBrowsingForGroup(g)
+	c.SafeBrowsing, c.AdultBlocking, c.NewRegisteredDomains = s.safeBrowsingForGroup(g)
 	c.GeneralSafeSearch, c.YouTubeSafeSearch = s.safeSearchForGroup(g)
 
 	return composite.New(c)
@@ -267,7 +318,7 @@ func (s *DefaultStorage) filterForProfile(ctx context.Context, ri *agd.RequestIn
 
 	c.ServiceLists = s.serviceFilters(ctx, p, parentalEnabled)
 
-	c.SafeBrowsing, c.AdultBlocking = s.safeBrowsingForProfile(p, parentalEnabled)
+	c.SafeBrowsing, c.AdultBlocking, c.NewRegisteredDomains = s.safeBrowsingForProfile(p, parentalEnabled)
 	c.GeneralSafeSearch, c.YouTubeSafeSearch = s.safeSearchForProfile(p, parentalEnabled)
 
 	return composite.New(c)
@@ -302,16 +353,22 @@ func (s *DefaultStorage) pcBySchedule(sch *agd.ParentalProtectionSchedule) (ok b
 func (s *DefaultStorage) safeBrowsingForProfile(
 	p *agd.Profile,
 	parentalEnabled bool,
-) (safeBrowsing, adultBlocking *hashprefix.Filter) {
+) (safeBrowsing, adultBlocking, newRegDomains *hashprefix.Filter) {
-	if p.SafeBrowsingEnabled {
+	if p.SafeBrowsing != nil && p.SafeBrowsing.Enabled {
+		if p.SafeBrowsing.BlockDangerousDomains {
 			safeBrowsing = s.safeBrowsing
 		}
 
+		if p.SafeBrowsing.BlockNewlyRegisteredDomains {
+			newRegDomains = s.newRegDomains
+		}
+	}
+
 	if parentalEnabled && p.Parental.BlockAdult {
 		adultBlocking = s.adultBlocking
 	}
 
-	return safeBrowsing, adultBlocking
+	return safeBrowsing, adultBlocking, newRegDomains
 }
 
 // safeSearchForProfile returns safe search filters based on the information in
@@ -339,16 +396,22 @@ func (s *DefaultStorage) safeSearchForProfile(
 // in the filtering group.  g must not be nil.
 func (s *DefaultStorage) safeBrowsingForGroup(
 	g *agd.FilteringGroup,
-) (safeBrowsing, adultBlocking *hashprefix.Filter) {
+) (safeBrowsing, adultBlocking, newRegDomains *hashprefix.Filter) {
 	if g.SafeBrowsingEnabled {
+		if g.BlockDangerousDomains {
 			safeBrowsing = s.safeBrowsing
 		}
 
+		if g.BlockNewlyRegisteredDomains {
+			newRegDomains = s.newRegDomains
+		}
+	}
+
 	if g.ParentalEnabled && g.BlockAdult {
 		adultBlocking = s.adultBlocking
 	}
 
-	return safeBrowsing, adultBlocking
+	return safeBrowsing, adultBlocking, newRegDomains
 }
 
 // safeSearchForGroup returns safe search filters based on the information in
@@ -414,9 +477,7 @@ func (s *DefaultStorage) Refresh(ctx context.Context) (err error) {
 // refreshes the index from the index URL and updates all rule list filters, as
 // well as the service filters.
 func (s *DefaultStorage) refresh(ctx context.Context, acceptStale bool) (err error) {
-	log.Info("%s: requesting %s", strgLogPrefix, s.url)
+	resp, err := s.loadIndex(ctx, acceptStale)
-
-	resp, err := s.loadIndex(ctx)
 	if err != nil {
 		// Don't wrap the error, because it's informative enough as is.
 		return err
@@ -424,50 +485,18 @@ func (s *DefaultStorage) refresh(ctx context.Context, acceptStale bool) (err err
 
 	log.Info("%s: got %d filters from index", strgLogPrefix, len(resp.Filters))
 
-	fls := resp.toInternal(ctx, s.errColl, s.refreshIvl)
+	fls := resp.toInternal(ctx, s.errColl)
 
 	log.Info("%s: got %d filter lists from index after validations", strgLogPrefix, len(fls))
 
-	ruleLists := make(map[agd.FilterListID]*rulelist.Refreshable, len(resp.Filters))
+	ruleLists := make(filteringRuleLists, len(resp.Filters))
 	for _, fl := range fls {
-		if _, ok := ruleLists[fl.ID]; ok {
+		s.addRuleList(ctx, ruleLists, fl, acceptStale)
-			agd.Collectf(ctx, s.errColl, "%s: duplicated id %q", strgLogPrefix, fl.ID)
-
-			continue
-		}
-
-		fltIDStr := string(fl.ID)
-		rl := rulelist.NewRefreshable(
-			fl,
-			s.cacheDir,
-			s.ruleListCacheSize,
-			s.useRuleListCache,
-		)
-		err = rl.Refresh(ctx, acceptStale)
-		if err == nil {
-			ruleLists[fl.ID] = rl
-
-			metrics.FilterUpdatedStatus.WithLabelValues(fltIDStr).Set(1)
-			metrics.FilterUpdatedTime.WithLabelValues(fltIDStr).SetToCurrentTime()
-			metrics.FilterRulesTotal.WithLabelValues(fltIDStr).Set(float64(rl.RulesCount()))
-
-			continue
-		}
-
-		agd.Collectf(ctx, s.errColl, "%s: refreshing %q: %w", strgLogPrefix, fl.ID, err)
-		metrics.FilterUpdatedStatus.WithLabelValues(fltIDStr).Set(0)
-
-		// If we can't get the new filter, and there is an old version of the
-		// same rule list, use it.
-		rls := s.filters([]agd.FilterListID{fl.ID})
-		if len(rls) > 0 {
-			ruleLists[fl.ID] = rls[0]
-		}
 	}
 
 	log.Info("%s: got %d filter lists from index after compilation", strgLogPrefix, len(ruleLists))
 
-	err = s.services.Refresh(ctx, s.ruleListCacheSize, s.useRuleListCache)
+	err = s.services.Refresh(ctx, s.ruleListCacheSize, s.useRuleListCache, acceptStale)
 	if err != nil {
 		const errFmt = "refreshing blocked services: %w"
 		agd.Collectf(ctx, s.errColl, errFmt, err)
@@ -490,90 +519,80 @@ func (s *DefaultStorage) refresh(ctx context.Context, acceptStale bool) (err err
 	return nil
 }
 
+// addRuleList adds the data from fl to ruleLists and handles all validations
+// and errors.
+func (s *DefaultStorage) addRuleList(
+	ctx context.Context,
+	ruleLists filteringRuleLists,
+	fl *filterIndexFilterData,
+	acceptStale bool,
+) {
+	if _, ok := ruleLists[fl.id]; ok {
+		agd.Collectf(ctx, s.errColl, "%s: duplicated id %q", strgLogPrefix, fl.id)
+
+		return
+	}
+
+	fltIDStr := string(fl.id)
+	rl := rulelist.NewRefreshable(
+		&internal.RefreshableConfig{
+			URL:       fl.url,
+			ID:        fl.id,
+			CachePath: filepath.Join(s.cacheDir, fltIDStr),
+			Staleness: s.refreshIvl,
+			// TODO(ameshkov): Consider making configurable.
+			Timeout: internal.DefaultFilterRefreshTimeout,
+			MaxSize: s.maxRuleListSize,
+		},
+		s.ruleListCacheSize,
+		s.useRuleListCache,
+	)
+	err := rl.Refresh(ctx, acceptStale)
+	if err == nil {
+		ruleLists[fl.id] = rl
+
+		metrics.FilterUpdatedStatus.WithLabelValues(fltIDStr).Set(1)
+		metrics.FilterUpdatedTime.WithLabelValues(fltIDStr).SetToCurrentTime()
+		metrics.FilterRulesTotal.WithLabelValues(fltIDStr).Set(float64(rl.RulesCount()))
+
+		return
+	}
+
+	agd.Collectf(ctx, s.errColl, "%s: refreshing %q: %w", strgLogPrefix, fl.id, err)
+	metrics.FilterUpdatedStatus.WithLabelValues(fltIDStr).Set(0)
+
+	// If we can't get the new filter, and there is an old version of the same
+	// rule list, use it.
+	rls := s.filters([]agd.FilterListID{fl.id})
+	if len(rls) > 0 {
+		ruleLists[fl.id] = rls[0]
+	}
+}
+
 // loadIndex fetches, decodes, and returns the filter list index data of the
 // storage.
-func (s *DefaultStorage) loadIndex(ctx context.Context) (resp *filterIndexResp, err error) {
+func (s *DefaultStorage) loadIndex(
-	defer func() { err = errors.Annotate(err, "loading filter index from %q: %w", s.url) }()
+	ctx context.Context,
-
+	acceptStale bool,
-	httpResp, err := s.http.Get(ctx, s.url)
+) (resp *filterIndexResp, err error) {
+	text, err := s.refr.Refresh(ctx, acceptStale)
 	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
+		return nil, fmt.Errorf("loading index: %w", err)
-	}
-	defer func() { err = errors.WithDeferred(err, httpResp.Body.Close()) }()
-
-	err = agdhttp.CheckStatus(httpResp, http.StatusOK)
-	if err != nil {
-		// Don't wrap the error, because it's informative enough as is.
-		return nil, err
 	}
 
 	resp = &filterIndexResp{}
-	err = json.NewDecoder(httpResp.Body).Decode(resp)
+	err = json.NewDecoder(strings.NewReader(text)).Decode(resp)
 	if err != nil {
 		return nil, fmt.Errorf("decoding: %w", err)
 	}
 
-	log.Debug("%s: loaded index with %d filters", strgLogPrefix, len(resp.Filters))
-
 	return resp, nil
 }
 
 // setRuleLists replaces the storage's rule lists.
-func (s *DefaultStorage) setRuleLists(ruleLists map[agd.FilterListID]*rulelist.Refreshable) {
+func (s *DefaultStorage) setRuleLists(ruleLists filteringRuleLists) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	s.ruleLists = ruleLists
 }
-
-// filterIndexResp is the struct for the JSON response from a filter index API.
-type filterIndexResp struct {
-	Filters []*filterIndexRespFilter `json:"filters"`
-}
-
-// toInternal converts the filters from the index to []*agd.FilterList.
-func (r *filterIndexResp) toInternal(
-	ctx context.Context,
-	errColl agd.ErrorCollector,
-	refreshIvl time.Duration,
-) (fls []*agd.FilterList) {
-	fls = make([]*agd.FilterList, 0, len(r.Filters))
-	for _, rf := range r.Filters {
-		id, err := agd.NewFilterListID(rf.ID)
-		if err != nil {
-			agd.Collectf(ctx, errColl, "%s: validating id %q: %w", strgLogPrefix, rf.ID, err)
-
-			continue
-		}
-
-		var u *url.URL
-		u, err = agdhttp.ParseHTTPURL(rf.DownloadURL)
-		if err != nil {
-			agd.Collectf(
-				ctx,
-				errColl,
-				"%s: validating url %q: %w",
-				strgLogPrefix,
-				rf.DownloadURL,
-				err,
-			)
-
-			continue
-		}
-
-		fls = append(fls, &agd.FilterList{
-			URL:        u,
-			ID:         id,
-			RefreshIvl: refreshIvl,
-		})
-	}
-
-	return fls
-}
-
-// filterIndexRespFilter is the struct for a filter from the JSON response from
-// a filter index API.
-type filterIndexRespFilter struct {
-	DownloadURL string `json:"downloadUrl"`
-	ID          string `json:"filterId"`
-}

internal/filter/storage_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/AdguardTeam/AdGuardDNS/internal/dnsserver/dnsservertest"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter"
 	"github.com/AdguardTeam/AdGuardDNS/internal/filter/hashprefix"
+	"github.com/AdguardTeam/AdGuardDNS/internal/filter/internal/filtertest"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/testutil"
 	"github.com/miekg/dns"
@@ -159,8 +160,8 @@ func TestStorage_FilterFromContext_customAllow(t *testing.T) {
 		ID:              agd.FilterListIDSafeBrowsing,
 		CachePath:       tmpFile.Name(),
 		ReplacementHost: safeBrowsingSafeHost,
-		Staleness:       1 * time.Hour,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        10 * time.Second,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       100,
 	})
 	require.NoError(t, err)
@@ -178,7 +179,11 @@ func TestStorage_FilterFromContext_customAllow(t *testing.T) {
 		},
 		ID:               "prof1234",
 		FilteringEnabled: true,
-		SafeBrowsingEnabled: true,
+		SafeBrowsing: &agd.SafeBrowsingSettings{
+			Enabled:                     true,
+			BlockDangerousDomains:       true,
+			BlockNewlyRegisteredDomains: false,
+		},
 		CustomRules: []agd.FilterRuleText{
 			safeBrowsingAllowRule,
 		},
@@ -252,8 +257,8 @@ func TestStorage_FilterFromContext_schedule(t *testing.T) {
 		ID:              agd.FilterListIDAdultBlocking,
 		CachePath:       tmpFile.Name(),
 		ReplacementHost: safeBrowsingSafeHost,
-		Staleness:       1 * time.Hour,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        10 * time.Second,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       100,
 	})
 	require.NoError(t, err)
@@ -754,8 +759,8 @@ func TestStorage_FilterFromContext_safeBrowsing(t *testing.T) {
 		ID:              agd.FilterListIDSafeBrowsing,
 		CachePath:       cachePath,
 		ReplacementHost: safeBrowsingSafeHost,
-		Staleness:       1 * time.Hour,
+		Staleness:       filtertest.Staleness,
-		CacheTTL:        10 * time.Second,
+		CacheTTL:        filtertest.CacheTTL,
 		CacheSize:       100,
 	})
 	require.NoError(t, err)
@@ -771,6 +776,8 @@ func TestStorage_FilterFromContext_safeBrowsing(t *testing.T) {
 		RuleListIDs:                 []agd.FilterListID{},
 		ParentalEnabled:             true,
 		SafeBrowsingEnabled:         true,
+		BlockDangerousDomains:       true,
+		BlockNewlyRegisteredDomains: true,
 	}
 
 	// Test

internal/geoip/asntops.go

@@ -4,8 +4,9 @@ package geoip
 
 import "github.com/AdguardTeam/AdGuardDNS/internal/agd"
 
-// allTopASNs contains all specially handled ASNs.
+// DefaultTopASNs contains all specially handled ASNs.
-var allTopASNs = map[agd.ASN]struct{}{
+var DefaultTopASNs = map[agd.ASN]struct{}{
+	2:      {},
 	577:    {},
 	701:    {},
 	812:    {},
@@ -26,12 +27,14 @@ var allTopASNs = map[agd.ASN]struct{}{
 	2519:   {},
 	2527:   {},
 	2586:   {},
+	2609:   {},
 	2740:   {},
 	2856:   {},
 	2860:   {},
 	3209:   {},
 	3212:   {},
 	3215:   {},
+	3216:   {},
 	3238:   {},
 	3243:   {},
 	3249:   {},
@@ -52,7 +55,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	4134:   {},
 	4609:   {},
 	4638:   {},
-	4648:   {},
 	4657:   {},
 	4713:   {},
 	4760:   {},
@@ -64,6 +66,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	4775:   {},
 	4788:   {},
 	4804:   {},
+	4808:   {},
 	4812:   {},
 	4817:   {},
 	4818:   {},
@@ -83,7 +86,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	5610:   {},
 	5617:   {},
 	5639:   {},
-	5645:   {},
 	5769:   {},
 	6057:   {},
 	6147:   {},
@@ -91,7 +93,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	6306:   {},
 	6327:   {},
 	6400:   {},
-	6535:   {},
 	6568:   {},
 	6639:   {},
 	6661:   {},
@@ -131,23 +132,19 @@ var allTopASNs = map[agd.ASN]struct{}{
 	8359:   {},
 	8374:   {},
 	8376:   {},
-	8386:   {},
 	8400:   {},
 	8402:   {},
 	8412:   {},
 	8447:   {},
 	8452:   {},
 	8473:   {},
+	8544:   {},
 	8551:   {},
-	8560:   {},
 	8585:   {},
-	8632:   {},
-	8661:   {},
 	8680:   {},
 	8681:   {},
 	8697:   {},
 	8708:   {},
-	8728:   {},
 	8764:   {},
 	8781:   {},
 	8818:   {},
@@ -155,11 +152,10 @@ var allTopASNs = map[agd.ASN]struct{}{
 	8881:   {},
 	8926:   {},
 	8953:   {},
-	8978:   {},
+	8966:   {},
 	9009:   {},
 	9038:   {},
 	9051:   {},
-	9105:   {},
 	9121:   {},
 	9146:   {},
 	9158:   {},
@@ -199,12 +195,13 @@ var allTopASNs = map[agd.ASN]struct{}{
 	10269:  {},
 	10396:  {},
 	10620:  {},
-	10796:  {},
 	11081:  {},
 	11139:  {},
+	11269:  {},
 	11315:  {},
 	11427:  {},
 	11556:  {},
+	11562:  {},
 	11594:  {},
 	11664:  {},
 	11816:  {},
@@ -219,9 +216,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	12392:  {},
 	12400:  {},
 	12430:  {},
-	12455:  {},
 	12479:  {},
-	12576:  {},
 	12578:  {},
 	12709:  {},
 	12716:  {},
@@ -240,9 +235,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	12975:  {},
 	12997:  {},
 	13036:  {},
-	13046:  {},
 	13122:  {},
-	13124:  {},
 	13194:  {},
 	13280:  {},
 	13285:  {},
@@ -255,6 +248,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	14117:  {},
 	14434:  {},
 	14522:  {},
+	14593:  {},
 	14638:  {},
 	14709:  {},
 	14754:  {},
@@ -270,7 +264,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	15480:  {},
 	15502:  {},
 	15557:  {},
-	15659:  {},
 	15704:  {},
 	15706:  {},
 	15735:  {},
@@ -291,13 +284,13 @@ var allTopASNs = map[agd.ASN]struct{}{
 	16135:  {},
 	16232:  {},
 	16276:  {},
-	16322:  {},
 	16345:  {},
-	16437:  {},
+	16509:  {},
 	16637:  {},
 	16705:  {},
 	17072:  {},
 	17079:  {},
+	17400:  {},
 	17421:  {},
 	17458:  {},
 	17470:  {},
@@ -322,6 +315,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	18809:  {},
 	18881:  {},
 	19114:  {},
+	19246:  {},
 	19429:  {},
 	19711:  {},
 	19863:  {},
@@ -350,12 +344,13 @@ var allTopASNs = map[agd.ASN]struct{}{
 	21575:  {},
 	21744:  {},
 	21826:  {},
+	21859:  {},
 	21928:  {},
 	21996:  {},
 	22047:  {},
 	22069:  {},
 	22085:  {},
-	22351:  {},
+	22363:  {},
 	22724:  {},
 	22773:  {},
 	22927:  {},
@@ -369,7 +364,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	23693:  {},
 	23752:  {},
 	23889:  {},
-	23917:  {},
 	23955:  {},
 	23969:  {},
 	24158:  {},
@@ -384,7 +378,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	24722:  {},
 	24757:  {},
 	24835:  {},
-	24852:  {},
 	24921:  {},
 	24940:  {},
 	25019:  {},
@@ -418,7 +411,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	27781:  {},
 	27800:  {},
 	27831:  {},
-	27839:  {},
 	27882:  {},
 	27884:  {},
 	27895:  {},
@@ -430,12 +422,14 @@ var allTopASNs = map[agd.ASN]struct{}{
 	28005:  {},
 	28006:  {},
 	28036:  {},
+	28094:  {},
 	28104:  {},
 	28118:  {},
 	28126:  {},
 	28403:  {},
 	28548:  {},
 	28573:  {},
+	28683:  {},
 	28698:  {},
 	28787:  {},
 	28884:  {},
@@ -448,7 +442,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	29244:  {},
 	29247:  {},
 	29256:  {},
-	29310:  {},
 	29314:  {},
 	29355:  {},
 	29357:  {},
@@ -458,37 +451,32 @@ var allTopASNs = map[agd.ASN]struct{}{
 	29544:  {},
 	29555:  {},
 	29571:  {},
-	29580:  {},
 	29614:  {},
 	29695:  {},
 	29975:  {},
 	30689:  {},
 	30722:  {},
+	30844:  {},
 	30873:  {},
 	30969:  {},
 	30985:  {},
 	30986:  {},
-	30987:  {},
 	30990:  {},
 	30992:  {},
 	30999:  {},
 	31012:  {},
 	31037:  {},
 	31042:  {},
-	31122:  {},
 	31133:  {},
 	31163:  {},
-	31204:  {},
 	31205:  {},
 	31213:  {},
 	31224:  {},
 	31252:  {},
 	31452:  {},
-	31549:  {},
 	31615:  {},
 	31721:  {},
 	32020:  {},
-	33363:  {},
 	33392:  {},
 	33567:  {},
 	33576:  {},
@@ -513,15 +501,14 @@ var allTopASNs = map[agd.ASN]struct{}{
 	35432:  {},
 	35444:  {},
 	35805:  {},
-	35807:  {},
 	35819:  {},
 	35900:  {},
 	36290:  {},
 	36549:  {},
+	36866:  {},
 	36873:  {},
 	36884:  {},
 	36890:  {},
-	36902:  {},
 	36903:  {},
 	36907:  {},
 	36908:  {},
@@ -538,7 +525,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	36974:  {},
 	36988:  {},
 	36992:  {},
-	36994:  {},
 	36996:  {},
 	36998:  {},
 	36999:  {},
@@ -572,7 +558,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	37284:  {},
 	37287:  {},
 	37294:  {},
-	37303:  {},
 	37309:  {},
 	37323:  {},
 	37336:  {},
@@ -582,16 +567,15 @@ var allTopASNs = map[agd.ASN]struct{}{
 	37371:  {},
 	37376:  {},
 	37385:  {},
+	37406:  {},
 	37410:  {},
 	37424:  {},
 	37440:  {},
 	37447:  {},
-	37451:  {},
 	37453:  {},
 	37457:  {},
 	37460:  {},
 	37461:  {},
-	37463:  {},
 	37473:  {},
 	37492:  {},
 	37508:  {},
@@ -600,6 +584,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	37526:  {},
 	37529:  {},
 	37531:  {},
+	37541:  {},
 	37550:  {},
 	37552:  {},
 	37559:  {},
@@ -608,17 +593,18 @@ var allTopASNs = map[agd.ASN]struct{}{
 	37577:  {},
 	37594:  {},
 	37611:  {},
-	37612:  {},
+	37614:  {},
 	37616:  {},
 	37645:  {},
-	37649:  {},
 	37671:  {},
+	37678:  {},
 	37693:  {},
 	37705:  {},
 	38008:  {},
 	38009:  {},
 	38077:  {},
 	38195:  {},
+	38198:  {},
 	38201:  {},
 	38235:  {},
 	38442:  {},
@@ -626,17 +612,15 @@ var allTopASNs = map[agd.ASN]struct{}{
 	38565:  {},
 	38623:  {},
 	38742:  {},
-	38800:  {},
 	38819:  {},
-	38875:  {},
 	38901:  {},
 	38999:  {},
 	39010:  {},
 	39232:  {},
+	39402:  {},
 	39603:  {},
 	39611:  {},
 	39642:  {},
-	39737:  {},
 	39891:  {},
 	40945:  {},
 	41164:  {},
@@ -644,6 +628,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	41329:  {},
 	41330:  {},
 	41557:  {},
+	41697:  {},
 	41738:  {},
 	41750:  {},
 	41897:  {},
@@ -653,21 +638,18 @@ var allTopASNs = map[agd.ASN]struct{}{
 	42298:  {},
 	42313:  {},
 	42437:  {},
+	42532:  {},
 	42560:  {},
 	42610:  {},
 	42772:  {},
 	42779:  {},
-	42837:  {},
-	42841:  {},
 	42863:  {},
-	42960:  {},
 	42961:  {},
 	43019:  {},
 	43197:  {},
+	43242:  {},
 	43447:  {},
-	43513:  {},
 	43557:  {},
-	43571:  {},
 	43612:  {},
 	43733:  {},
 	43766:  {},
@@ -679,12 +661,14 @@ var allTopASNs = map[agd.ASN]struct{}{
 	44143:  {},
 	44244:  {},
 	44395:  {},
+	44477:  {},
 	44489:  {},
 	44558:  {},
 	44575:  {},
+	44735:  {},
 	44869:  {},
+	44925:  {},
 	45143:  {},
-	45168:  {},
 	45177:  {},
 	45178:  {},
 	45245:  {},
@@ -693,6 +677,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	45356:  {},
 	45498:  {},
 	45609:  {},
+	45629:  {},
 	45650:  {},
 	45669:  {},
 	45727:  {},
@@ -708,20 +693,21 @@ var allTopASNs = map[agd.ASN]struct{}{
 	47331:  {},
 	47377:  {},
 	47394:  {},
-	47524:  {},
 	47589:  {},
 	47883:  {},
-	47956:  {},
 	47975:  {},
 	48092:  {},
 	48190:  {},
 	48206:  {},
 	48252:  {},
 	48503:  {},
+	48675:  {},
 	48728:  {},
 	48832:  {},
+	48847:  {},
 	48887:  {},
 	49273:  {},
+	49628:  {},
 	49800:  {},
 	49902:  {},
 	49981:  {},
@@ -729,20 +715,19 @@ var allTopASNs = map[agd.ASN]struct{}{
 	50223:  {},
 	50251:  {},
 	50266:  {},
+	50360:  {},
 	50616:  {},
-	50810:  {},
+	50710:  {},
 	50973:  {},
 	51207:  {},
 	51375:  {},
 	51407:  {},
 	51495:  {},
 	51765:  {},
-	51852:  {},
 	51896:  {},
 	52228:  {},
 	52233:  {},
 	52253:  {},
-	52257:  {},
 	52260:  {},
 	52262:  {},
 	52263:  {},
@@ -758,31 +743,33 @@ var allTopASNs = map[agd.ASN]struct{}{
 	55850:  {},
 	55943:  {},
 	55944:  {},
-	56017:  {},
 	56055:  {},
 	56089:  {},
 	56167:  {},
 	56300:  {},
-	56369:  {},
-	56653:  {},
 	56665:  {},
 	56696:  {},
 	56902:  {},
+	57218:  {},
 	57269:  {},
 	57293:  {},
 	57388:  {},
 	57513:  {},
-	57704:  {},
 	58224:  {},
 	58460:  {},
 	58731:  {},
 	58952:  {},
 	59257:  {},
+	59974:  {},
 	59989:  {},
 	60068:  {},
 	60258:  {},
+	60471:  {},
+	60781:  {},
 	61143:  {},
+	61272:  {},
 	61461:  {},
+	63473:  {},
 	63949:  {},
 	64466:  {},
 	131178: {},
@@ -796,7 +783,6 @@ var allTopASNs = map[agd.ASN]struct{}{
 	132167: {},
 	132199: {},
 	132471: {},
-	132486: {},
 	132618: {},
 	133385: {},
 	133481: {},
@@ -804,9 +790,10 @@ var allTopASNs = map[agd.ASN]struct{}{
 	133606: {},
 	133612: {},
 	134783: {},
+	134840: {},
 	135409: {},
 	136255: {},
-	136950: {},
+	136557: {},
 	137412: {},
 	137824: {},
 	138179: {},
@@ -815,27 +802,31 @@ var allTopASNs = map[agd.ASN]struct{}{
 	139898: {},
 	139922: {},
 	140504: {},
-	196838: {},
 	197207: {},
 	197830: {},
 	198279: {},
+	198288: {},
 	198605: {},
 	199140: {},
 	199276: {},
 	199731: {},
 	200134: {},
+	201019: {},
 	201167: {},
 	201767: {},
+	201884: {},
 	201986: {},
 	202087: {},
 	202254: {},
-	202422: {},
+	203020: {},
-	202448: {},
 	203214: {},
 	203953: {},
 	203995: {},
+	204106: {},
 	204170: {},
-	204279: {},
+	204317: {},
+	204342: {},
+	204649: {},
 	205110: {},
 	205368: {},
 	205714: {},
@@ -843,20 +834,18 @@ var allTopASNs = map[agd.ASN]struct{}{
 	206067: {},
 	206206: {},
 	206262: {},
-	207369: {},
-	207569: {},
 	207651: {},
 	207810: {},
-	209424: {},
+	207876: {},
-	210003: {},
+	209442: {},
+	209854: {},
 	210315: {},
 	210542: {},
 	211144: {},
 	212238: {},
-	212370: {},
 	213155: {},
-	213371: {},
 	262145: {},
+	262181: {},
 	262186: {},
 	262197: {},
 	262202: {},
@@ -864,6 +853,7 @@ var allTopASNs = map[agd.ASN]struct{}{
 	262239: {},
 	262589: {},
 	263238: {},
+	263703: {},
 	263725: {},
 	263783: {},
 	263824: {},
@@ -872,19 +862,19 @@ var allTopASNs = map[agd.ASN]struct{}{
 	264663: {},
 	264668: {},
 	264731: {},
-	266673: {},
-	269729: {},
 	271773: {},
 	327697: {},
+	327707: {},
 	327712: {},
-	327725: {},
 	327738: {},
 	327756: {},
 	327765: {},
 	327769: {},
+	327776: {},
 	327799: {},
 	327802: {},
 	327885: {},
+	327903: {},
 	327931: {},
 	327934: {},
 	328061: {},
@@ -903,9 +893,11 @@ var allTopASNs = map[agd.ASN]struct{}{
 	328469: {},
 	328488: {},
 	328539: {},
+	328605: {},
+	328708: {},
+	328727: {},
 	328755: {},
 	328943: {},
-	393275: {},
 	394311: {},
 	395561: {},
 	396304: {},
@@ -914,13 +906,13 @@ var allTopASNs = map[agd.ASN]struct{}{
 	399724: {},
 }
 
-// countryTopASNs is a mapping of a country to their top ASNs.
+// DefaultCountryTopASNs is a mapping of a country to their top ASNs.
-var countryTopASNs = map[agd.Country]agd.ASN{
+var DefaultCountryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryAD: 6752,
 	agd.CountryAE: 5384,
 	agd.CountryAF: 132471,
 	agd.CountryAG: 11594,
-	agd.CountryAI: 2740,
+	agd.CountryAI: 11139,
 	agd.CountryAL: 21183,
 	agd.CountryAM: 12297,
 	agd.CountryAO: 37119,
@@ -930,7 +922,7 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryAU: 1221,
 	agd.CountryAW: 11816,
 	agd.CountryAX: 3238,
-	agd.CountryAZ: 28787,
+	agd.CountryAZ: 34170,
 	agd.CountryBA: 9146,
 	agd.CountryBB: 14813,
 	agd.CountryBD: 24389,
@@ -940,17 +932,17 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryBH: 5416,
 	agd.CountryBI: 327799,
 	agd.CountryBJ: 37424,
-	agd.CountryBL: 3215,
+	agd.CountryBL: 14593,
-	agd.CountryBM: 32020,
+	agd.CountryBM: 3855,
 	agd.CountryBN: 10094,
 	agd.CountryBO: 6568,
 	agd.CountryBQ: 27745,
-	agd.CountryBR: 28573,
+	agd.CountryBR: 26599,
 	agd.CountryBS: 15146,
 	agd.CountryBT: 18024,
 	agd.CountryBW: 14988,
-	agd.CountryBY: 6697,
+	agd.CountryBY: 25106,
-	agd.CountryBZ: 212370,
+	agd.CountryBZ: 10269,
 	agd.CountryCA: 812,
 	agd.CountryCD: 37020,
 	agd.CountryCF: 37460,
@@ -958,15 +950,15 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryCH: 3303,
 	agd.CountryCI: 29571,
 	agd.CountryCK: 10131,
-	agd.CountryCL: 7418,
+	agd.CountryCL: 27651,
 	agd.CountryCM: 30992,
 	agd.CountryCN: 4134,
-	agd.CountryCO: 10620,
+	agd.CountryCO: 27831,
 	agd.CountryCR: 11830,
 	agd.CountryCU: 27725,
 	agd.CountryCV: 37517,
 	agd.CountryCW: 52233,
-	agd.CountryCY: 202448,
+	agd.CountryCY: 6866,
 	agd.CountryCZ: 5610,
 	agd.CountryDE: 3320,
 	agd.CountryDJ: 30990,
@@ -975,20 +967,20 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryDO: 6400,
 	agd.CountryDZ: 36947,
 	agd.CountryEC: 27947,
-	agd.CountryEE: 3249,
+	agd.CountryEE: 44477,
 	agd.CountryEG: 8452,
-	agd.CountryEH: 6713,
+	agd.CountryEH: 36947,
 	agd.CountryER: 24757,
-	agd.CountryES: 12479,
+	agd.CountryES: 3352,
 	agd.CountryET: 24757,
 	agd.CountryFI: 51765,
 	agd.CountryFJ: 38442,
-	agd.CountryFK: 198605,
+	agd.CountryFK: 204649,
 	agd.CountryFM: 139759,
 	agd.CountryFO: 15389,
 	agd.CountryFR: 3215,
 	agd.CountryGA: 36924,
-	agd.CountryGB: 60068,
+	agd.CountryGB: 2856,
 	agd.CountryGD: 46650,
 	agd.CountryGE: 16010,
 	agd.CountryGF: 3215,
@@ -996,29 +988,29 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryGH: 30986,
 	agd.CountryGI: 8301,
 	agd.CountryGL: 8818,
-	agd.CountryGM: 25250,
+	agd.CountryGM: 37309,
 	agd.CountryGN: 37461,
 	agd.CountryGP: 3215,
-	agd.CountryGQ: 37173,
+	agd.CountryGQ: 37529,
 	agd.CountryGR: 6799,
 	agd.CountryGT: 14754,
-	agd.CountryGU: 3605,
+	agd.CountryGU: 9246,
 	agd.CountryGW: 37559,
 	agd.CountryGY: 19863,
 	agd.CountryHK: 4760,
-	agd.CountryHN: 52262,
+	agd.CountryHN: 14754,
 	agd.CountryHR: 5391,
 	agd.CountryHT: 27653,
 	agd.CountryHU: 5483,
 	agd.CountryID: 7713,
-	agd.CountryIE: 6830,
+	agd.CountryIE: 16509,
 	agd.CountryIL: 1680,
 	agd.CountryIM: 13122,
 	agd.CountryIN: 55836,
 	agd.CountryIO: 17458,
 	agd.CountryIQ: 203214,
-	agd.CountryIR: 58224,
+	agd.CountryIR: 44244,
-	agd.CountryIS: 43571,
+	agd.CountryIS: 44735,
 	agd.CountryIT: 1267,
 	agd.CountryJE: 8680,
 	agd.CountryJM: 30689,
@@ -1027,7 +1019,7 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryKE: 33771,
 	agd.CountryKG: 50223,
 	agd.CountryKH: 38623,
-	agd.CountryKI: 134783,
+	agd.CountryKI: 135409,
 	agd.CountryKM: 36939,
 	agd.CountryKN: 11139,
 	agd.CountryKR: 4766,
@@ -1039,10 +1031,10 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryLC: 15344,
 	agd.CountryLI: 20634,
 	agd.CountryLK: 18001,
-	agd.CountryLR: 37410,
+	agd.CountryLR: 37094,
 	agd.CountryLS: 33567,
 	agd.CountryLT: 8764,
-	agd.CountryLU: 6661,
+	agd.CountryLU: 53667,
 	agd.CountryLV: 24921,
 	agd.CountryLY: 21003,
 	agd.CountryMA: 36903,
@@ -1071,22 +1063,22 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryNA: 36996,
 	agd.CountryNC: 18200,
 	agd.CountryNE: 37531,
-	agd.CountryNF: 45168,
 	agd.CountryNG: 29465,
 	agd.CountryNI: 14754,
 	agd.CountryNL: 1136,
 	agd.CountryNO: 2119,
 	agd.CountryNP: 17501,
-	agd.CountryNR: 140504,
+	agd.CountryNR: 198605,
+	agd.CountryNU: 198605,
 	agd.CountryNZ: 9790,
 	agd.CountryOM: 28885,
-	agd.CountryPA: 18809,
+	agd.CountryPA: 11556,
 	agd.CountryPE: 12252,
 	agd.CountryPF: 9471,
 	agd.CountryPG: 139898,
 	agd.CountryPH: 9299,
 	agd.CountryPK: 45669,
-	agd.CountryPL: 5617,
+	agd.CountryPL: 43447,
 	agd.CountryPM: 3695,
 	agd.CountryPR: 14638,
 	agd.CountryPS: 12975,
@@ -1094,7 +1086,7 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryPW: 17893,
 	agd.CountryPY: 23201,
 	agd.CountryQA: 42298,
-	agd.CountryRE: 3215,
+	agd.CountryRE: 37002,
 	agd.CountryRO: 8708,
 	agd.CountryRS: 8400,
 	agd.CountryRU: 8359,
@@ -1105,7 +1097,9 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountrySD: 15706,
 	agd.CountrySE: 60068,
 	agd.CountrySG: 4773,
-	agd.CountrySI: 3212,
+	agd.CountrySH: 33763,
+	agd.CountrySI: 5603,
+	agd.CountrySJ: 29695,
 	agd.CountrySK: 6855,
 	agd.CountrySL: 37164,
 	agd.CountrySM: 15433,
@@ -1123,33 +1117,31 @@ var countryTopASNs = map[agd.Country]agd.ASN{
 	agd.CountryTG: 36924,
 	agd.CountryTH: 131445,
 	agd.CountryTJ: 43197,
-	agd.CountryTK: 4648,
 	agd.CountryTL: 58731,
 	agd.CountryTM: 51495,
 	agd.CountryTN: 37705,
 	agd.CountryTO: 38201,
 	agd.CountryTR: 47331,
 	agd.CountryTT: 27800,
-	agd.CountryTV: 23917,
+	agd.CountryTV: 198605,
 	agd.CountryTW: 3462,
 	agd.CountryTZ: 36908,
 	agd.CountryUA: 15895,
 	agd.CountryUG: 37075,
-	agd.CountryUS: 7922,
+	agd.CountryUS: 21928,
 	agd.CountryUY: 6057,
 	agd.CountryUZ: 8193,
-	agd.CountryVA: 8978,
 	agd.CountryVC: 46408,
 	agd.CountryVE: 8048,
-	agd.CountryVG: 14813,
+	agd.CountryVG: 396357,
 	agd.CountryVI: 14434,
 	agd.CountryVN: 7552,
 	agd.CountryVU: 9249,
 	agd.CountryWF: 45879,
-	agd.CountryWS: 38800,
+	agd.CountryWS: 17993,
 	agd.CountryXK: 21246,
 	agd.CountryYE: 30873,
-	agd.CountryYT: 3215,
+	agd.CountryYT: 49902,
 	agd.CountryZA: 37457,
 	agd.CountryZM: 37287,
 	agd.CountryZW: 37204,