Merge pull request #1 in DNS/adguard-internal-dns from move_plugin to master

* commit 'd54fd77204cc1e9444085ca13b87953bf096fff8':
  Fix linter warnings.
  Delete unused code from coredns plugin
  Don't use coredns's resolver because it breaks when serving DNS-over-TLS
  Move coredns plugin to internal dns repo

Makefile

@@ -8,20 +8,22 @@ SHELL := /bin/bash
 .PHONY: default
 default: repo
 
-GOPATH=$(shell pwd)/go_$(VERSION)
+mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
+mkfile_dir := $(patsubst %/,%,$(dir $(mkfile_path)))
+GOPATH := $(mkfile_dir)/go_$(VERSION)
 
 clean:
 	rm -fv *.deb
 
 build: check-vars clean
-	mkdir -p $(GOPATH)
-	GOPATH=$(GOPATH) go get -v -d github.com/AdguardTeam/AdguardDNS
+	mkdir -p $(GOPATH)/src/bit.adguard.com/dns
+	if [ ! -h $(GOPATH)/src/bit.adguard.com/dns/adguard-internal-dns ]; then rm -rf $(GOPATH)/src/bit.adguard.com/dns/adguard-internal-dns && ln -fs $(mkfile_dir) $(GOPATH)/src/bit.adguard.com/dns/adguard-internal-dns; fi
 	GOPATH=$(GOPATH) go get -v -d github.com/coredns/coredns
 	cp plugin.cfg $(GOPATH)/src/github.com/coredns/coredns
 	cd $(GOPATH)/src/github.com/coredns/coredns; GOPATH=$(GOPATH) go generate
 	cd $(GOPATH)/src/github.com/coredns/coredns; GOPATH=$(GOPATH) go get -v -d -t .
 	cd $(GOPATH)/src/github.com/coredns/coredns; GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) make
-	cd $(GOPATH)/src/github.com/coredns/coredns; GOPATH=$(GOPATH) go build -x -v -ldflags="-X github.com/coredns/coredns/coremain.GitCommit=$(VERSION)" -o $(GOPATH)/bin/coredns
+	cd $(GOPATH)/src/github.com/coredns/coredns; GOPATH=$(GOPATH) go build -x -v -ldflags="-X github.com/coredns/coredns/coremain.GitCommit=$(VERSION)" -asmflags="-trimpath=$(GOPATH)" -gcflags="-trimpath=$(GOPATH)" -o $(GOPATH)/bin/coredns
 
 package: build
 	fpm --prefix /usr/local/bin \

coredns_plugin/coredns_plugin.go

@@ -0,0 +1,539 @@
+package dnsfilter
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
+	"github.com/coredns/coredns/core/dnsserver"
+	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/metrics"
+	"github.com/coredns/coredns/plugin/pkg/dnstest"
+	"github.com/coredns/coredns/request"
+	"github.com/mholt/caddy"
+	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+)
+
+var defaultSOA = &dns.SOA{
+	// values copied from verisign's nonexistent .com domain
+	// their exact values are not important in our use case because they are used for domain transfers between primary/secondary DNS servers
+	Refresh: 1800,
+	Retry:   900,
+	Expire:  604800,
+	Minttl:  86400,
+}
+
+func init() {
+	caddy.RegisterPlugin("dnsfilter", caddy.Plugin{
+		ServerType: "dns",
+		Action:     setup,
+	})
+}
+
+type plugFilter struct {
+	ID   int64
+	Path string
+}
+
+type plugSettings struct {
+	SafeBrowsingBlockHost string
+	ParentalBlockHost     string
+	QueryLogEnabled       bool
+	BlockedTTL            uint32 // in seconds, default 3600
+	Filters               []plugFilter
+}
+
+type plug struct {
+	d        *dnsfilter.Dnsfilter
+	Next     plugin.Handler
+	settings plugSettings
+
+	sync.RWMutex
+}
+
+var defaultPluginSettings = plugSettings{
+	SafeBrowsingBlockHost: "safebrowsing.block.dns.adguard.com",
+	ParentalBlockHost:     "family.block.dns.adguard.com",
+	BlockedTTL:            3600, // in seconds
+	Filters:               make([]plugFilter, 0),
+}
+
+//
+// coredns handling functions
+//
+func setupPlugin(c *caddy.Controller) (*plug, error) {
+	// create new Plugin and copy default values
+	p := &plug{
+		settings: defaultPluginSettings,
+		d:        dnsfilter.New(),
+	}
+
+	log.Println("Initializing the CoreDNS plugin")
+
+	for c.Next() {
+		for c.NextBlock() {
+			blockValue := c.Val()
+			switch blockValue {
+			case "safebrowsing":
+				log.Println("Browsing security service is enabled")
+				p.d.EnableSafeBrowsing()
+				if c.NextArg() {
+					if len(c.Val()) == 0 {
+						return nil, c.ArgErr()
+					}
+					p.d.SetSafeBrowsingServer(c.Val())
+				}
+			case "safesearch":
+				log.Println("Safe search is enabled")
+				p.d.EnableSafeSearch()
+			case "parental":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				sensitivity, err := strconv.Atoi(c.Val())
+				if err != nil {
+					return nil, c.ArgErr()
+				}
+
+				log.Println("Parental control is enabled")
+				err = p.d.EnableParental(sensitivity)
+				if err != nil {
+					return nil, c.ArgErr()
+				}
+				if c.NextArg() {
+					if len(c.Val()) == 0 {
+						return nil, c.ArgErr()
+					}
+					p.settings.ParentalBlockHost = c.Val()
+				}
+			case "blocked_ttl":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				blockedTTL, err := strconv.ParseUint(c.Val(), 10, 32)
+				if err != nil {
+					return nil, c.ArgErr()
+				}
+				log.Printf("Blocked request TTL is %d", blockedTTL)
+				p.settings.BlockedTTL = uint32(blockedTTL)
+			case "querylog":
+				log.Println("Query log is enabled")
+				p.settings.QueryLogEnabled = true
+			case "filter":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+
+				filterID, err := strconv.ParseInt(c.Val(), 10, 64)
+				if err != nil {
+					return nil, c.ArgErr()
+				}
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				filterPath := c.Val()
+
+				// Initialize filter and add it to the list
+				p.settings.Filters = append(p.settings.Filters, plugFilter{
+					ID:   filterID,
+					Path: filterPath,
+				})
+			}
+		}
+	}
+
+	for _, filter := range p.settings.Filters {
+		log.Printf("Loading rules from %s", filter.Path)
+
+		file, err := os.Open(filter.Path)
+		if err != nil {
+			return nil, err
+		}
+		defer file.Close()
+
+		count := 0
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			text := scanner.Text()
+
+			err = p.d.AddRule(text, filter.ID)
+			if err == dnsfilter.ErrAlreadyExists || err == dnsfilter.ErrInvalidSyntax {
+				continue
+			}
+			if err != nil {
+				log.Printf("Cannot add rule %s: %s", text, err)
+				// Just ignore invalid rules
+				continue
+			}
+			count++
+		}
+		log.Printf("Added %d rules from filter ID=%d", count, filter.ID)
+
+		if err = scanner.Err(); err != nil {
+			return nil, err
+		}
+	}
+
+	return p, nil
+}
+
+func setup(c *caddy.Controller) error {
+	p, err := setupPlugin(c)
+	if err != nil {
+		return err
+	}
+	config := dnsserver.GetConfig(c)
+	config.AddPlugin(func(next plugin.Handler) plugin.Handler {
+		p.Next = next
+		return p
+	})
+
+	c.OnStartup(func() error {
+		m := dnsserver.GetConfig(c).Handler("prometheus")
+		if m == nil {
+			return nil
+		}
+		if x, ok := m.(*metrics.Metrics); ok {
+			x.MustRegister(requests)
+			x.MustRegister(filtered)
+			x.MustRegister(filteredLists)
+			x.MustRegister(filteredSafebrowsing)
+			x.MustRegister(filteredParental)
+			x.MustRegister(whitelisted)
+			x.MustRegister(safesearch)
+			x.MustRegister(errorsTotal)
+			x.MustRegister(elapsedTime)
+			x.MustRegister(p)
+		}
+		return nil
+	})
+	c.OnShutdown(p.onShutdown)
+	c.OnFinalShutdown(p.onFinalShutdown)
+
+	return nil
+}
+
+func (p *plug) onShutdown() error {
+	p.Lock()
+	p.d.Destroy()
+	p.d = nil
+	p.Unlock()
+	return nil
+}
+
+func (p *plug) onFinalShutdown() error {
+	return nil
+}
+
+type statsFunc func(ch interface{}, name string, text string, value float64, valueType prometheus.ValueType)
+
+func doDesc(ch interface{}, name string, text string, value float64, valueType prometheus.ValueType) {
+	realch, ok := ch.(chan<- *prometheus.Desc)
+	if !ok {
+		log.Printf("Couldn't convert ch to chan<- *prometheus.Desc\n")
+		return
+	}
+	realch <- prometheus.NewDesc(name, text, nil, nil)
+}
+
+func doMetric(ch interface{}, name string, text string, value float64, valueType prometheus.ValueType) {
+	realch, ok := ch.(chan<- prometheus.Metric)
+	if !ok {
+		log.Printf("Couldn't convert ch to chan<- prometheus.Metric\n")
+		return
+	}
+	desc := prometheus.NewDesc(name, text, nil, nil)
+	realch <- prometheus.MustNewConstMetric(desc, valueType, value)
+}
+
+func gen(ch interface{}, doFunc statsFunc, name string, text string, value float64, valueType prometheus.ValueType) {
+	doFunc(ch, name, text, value, valueType)
+}
+
+func doStatsLookup(ch interface{}, doFunc statsFunc, name string, lookupstats *dnsfilter.LookupStats) {
+	gen(ch, doFunc, fmt.Sprintf("coredns_dnsfilter_%s_requests", name), fmt.Sprintf("Number of %s HTTP requests that were sent", name), float64(lookupstats.Requests), prometheus.CounterValue)
+	gen(ch, doFunc, fmt.Sprintf("coredns_dnsfilter_%s_cachehits", name), fmt.Sprintf("Number of %s lookups that didn't need HTTP requests", name), float64(lookupstats.CacheHits), prometheus.CounterValue)
+	gen(ch, doFunc, fmt.Sprintf("coredns_dnsfilter_%s_pending", name), fmt.Sprintf("Number of currently pending %s HTTP requests", name), float64(lookupstats.Pending), prometheus.GaugeValue)
+	gen(ch, doFunc, fmt.Sprintf("coredns_dnsfilter_%s_pending_max", name), fmt.Sprintf("Maximum number of pending %s HTTP requests", name), float64(lookupstats.PendingMax), prometheus.GaugeValue)
+}
+
+func (p *plug) doStats(ch interface{}, doFunc statsFunc) {
+	p.RLock()
+	stats := p.d.GetStats()
+	doStatsLookup(ch, doFunc, "safebrowsing", &stats.Safebrowsing)
+	doStatsLookup(ch, doFunc, "parental", &stats.Parental)
+	p.RUnlock()
+}
+
+// Describe is called by prometheus handler to know stat types
+func (p *plug) Describe(ch chan<- *prometheus.Desc) {
+	p.doStats(ch, doDesc)
+}
+
+// Collect is called by prometheus handler to collect stats
+func (p *plug) Collect(ch chan<- prometheus.Metric) {
+	p.doStats(ch, doMetric)
+}
+
+// lookup host, but return answer as if it was a result of origname lookup
+func lookupReplaced(host string, origname string) ([]dns.RR, error) {
+	var records []dns.RR
+	var res *net.Resolver // nil resolver is default resolver
+	addrs, err := res.LookupIPAddr(context.TODO(), host)
+	if err != nil {
+		return nil, err
+	}
+	for _, addr := range addrs {
+		ip := addr.IP
+		var rr dns.RR
+		var err error
+		if ip.To4() != nil {
+			// ipv4 -> A
+			rr, err = dns.NewRR(fmt.Sprintf("%s A %s", origname, ip.String()))
+			if err != nil {
+				return nil, err // fail entire request
+				// TODO: return only successful answers?
+			}
+		} else {
+			// ipv6 -> AAAA
+			rr, err = dns.NewRR(fmt.Sprintf("%s AAAA %s", origname, ip.String()))
+			if err != nil {
+				return nil, err // fail entire request
+				// TODO: return only successful answers?
+			}
+		}
+		records = append(records, rr)
+	}
+	return records, nil
+}
+
+func (p *plug) replaceHostWithValAndReply(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, host string, val string, question dns.Question) (int, error) {
+	// check if it's a domain name or IP address
+	addr := net.ParseIP(val)
+	var records []dns.RR
+	// log.Println("Will give", val, "instead of", host) // debug logging
+	if addr != nil {
+		// this is an IP address, return it
+		result, err := dns.NewRR(fmt.Sprintf("%s %d A %s", host, p.settings.BlockedTTL, val))
+		if err != nil {
+			log.Printf("Got error %s\n", err)
+			return dns.RcodeServerFailure, fmt.Errorf("plugin/dnsfilter: %s", err)
+		}
+		records = append(records, result)
+	} else {
+		// this is a domain name, need to look it up
+		var err error
+		records, err = lookupReplaced(dns.Fqdn(val), question.Name)
+		if err != nil {
+			log.Printf("Got error %s\n", err)
+			return dns.RcodeServerFailure, fmt.Errorf("plugin/dnsfilter: %s", err)
+		}
+	}
+	m := new(dns.Msg)
+	m.SetReply(r)
+	m.Authoritative, m.RecursionAvailable, m.Compress = true, true, true
+	m.Answer = append(m.Answer, records...)
+	state := request.Request{W: w, Req: r, Context: ctx}
+	state.SizeAndDo(m)
+	err := state.W.WriteMsg(m)
+	if err != nil {
+		log.Printf("Got error %s\n", err)
+		return dns.RcodeServerFailure, fmt.Errorf("plugin/dnsfilter: %s", err)
+	}
+	return dns.RcodeSuccess, nil
+}
+
+// generate SOA record that makes DNS clients cache NXdomain results
+// the only value that is important is TTL in header, other values like refresh, retry, expire and minttl are irrelevant
+func (p *plug) genSOA(r *dns.Msg) []dns.RR {
+	zone := r.Question[0].Name
+	header := dns.RR_Header{Name: zone, Rrtype: dns.TypeSOA, Ttl: p.settings.BlockedTTL, Class: dns.ClassINET}
+
+	Mbox := "hostmaster."
+	if zone[0] != '.' {
+		Mbox += zone
+	}
+	Ns := "fake-for-negative-caching.adguard.com."
+
+	soa := *defaultSOA
+	soa.Hdr = header
+	soa.Mbox = Mbox
+	soa.Ns = Ns
+	soa.Serial = 100500 // faster than uint32(time.Now().Unix())
+	return []dns.RR{&soa}
+}
+
+func (p *plug) writeNXdomain(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	state := request.Request{W: w, Req: r, Context: ctx}
+	m := new(dns.Msg)
+	m.SetRcode(state.Req, dns.RcodeNameError)
+	m.Authoritative, m.RecursionAvailable, m.Compress = true, true, true
+	m.Ns = p.genSOA(r)
+
+	state.SizeAndDo(m)
+	err := state.W.WriteMsg(m)
+	if err != nil {
+		log.Printf("Got error %s\n", err)
+		return dns.RcodeServerFailure, err
+	}
+	return dns.RcodeNameError, nil
+}
+
+func (p *plug) serveDNSInternal(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, dnsfilter.Result, error) {
+	if len(r.Question) != 1 {
+		// google DNS, bind and others do the same
+		return dns.RcodeFormatError, dnsfilter.Result{}, fmt.Errorf("got a DNS request with more than one Question")
+	}
+	for _, question := range r.Question {
+		host := strings.ToLower(strings.TrimSuffix(question.Name, "."))
+		// is it a safesearch domain?
+		p.RLock()
+		if val, ok := p.d.SafeSearchDomain(host); ok {
+			rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, val, question)
+			if err != nil {
+				p.RUnlock()
+				return rcode, dnsfilter.Result{}, err
+			}
+			p.RUnlock()
+			return rcode, dnsfilter.Result{Reason: dnsfilter.FilteredSafeSearch}, err
+		}
+		p.RUnlock()
+
+		// needs to be filtered instead
+		p.RLock()
+		result, err := p.d.CheckHost(host)
+		if err != nil {
+			log.Printf("plugin/dnsfilter: %s\n", err)
+			p.RUnlock()
+			return dns.RcodeServerFailure, dnsfilter.Result{}, fmt.Errorf("plugin/dnsfilter: %s", err)
+		}
+		p.RUnlock()
+
+		if result.IsFiltered {
+			switch result.Reason {
+			case dnsfilter.FilteredSafeBrowsing:
+				// return cname safebrowsing.block.dns.adguard.com
+				val := p.settings.SafeBrowsingBlockHost
+				rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, val, question)
+				if err != nil {
+					return rcode, dnsfilter.Result{}, err
+				}
+				return rcode, result, err
+			case dnsfilter.FilteredParental:
+				// return cname family.block.dns.adguard.com
+				val := p.settings.ParentalBlockHost
+				rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, val, question)
+				if err != nil {
+					return rcode, dnsfilter.Result{}, err
+				}
+				return rcode, result, err
+			case dnsfilter.FilteredBlackList:
+				if result.Ip == nil {
+					// return NXDomain
+					rcode, err := p.writeNXdomain(ctx, w, r)
+					if err != nil {
+						return rcode, dnsfilter.Result{}, err
+					}
+					return rcode, result, err
+				}
+
+				// This is a hosts-syntax rule
+				rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, result.Ip.String(), question)
+				if err != nil {
+					return rcode, dnsfilter.Result{}, err
+				}
+				return rcode, result, err
+			case dnsfilter.FilteredInvalid:
+				// return NXdomain
+				rcode, err := p.writeNXdomain(ctx, w, r)
+				if err != nil {
+					return rcode, dnsfilter.Result{}, err
+				}
+				return rcode, result, err
+			default:
+				log.Printf("SHOULD NOT HAPPEN -- got unknown reason for filtering host \"%s\": %v, %+v", host, result.Reason, result)
+			}
+		} else {
+			switch result.Reason {
+			case dnsfilter.NotFilteredWhiteList:
+				rcode, err := plugin.NextOrFailure(p.Name(), p.Next, ctx, w, r)
+				return rcode, result, err
+			case dnsfilter.NotFilteredNotFound:
+				// do nothing, pass through to lower code
+			default:
+				log.Printf("SHOULD NOT HAPPEN -- got unknown reason for not filtering host \"%s\": %v, %+v", host, result.Reason, result)
+			}
+		}
+	}
+	rcode, err := plugin.NextOrFailure(p.Name(), p.Next, ctx, w, r)
+	return rcode, dnsfilter.Result{}, err
+}
+
+// ServeDNS handles the DNS request and refuses if it's in filterlists
+func (p *plug) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	start := time.Now()
+	requests.Inc()
+	state := request.Request{W: w, Req: r}
+
+	// capture the written answer
+	rrw := dnstest.NewRecorder(w)
+	rcode, result, err := p.serveDNSInternal(ctx, rrw, r)
+	if rcode > 0 {
+		// actually send the answer if we have one
+		answer := new(dns.Msg)
+		answer.SetRcode(r, rcode)
+		state.SizeAndDo(answer)
+		err = w.WriteMsg(answer)
+		if err != nil {
+			return dns.RcodeServerFailure, err
+		}
+	}
+
+	// increment counters
+	switch {
+	case err != nil:
+		errorsTotal.Inc()
+	case result.Reason == dnsfilter.FilteredBlackList:
+		filtered.Inc()
+		filteredLists.Inc()
+	case result.Reason == dnsfilter.FilteredSafeBrowsing:
+		filtered.Inc()
+		filteredSafebrowsing.Inc()
+	case result.Reason == dnsfilter.FilteredParental:
+		filtered.Inc()
+		filteredParental.Inc()
+	case result.Reason == dnsfilter.FilteredInvalid:
+		filtered.Inc()
+		filteredInvalid.Inc()
+	case result.Reason == dnsfilter.FilteredSafeSearch:
+		// the request was passsed through but not filtered, don't increment filtered
+		safesearch.Inc()
+	case result.Reason == dnsfilter.NotFilteredWhiteList:
+		whitelisted.Inc()
+	case result.Reason == dnsfilter.NotFilteredNotFound:
+		// do nothing
+	case result.Reason == dnsfilter.NotFilteredError:
+		text := "SHOULD NOT HAPPEN: got DNSFILTER_NOTFILTERED_ERROR without err != nil!"
+		log.Println(text)
+		err = errors.New(text)
+		rcode = dns.RcodeServerFailure
+	}
+
+	// log
+	elapsed := time.Since(start)
+	elapsedTime.Observe(elapsed.Seconds())
+	return rcode, err
+}
+
+// Name returns name of the plugin as seen in Corefile and plugin.cfg
+func (p *plug) Name() string { return "dnsfilter" }

coredns_plugin/coredns_plugin_test.go

@@ -0,0 +1,131 @@
+package dnsfilter
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/pkg/dnstest"
+	"github.com/coredns/coredns/plugin/test"
+	"github.com/mholt/caddy"
+	"github.com/miekg/dns"
+)
+
+func TestSetup(t *testing.T) {
+	for i, testcase := range []struct {
+		config  string
+		failing bool
+	}{
+		{`dnsfilter`, false},
+		{`dnsfilter { 
+					filter 0 /dev/nonexistent/abcdef
+				}`, true},
+		{`dnsfilter { 
+					filter 0 ../tests/dns.txt
+				}`, false},
+		{`dnsfilter { 
+					safebrowsing
+					filter 0 ../tests/dns.txt 
+				}`, false},
+		{`dnsfilter { 
+					parental
+					filter 0 ../tests/dns.txt
+				}`, true},
+	} {
+		c := caddy.NewTestController("dns", testcase.config)
+		err := setup(c)
+		if err != nil {
+			if !testcase.failing {
+				t.Fatalf("Test #%d expected no errors, but got: %v", i, err)
+			}
+			continue
+		}
+		if testcase.failing {
+			t.Fatalf("Test #%d expected to fail but it didn't", i)
+		}
+	}
+}
+
+func TestEtcHostsFilter(t *testing.T) {
+	text := []byte("127.0.0.1 doubleclick.net\n" + "127.0.0.1 example.org example.net www.example.org www.example.net")
+	tmpfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = tmpfile.Write(text); err != nil {
+		t.Fatal(err)
+	}
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.Remove(tmpfile.Name())
+
+	configText := fmt.Sprintf("dnsfilter {\nfilter 0 %s\n}", tmpfile.Name())
+	c := caddy.NewTestController("dns", configText)
+	p, err := setupPlugin(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p.Next = zeroTTLBackend()
+
+	ctx := context.TODO()
+
+	for _, testcase := range []struct {
+		host     string
+		filtered bool
+	}{
+		{"www.doubleclick.net", false},
+		{"doubleclick.net", true},
+		{"www2.example.org", false},
+		{"www2.example.net", false},
+		{"test.www.example.org", false},
+		{"test.www.example.net", false},
+		{"example.org", true},
+		{"example.net", true},
+		{"www.example.org", true},
+		{"www.example.net", true},
+	} {
+		req := new(dns.Msg)
+		req.SetQuestion(testcase.host+".", dns.TypeA)
+
+		resp := test.ResponseWriter{}
+		rrw := dnstest.NewRecorder(&resp)
+		rcode, err := p.ServeDNS(ctx, rrw, req)
+		if err != nil {
+			t.Fatalf("ServeDNS returned error: %s", err)
+		}
+		if rcode != rrw.Rcode {
+			t.Fatalf("ServeDNS return value for host %s has rcode %d that does not match captured rcode %d", testcase.host, rcode, rrw.Rcode)
+		}
+		A, ok := rrw.Msg.Answer[0].(*dns.A)
+		if !ok {
+			t.Fatalf("Host %s expected to have result A", testcase.host)
+		}
+		ip := net.IPv4(127, 0, 0, 1)
+		filtered := ip.Equal(A.A)
+		if testcase.filtered && testcase.filtered != filtered {
+			t.Fatalf("Host %s expected to be filtered, instead it is not filtered", testcase.host)
+		}
+		if !testcase.filtered && testcase.filtered != filtered {
+			t.Fatalf("Host %s expected to be not filtered, instead it is filtered", testcase.host)
+		}
+	}
+}
+
+func zeroTTLBackend() plugin.Handler {
+	return plugin.HandlerFunc(func(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+		m := new(dns.Msg)
+		m.SetReply(r)
+		m.Response, m.RecursionAvailable = true, true
+
+		m.Answer = []dns.RR{test.A("example.org. 0 IN A 127.0.0.53")}
+		w.WriteMsg(m)
+		return dns.RcodeSuccess, nil
+	})
+}

coredns_plugin/coredns_stats.go

@@ -0,0 +1,189 @@
+package dnsfilter
+
+import (
+	"sync"
+	"time"
+
+	"github.com/coredns/coredns/plugin"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	requests             = newDNSCounter("requests_total", "Count of requests seen by dnsfilter.")
+	filtered             = newDNSCounter("filtered_total", "Count of requests filtered by dnsfilter.")
+	filteredLists        = newDNSCounter("filtered_lists_total", "Count of requests filtered by dnsfilter using lists.")
+	filteredSafebrowsing = newDNSCounter("filtered_safebrowsing_total", "Count of requests filtered by dnsfilter using safebrowsing.")
+	filteredParental     = newDNSCounter("filtered_parental_total", "Count of requests filtered by dnsfilter using parental.")
+	filteredInvalid      = newDNSCounter("filtered_invalid_total", "Count of requests filtered by dnsfilter because they were invalid.")
+	whitelisted          = newDNSCounter("whitelisted_total", "Count of requests not filtered by dnsfilter because they are whitelisted.")
+	safesearch           = newDNSCounter("safesearch_total", "Count of requests replaced by dnsfilter safesearch.")
+	errorsTotal          = newDNSCounter("errors_total", "Count of requests that dnsfilter couldn't process because of transitive errors.")
+	elapsedTime          = newDNSHistogram("request_duration", "Histogram of the time (in seconds) each request took.")
+)
+
+// entries for single time period (for example all per-second entries)
+type statsEntries map[string][statsHistoryElements]float64
+
+// how far back to keep the stats
+const statsHistoryElements = 60 + 1 // +1 for calculating delta
+
+// each periodic stat is a map of arrays
+type periodicStats struct {
+	Entries    statsEntries
+	period     time.Duration // how long one entry lasts
+	LastRotate time.Time     // last time this data was rotated
+
+	sync.RWMutex
+}
+
+type stats struct {
+	PerSecond periodicStats
+	PerMinute periodicStats
+	PerHour   periodicStats
+	PerDay    periodicStats
+}
+
+// per-second/per-minute/per-hour/per-day stats
+var statistics stats
+
+func initPeriodicStats(periodic *periodicStats, period time.Duration) {
+	periodic.Entries = statsEntries{}
+	periodic.LastRotate = time.Now()
+	periodic.period = period
+}
+
+func init() {
+	purgeStats()
+}
+
+func purgeStats() {
+	initPeriodicStats(&statistics.PerSecond, time.Second)
+	initPeriodicStats(&statistics.PerMinute, time.Minute)
+	initPeriodicStats(&statistics.PerHour, time.Hour)
+	initPeriodicStats(&statistics.PerDay, time.Hour*24)
+}
+
+func (p *periodicStats) Inc(name string, when time.Time) {
+	// calculate how many periods ago this happened
+	elapsed := int64(time.Since(when) / p.period)
+	// trace("%s: %v as %v -> [%v]", name, time.Since(when), p.period, elapsed)
+	if elapsed >= statsHistoryElements {
+		return // outside of our timeframe
+	}
+	p.Lock()
+	currentValues := p.Entries[name]
+	currentValues[elapsed]++
+	p.Entries[name] = currentValues
+	p.Unlock()
+}
+
+func (p *periodicStats) Observe(name string, when time.Time, value float64) {
+	// calculate how many periods ago this happened
+	elapsed := int64(time.Since(when) / p.period)
+	// trace("%s: %v as %v -> [%v]", name, time.Since(when), p.period, elapsed)
+	if elapsed >= statsHistoryElements {
+		return // outside of our timeframe
+	}
+	p.Lock()
+	{
+		countname := name + "_count"
+		currentValues := p.Entries[countname]
+		value := currentValues[elapsed]
+		// trace("Will change p.Entries[%s][%d] from %v to %v", countname, elapsed, value, value+1)
+		value++
+		currentValues[elapsed] = value
+		p.Entries[countname] = currentValues
+	}
+	{
+		totalname := name + "_sum"
+		currentValues := p.Entries[totalname]
+		currentValues[elapsed] += value
+		p.Entries[totalname] = currentValues
+	}
+	p.Unlock()
+}
+
+// counter that wraps around prometheus Counter but also adds to periodic stats
+type counter struct {
+	name  string // used as key in periodic stats
+	value int64
+	prom  prometheus.Counter
+}
+
+func newDNSCounter(name string, help string) *counter {
+	// trace("called")
+	c := &counter{}
+	c.prom = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "dnsfilter",
+		Name:      name,
+		Help:      help,
+	})
+	c.name = name
+
+	return c
+}
+
+func (c *counter) IncWithTime(when time.Time) {
+	statistics.PerSecond.Inc(c.name, when)
+	statistics.PerMinute.Inc(c.name, when)
+	statistics.PerHour.Inc(c.name, when)
+	statistics.PerDay.Inc(c.name, when)
+	c.value++
+	c.prom.Inc()
+}
+
+func (c *counter) Inc() {
+	c.IncWithTime(time.Now())
+}
+
+func (c *counter) Describe(ch chan<- *prometheus.Desc) {
+	c.prom.Describe(ch)
+}
+
+func (c *counter) Collect(ch chan<- prometheus.Metric) {
+	c.prom.Collect(ch)
+}
+
+type histogram struct {
+	name  string // used as key in periodic stats
+	count int64
+	total float64
+	prom  prometheus.Histogram
+}
+
+func newDNSHistogram(name string, help string) *histogram {
+	// trace("called")
+	h := &histogram{}
+	h.prom = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "dnsfilter",
+		Name:      name,
+		Help:      help,
+	})
+	h.name = name
+
+	return h
+}
+
+func (h *histogram) ObserveWithTime(value float64, when time.Time) {
+	statistics.PerSecond.Observe(h.name, when, value)
+	statistics.PerMinute.Observe(h.name, when, value)
+	statistics.PerHour.Observe(h.name, when, value)
+	statistics.PerDay.Observe(h.name, when, value)
+	h.count++
+	h.total += value
+	h.prom.Observe(value)
+}
+
+func (h *histogram) Observe(value float64) {
+	h.ObserveWithTime(value, time.Now())
+}
+
+func (h *histogram) Describe(ch chan<- *prometheus.Desc) {
+	h.prom.Describe(ch)
+}
+
+func (h *histogram) Collect(ch chan<- prometheus.Metric) {
+	h.prom.Collect(ch)
+}

coredns_plugin/ratelimit/ratelimit.go

@@ -0,0 +1,182 @@
+package ratelimit
+
+import (
+	"errors"
+	"log"
+	"sort"
+	"strconv"
+	"time"
+
+	// ratelimiting and per-ip buckets
+	"github.com/beefsack/go-rate"
+	"github.com/patrickmn/go-cache"
+
+	// coredns plugin
+	"github.com/coredns/coredns/core/dnsserver"
+	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/metrics"
+	"github.com/coredns/coredns/plugin/pkg/dnstest"
+	"github.com/coredns/coredns/request"
+	"github.com/mholt/caddy"
+	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+)
+
+const defaultRatelimit = 30
+const defaultResponseSize = 1000
+
+var (
+	tokenBuckets = cache.New(time.Hour, time.Hour)
+)
+
+// ServeDNS handles the DNS request and refuses if it's an beyind specified ratelimit
+func (p *plug) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	state := request.Request{W: w, Req: r}
+	ip := state.IP()
+	allow, err := p.allowRequest(ip)
+	if err != nil {
+		return 0, err
+	}
+	if !allow {
+		ratelimited.Inc()
+		return 0, nil
+	}
+
+	// Record response to get status code and size of the reply.
+	rw := dnstest.NewRecorder(w)
+	status, err := plugin.NextOrFailure(p.Name(), p.Next, ctx, rw, r)
+
+	size := rw.Len
+
+	if size > defaultResponseSize && state.Proto() == "udp" {
+		// For large UDP responses we call allowRequest more times
+		// The exact number of times depends on the response size
+		for i := 0; i < size/defaultResponseSize; i++ {
+			p.allowRequest(ip)
+		}
+	}
+
+	return status, err
+}
+
+func (p *plug) allowRequest(ip string) (bool, error) {
+	if len(p.whitelist) > 0 {
+		i := sort.SearchStrings(p.whitelist, ip)
+
+		if i < len(p.whitelist) && p.whitelist[i] == ip {
+			return true, nil
+		}
+	}
+
+	if _, found := tokenBuckets.Get(ip); !found {
+		tokenBuckets.Set(ip, rate.New(p.ratelimit, time.Second), time.Hour)
+	}
+
+	value, found := tokenBuckets.Get(ip)
+	if !found {
+		// should not happen since we've just inserted it
+		text := "SHOULD NOT HAPPEN: just-inserted ratelimiter disappeared"
+		log.Println(text)
+		err := errors.New(text)
+		return true, err
+	}
+
+	rl, ok := value.(*rate.RateLimiter)
+	if !ok {
+		text := "SHOULD NOT HAPPEN: non-bool entry found in safebrowsing lookup cache"
+		log.Println(text)
+		err := errors.New(text)
+		return true, err
+	}
+
+	allow, _ := rl.Try()
+	return allow, nil
+}
+
+//
+// helper functions
+//
+func init() {
+	caddy.RegisterPlugin("ratelimit", caddy.Plugin{
+		ServerType: "dns",
+		Action:     setup,
+	})
+}
+
+type plug struct {
+	Next plugin.Handler
+
+	// configuration for creating above
+	ratelimit int      // in requests per second per IP
+	whitelist []string // a list of whitelisted IP addresses
+}
+
+func setupPlugin(c *caddy.Controller) (*plug, error) {
+	p := &plug{ratelimit: defaultRatelimit}
+
+	for c.Next() {
+		args := c.RemainingArgs()
+		if len(args) > 0 {
+			ratelimit, err := strconv.Atoi(args[0])
+			if err != nil {
+				return nil, c.ArgErr()
+			}
+			p.ratelimit = ratelimit
+		}
+		for c.NextBlock() {
+			switch c.Val() {
+			case "whitelist":
+				p.whitelist = c.RemainingArgs()
+
+				if len(p.whitelist) > 0 {
+					sort.Strings(p.whitelist)
+				}
+			}
+		}
+	}
+
+	return p, nil
+}
+
+func setup(c *caddy.Controller) error {
+	p, err := setupPlugin(c)
+	if err != nil {
+		return err
+	}
+
+	config := dnsserver.GetConfig(c)
+	config.AddPlugin(func(next plugin.Handler) plugin.Handler {
+		p.Next = next
+		return p
+	})
+
+	c.OnStartup(func() error {
+		m := dnsserver.GetConfig(c).Handler("prometheus")
+		if m == nil {
+			return nil
+		}
+		if x, ok := m.(*metrics.Metrics); ok {
+			x.MustRegister(ratelimited)
+		}
+		return nil
+	})
+
+	return nil
+}
+
+func newDNSCounter(name string, help string) prometheus.Counter {
+	return prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "ratelimit",
+		Name:      name,
+		Help:      help,
+	})
+}
+
+var (
+	ratelimited = newDNSCounter("dropped_total", "Count of requests that have been dropped because of rate limit")
+)
+
+// Name returns name of the plugin as seen in Corefile and plugin.cfg
+func (p *plug) Name() string { return "ratelimit" }

coredns_plugin/ratelimit/ratelimit_test.go

@@ -0,0 +1,80 @@
+package ratelimit
+
+import (
+	"testing"
+
+	"github.com/mholt/caddy"
+)
+
+func TestSetup(t *testing.T) {
+	for i, testcase := range []struct {
+		config  string
+		failing bool
+	}{
+		{`ratelimit`, false},
+		{`ratelimit 100`, false},
+		{`ratelimit { 
+					whitelist 127.0.0.1
+				}`, false},
+		{`ratelimit 50 {
+					whitelist 127.0.0.1 176.103.130.130
+				}`, false},
+		{`ratelimit test`, true},
+	} {
+		c := caddy.NewTestController("dns", testcase.config)
+		err := setup(c)
+		if err != nil {
+			if !testcase.failing {
+				t.Fatalf("Test #%d expected no errors, but got: %v", i, err)
+			}
+			continue
+		}
+		if testcase.failing {
+			t.Fatalf("Test #%d expected to fail but it didn't", i)
+		}
+	}
+}
+
+func TestRatelimiting(t *testing.T) {
+	// rate limit is 1 per sec
+	c := caddy.NewTestController("dns", `ratelimit 1`)
+	p, err := setupPlugin(c)
+
+	if err != nil {
+		t.Fatal("Failed to initialize the plugin")
+	}
+
+	allowed, err := p.allowRequest("127.0.0.1")
+
+	if err != nil || !allowed {
+		t.Fatal("First request must have been allowed")
+	}
+
+	allowed, err = p.allowRequest("127.0.0.1")
+
+	if err != nil || allowed {
+		t.Fatal("Second request must have been ratelimited")
+	}
+}
+
+func TestWhitelist(t *testing.T) {
+	// rate limit is 1 per sec
+	c := caddy.NewTestController("dns", `ratelimit 1 { whitelist 127.0.0.2 127.0.0.1 127.0.0.125 }`)
+	p, err := setupPlugin(c)
+
+	if err != nil {
+		t.Fatal("Failed to initialize the plugin")
+	}
+
+	allowed, err := p.allowRequest("127.0.0.1")
+
+	if err != nil || !allowed {
+		t.Fatal("First request must have been allowed")
+	}
+
+	allowed, err = p.allowRequest("127.0.0.1")
+
+	if err != nil || !allowed {
+		t.Fatal("Second request must have been allowed due to whitelist")
+	}
+}

coredns_plugin/refuseany/refuseany.go

@@ -0,0 +1,91 @@
+package refuseany
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/coredns/coredns/core/dnsserver"
+	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/metrics"
+	"github.com/coredns/coredns/request"
+	"github.com/mholt/caddy"
+	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+)
+
+type plug struct {
+	Next plugin.Handler
+}
+
+// ServeDNS handles the DNS request and refuses if it's an ANY request
+func (p *plug) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	if len(r.Question) != 1 {
+		// google DNS, bind and others do the same
+		return dns.RcodeFormatError, fmt.Errorf("Got DNS request with != 1 questions")
+	}
+
+	q := r.Question[0]
+	if q.Qtype == dns.TypeANY {
+		state := request.Request{W: w, Req: r, Context: ctx}
+		rcode := dns.RcodeNotImplemented
+
+		m := new(dns.Msg)
+		m.SetRcode(r, rcode)
+		state.SizeAndDo(m)
+		err := state.W.WriteMsg(m)
+		if err != nil {
+			log.Printf("Got error %s\n", err)
+			return dns.RcodeServerFailure, err
+		}
+		return rcode, nil
+	}
+
+	return plugin.NextOrFailure(p.Name(), p.Next, ctx, w, r)
+}
+
+func init() {
+	caddy.RegisterPlugin("refuseany", caddy.Plugin{
+		ServerType: "dns",
+		Action:     setup,
+	})
+}
+
+func setup(c *caddy.Controller) error {
+	p := &plug{}
+	config := dnsserver.GetConfig(c)
+
+	config.AddPlugin(func(next plugin.Handler) plugin.Handler {
+		p.Next = next
+		return p
+	})
+
+	c.OnStartup(func() error {
+		m := dnsserver.GetConfig(c).Handler("prometheus")
+		if m == nil {
+			return nil
+		}
+		if x, ok := m.(*metrics.Metrics); ok {
+			x.MustRegister(ratelimited)
+		}
+		return nil
+	})
+
+	return nil
+}
+
+func newDNSCounter(name string, help string) prometheus.Counter {
+	return prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "refuseany",
+		Name:      name,
+		Help:      help,
+	})
+}
+
+var (
+	ratelimited = newDNSCounter("refusedany_total", "Count of ANY requests that have been dropped")
+)
+
+// Name returns name of the plugin as seen in Corefile and plugin.cfg
+func (p *plug) Name() string { return "refuseany" }

plugin.cfg

@@ -26,9 +26,9 @@ pprof:pprof
 prometheus:metrics
 errors:errors
 log:log
-ratelimit:github.com/AdguardTeam/AdguardDNS/coredns_plugin/ratelimit
-refuseany:github.com/AdguardTeam/AdguardDNS/coredns_plugin/refuseany
-dnsfilter:github.com/AdguardTeam/AdguardDNS/coredns_plugin
+ratelimit:bit.adguard.com/dns/adguard-internal-dns/coredns_plugin/ratelimit
+refuseany:bit.adguard.com/dns/adguard-internal-dns/coredns_plugin/refuseany
+dnsfilter:bit.adguard.com/dns/adguard-internal-dns/coredns_plugin
 cache:cache
 template:template
 file:file