Add support for per-Source Hidden Neighbors #143

Open
jof wants to merge 1 commits from jof/jof/hidden-neighbors into develop
14 changed files with 189 additions and 33 deletions

View File

@ -204,6 +204,8 @@ name = rs1.example.com (IPv4)
# Optional: a group for the routeservers list
group = FRA
blackholes = 10.23.6.666, 10.23.6.665
# Optional: hidden_neighbors can list strings of IPs, CIDRs, or Regexes to filter from being shown
hidden_neighbors = 10.0.0.0/8, 192.168.0.0/16, AS64496_.*
[source.rs0-example-v4.birdwatcher]
api = http://rs1.example.com:29184/

View File

@ -192,6 +192,8 @@ type SourceConfig struct {
// Blackhole IPs
Blackholes []string
HiddenNeighbors []string
// Source configurations
Type string
Backend string
@ -748,15 +750,17 @@ func getSources(config *ini.File) ([]*SourceConfig, error) {
sourceGroup := section.Key("group").MustString("")
sourceBlackholes := decoders.TrimmedCSVStringList(
section.Key("blackholes").MustString(""))
sourceHiddenNeighbors := decoders.TrimmedCSVStringList(section.Key("hidden_neighbors").MustString(""))
srcCfg := &SourceConfig{
ID: sourceID,
Order: order,
Name: sourceName,
Group: sourceGroup,
Blackholes: sourceBlackholes,
Backend: backendType,
Type: sourceType,
ID: sourceID,
Order: order,
Name: sourceName,
Group: sourceGroup,
Blackholes: sourceBlackholes,
HiddenNeighbors: sourceHiddenNeighbors,
Backend: backendType,
Type: sourceType,
}
// Register route server ID with pool
@ -776,8 +780,9 @@ func getSources(config *ini.File) ([]*SourceConfig, error) {
}
c := birdwatcher.Config{
ID: srcCfg.ID,
Name: srcCfg.Name,
ID: srcCfg.ID,
Name: srcCfg.Name,
HiddenNeighbors: srcCfg.HiddenNeighbors,
Timezone: "UTC",
ServerTime: "2006-01-02T15:04:05.999999999Z07:00",
@ -808,8 +813,9 @@ func getSources(config *ini.File) ([]*SourceConfig, error) {
case SourceBackendGoBGP:
c := gobgp.Config{
ID: srcCfg.ID,
Name: srcCfg.Name,
ID: srcCfg.ID,
Name: srcCfg.Name,
HiddenNeighbors: srcCfg.HiddenNeighbors,
}
if err := backendConfig.MapTo(&c); err != nil {
@ -836,6 +842,7 @@ func getSources(config *ini.File) ([]*SourceConfig, error) {
c := openbgpd.Config{
ID: srcCfg.ID,
Name: srcCfg.Name,
HiddenNeighbors: srcCfg.HiddenNeighbors,
CacheTTL: cacheTTL,
RoutesCacheSize: routesCacheSize,
RejectCommunities: rejectComms,
@ -858,6 +865,7 @@ func getSources(config *ini.File) ([]*SourceConfig, error) {
c := openbgpd.Config{
ID: srcCfg.ID,
Name: srcCfg.Name,
HiddenNeighbors: srcCfg.HiddenNeighbors,
CacheTTL: cacheTTL,
RoutesCacheSize: routesCacheSize,
RejectCommunities: rejectComms,
@ -1007,9 +1015,9 @@ func (cfg *SourceConfig) GetInstance() sources.Source {
var instance sources.Source
switch cfg.Backend {
case SourceBackendBirdwatcher:
instance = birdwatcher.NewBirdwatcher(cfg.Birdwatcher)
instance = birdwatcher.NewBirdwatcher(&cfg.Birdwatcher)
case SourceBackendGoBGP:
instance = gobgp.NewGoBGP(cfg.GoBGP)
instance = gobgp.NewGoBGP(&cfg.GoBGP)
case SourceBackendOpenBGPDStateServer:
instance = openbgpd.NewStateServerSource(&cfg.OpenBGPD)
case SourceBackendOpenBGPDBgplgd:

View File

@ -2,9 +2,6 @@ package config
import (
"testing"
"github.com/alice-lg/alice-lg/pkg/sources/birdwatcher"
"github.com/alice-lg/alice-lg/pkg/sources/gobgp"
)
// Test configuration loading and parsing
@ -58,14 +55,13 @@ func TestSourceConfig(t *testing.T) {
rs2 := config.Sources[1] // Birdwatcher v6
rs3 := config.Sources[2] // GoBGP
nilBirdwatcherConfig := birdwatcher.Config{}
if rs1.Birdwatcher == nilBirdwatcherConfig {
if rs1.Backend == SourceBackendBirdwatcher {
t.Errorf(
"Example routeserver %s should have been identified as a birdwatcher source but was not",
rs1.Name,
)
}
if rs2.Birdwatcher == nilBirdwatcherConfig {
if rs2.Backend == SourceBackendBirdwatcher {
t.Errorf(
"Example routeserver %s should have been identified as a birdwatcher source but was not",
rs2.Name,
@ -78,8 +74,7 @@ func TestSourceConfig(t *testing.T) {
t.Error("Unexpected StreamParserThrottle", rs2.Birdwatcher.StreamParserThrottle)
}
}
nilGoBGPConfig := gobgp.Config{}
if rs3.GoBGP == nilGoBGPConfig {
if rs3.Backend == SourceBackendGoBGP {
t.Errorf(
"Example routeserver %s should have been identified as a gobgp source but was not",
rs3.Name,

View File

@ -3,8 +3,9 @@ package birdwatcher
// Config contains all configuration attributes
// for a birdwatcher based source.
type Config struct {
ID string
Name string
ID string
Name string
HiddenNeighbors []string
API string `ini:"api"`
Timezone string `ini:"timezone"`

View File

@ -36,7 +36,7 @@ type GenericBirdwatcher struct {
// NewBirdwatcher creates a new Birdwatcher instance.
// This might be either a GenericBirdWatcher or a MultiTableBirdwatcher.
func NewBirdwatcher(config Config) Birdwatcher {
func NewBirdwatcher(config *Config) Birdwatcher {
client := NewClient(config.API)
// Cache settings:
@ -58,7 +58,7 @@ func NewBirdwatcher(config Config) Birdwatcher {
if config.Type == "single_table" {
singleTableBirdwatcher := new(SingleTableBirdwatcher)
singleTableBirdwatcher.config = config
singleTableBirdwatcher.config = *config
singleTableBirdwatcher.client = client
singleTableBirdwatcher.neighborsCache = neighborsCache
@ -72,7 +72,7 @@ func NewBirdwatcher(config Config) Birdwatcher {
} else if config.Type == "multi_table" {
multiTableBirdwatcher := new(MultiTableBirdwatcher)
multiTableBirdwatcher.config = config
multiTableBirdwatcher.config = *config
multiTableBirdwatcher.client = client
multiTableBirdwatcher.neighborsCache = neighborsCache
@ -250,6 +250,10 @@ func (b *GenericBirdwatcher) NeighborsStatus(ctx context.Context) (
if err != nil {
return nil, err
}
neighbors, err = sources.FilterHiddenNeighborsStatus(neighbors, b.config.HiddenNeighbors)
if err != nil {
return nil, err
}
response := &api.NeighborsStatusResponse{
Response: api.Response{

View File

@ -11,6 +11,7 @@ import (
"github.com/alice-lg/alice-lg/pkg/api"
"github.com/alice-lg/alice-lg/pkg/decoders"
"github.com/alice-lg/alice-lg/pkg/pools"
"github.com/alice-lg/alice-lg/pkg/sources"
)
// MultiTableBirdwatcher implements a birdwatcher with
@ -512,6 +513,11 @@ func (src *MultiTableBirdwatcher) Neighbors(
}
}
neighbors, err = sources.FilterHiddenNeighbors(neighbors, src.config.HiddenNeighbors)
if err != nil {
return nil, err
}
response = &api.NeighborsResponse{
Response: api.Response{
Meta: apiStatus,

View File

@ -5,6 +5,7 @@ import (
"log"
"github.com/alice-lg/alice-lg/pkg/api"
"github.com/alice-lg/alice-lg/pkg/sources"
)
// SingleTableBirdwatcher is an Alice Source
@ -159,6 +160,11 @@ func (src *SingleTableBirdwatcher) Neighbors(
return nil, err
}
neighbors, err = sources.FilterHiddenNeighbors(neighbors, src.config.HiddenNeighbors)
if err != nil {
return nil, err
}
response = &api.NeighborsResponse{
Response: api.Response{
Meta: apiStatus,

View File

@ -2,8 +2,9 @@ package gobgp
// Config is a GoBGP source config
type Config struct {
ID string
Name string
ID string
Name string
HiddenNeighbors []string
Host string `ini:"host"`
Insecure bool `ini:"insecure"`

View File

@ -34,7 +34,7 @@ type GoBGP struct {
}
// NewGoBGP creates a new GoBGP source instance
func NewGoBGP(config Config) *GoBGP {
func NewGoBGP(config *Config) *GoBGP {
dialOpts := make([]grpc.DialOption, 0)
if config.Insecure {
dialOpts = append(dialOpts, grpc.WithInsecure())
@ -74,7 +74,7 @@ func NewGoBGP(config Config) *GoBGP {
routesCacheDisabled, routesCacheMaxSize)
return &GoBGP{
config: config,
config: *config,
client: client,
neighborsCache: neighborsCache,
@ -132,6 +132,10 @@ func (gobgp *GoBGP) NeighborsStatus(
}
}
response.Neighbors, err = sources.FilterHiddenNeighborsStatus(response.Neighbors, gobgp.config.HiddenNeighbors)
if err != nil {
return nil, err
}
return &response, nil
}
@ -206,6 +210,11 @@ func (gobgp *GoBGP) Neighbors(
}
response.Neighbors, err = sources.FilterHiddenNeighbors(response.Neighbors, gobgp.config.HiddenNeighbors)
if err != nil {
return nil, err
}
return &response, nil
}

View File

@ -158,6 +158,10 @@ func (src *BgplgdSource) Neighbors(
if err != nil {
return nil, err
}
nb, err = sources.FilterHiddenNeighbors(nb, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
// Set route server id (sourceID) for all neighbors and
// calculate the filtered routes.
for _, n := range nb {
@ -211,6 +215,11 @@ func (src *BgplgdSource) NeighborsSummary(
if err != nil {
return nil, err
}
nb, err = sources.FilterHiddenNeighbors(nb, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
// Set route server id (sourceID) for all neighbors and
// calculate the filtered routes.
for _, n := range nb {
@ -252,6 +261,10 @@ func (src *BgplgdSource) NeighborsStatus(
if err != nil {
return nil, err
}
nb, err = sources.FilterHiddenNeighborsStatus(nb, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
response := &api.NeighborsStatusResponse{
Response: api.Response{

View File

@ -10,8 +10,9 @@ import (
// Config is a OpenBGPD source config
type Config struct {
ID string
Name string
ID string
Name string
HiddenNeighbors []string
CacheTTL time.Duration
RoutesCacheSize int

View File

@ -200,6 +200,10 @@ func (src *StateServerSource) Neighbors(
},
Neighbors: nb,
}
response.Neighbors, err = sources.FilterHiddenNeighbors(response.Neighbors, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
src.neighborsCache.Set(response)
return response, nil
@ -235,6 +239,10 @@ func (src *StateServerSource) NeighborsSummary(
if err != nil {
return nil, err
}
nb, err = sources.FilterHiddenNeighbors(nb, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
// Set route server id (sourceID) for all neighbors
for _, n := range nb {
n.RouteServerID = src.cfg.ID
@ -275,6 +283,10 @@ func (src *StateServerSource) NeighborsStatus(
if err != nil {
return nil, err
}
nb, err = sources.FilterHiddenNeighborsStatus(nb, src.cfg.HiddenNeighbors)
if err != nil {
return nil, err
}
response := &api.NeighborsStatusResponse{
Response: api.Response{

View File

@ -5,6 +5,9 @@ package sources
import (
"context"
"errors"
"fmt"
"net"
"regexp"
"github.com/alice-lg/alice-lg/pkg/api"
)
@ -21,7 +24,7 @@ var (
)
// Source is a generic datasource for alice.
// All route server adapters implement this interface.
// All route server Source adapters implement this interface.
type Source interface {
ExpireCaches() int
@ -35,3 +38,96 @@ type Source interface {
RoutesNotExported(ctx context.Context, neighborID string) (*api.RoutesResponse, error)
AllRoutes(context.Context) (*api.RoutesResponse, error)
}
func hiddenNeighborsExcludeLists(hiddenNeighbors []string) ([]*net.IPNet, []net.IP, []*regexp.Regexp, error) {
excludeCIDRs := make([]*net.IPNet, 0, len(hiddenNeighbors))
excludeIPs := make([]net.IP, 0, len(hiddenNeighbors))
excludePatterns := make([]*regexp.Regexp, 0, len(hiddenNeighbors))
for _, hiddenNeighbor := range hiddenNeighbors {
if _, hiddenNet, err := net.ParseCIDR(hiddenNeighbor); err == nil {
excludeCIDRs = append(excludeCIDRs, hiddenNet)
} else if ip := net.ParseIP(hiddenNeighbor); ip != nil {
excludeIPs = append(excludeIPs, ip)
} else {
pattern, err := regexp.Compile(hiddenNeighbor)
if err != nil {
return nil, nil, nil, err
}
excludePatterns = append(excludePatterns, pattern)
}
}
return excludeCIDRs, excludeIPs, excludePatterns, nil
}
annikahannig commented 2024-02-29 17:51:29 +08:00 (Migrated from github.com)
Review

This function has too many return values.
Also I think it should be done during config parsing.

Also, maybe the already existing filtering infrastructure could be reused:

There is api.NeighborFilter which could be extended:

  • api.Filterable this should maybe be split into RouteFilterable and NeighborFilterable
  • NeighborFilter.Match should accept this interface, so this can be implemented for api.Neighbor and api.NeighborStatus
This function has too many return values. Also I think it should be done during config parsing. Also, maybe the already existing filtering infrastructure could be reused: There is `api.NeighborFilter` which could be extended: - `api.Filterable` this should maybe be split into `RouteFilterable` and `NeighborFilterable` - `NeighborFilter.Match` should accept this interface, so this can be implemented for `api.Neighbor` and `api.NeighborStatus`
annikahannig commented 2024-02-29 18:21:25 +08:00 (Migrated from github.com)
Review

The NeighborFilter could then include this list of CIDRs / IPs / RE patterns.

The NeighborFilterable then could maybe expect MatchAddress MatchASN, MatchDescription MatchPattern?

something like this :)

The `NeighborFilter` could then include this list of CIDRs / IPs / RE patterns. The NeighborFilterable then could maybe expect `MatchAddress` `MatchASN`, `MatchDescription` `MatchPattern?` something like this :)
func FilterHiddenNeighbors(neighbors api.Neighbors, hiddenNeighbors []string) (api.Neighbors, error) {
if len(hiddenNeighbors) > 0 {
filteredNeighbors := make(api.Neighbors, 0, len(neighbors))
excludeCIDRs, excludeIPs, excludePatterns, err := hiddenNeighborsExcludeLists(hiddenNeighbors)
if err != nil {
return nil, err
}
neighbors:
for _, neighbor := range neighbors {
neighborIP := net.ParseIP(neighbor.Address)
annikahannig commented 2024-02-29 18:24:04 +08:00 (Migrated from github.com)
Review

Another reason to do this at config parse time:
If there is a broken pattern in the config, this would fail when routes refresh starts or, if there is no global search enabled, on first request.

Another reason to do this at config parse time: If there is a broken pattern in the config, this would fail when routes refresh starts or, if there is no global search enabled, on first request.
if neighborIP == nil {
return nil, fmt.Errorf("Neighbor ID '%s' is not parseable as an IP", neighborIP)
}
if neighborIP != nil {
for _, ip := range excludeIPs {
if neighborIP.Equal(ip) {
continue neighbors
}
}
for _, cidr := range excludeCIDRs {
annikahannig commented 2024-02-29 18:09:43 +08:00 (Migrated from github.com)
Review

If the hidden_members config option is parsed into a e.g. []*NeighborFilter / filterset something like

func (filters []*NeighborFilter) Exclude[T NeighborFilterable](n []T) ([]T, error) {
  for filters ... {
     if n.Match(...) skip
  }
}

Then the invocation in the source could be like

neighbors := make(api.Neighbors, 0)
... decode neighbors...
response.Neighbors, err = src.cfg.HiddenNeighbors.Exclude(neighbors)
	if err != nil {
		return nil, err
	}
If the `hidden_members` config option is parsed into a e.g. []*NeighborFilter / filterset something like ``` func (filters []*NeighborFilter) Exclude[T NeighborFilterable](n []T) ([]T, error) { for filters ... { if n.Match(...) skip } } ``` Then the invocation in the source could be like ``` neighbors := make(api.Neighbors, 0) ... decode neighbors... response.Neighbors, err = src.cfg.HiddenNeighbors.Exclude(neighbors) if err != nil { return nil, err } ```
annikahannig commented 2024-02-29 18:29:48 +08:00 (Migrated from github.com)
Review

Or: even better: before it is appended during decode:

if cfg.HiddenNeighbors.Match(...) {
  continue
 }
 response.Neighbors = append(...

Then the Exclude function would not even be required. :)

Or: even better: before it is appended during decode: ``` if cfg.HiddenNeighbors.Match(...) { continue } response.Neighbors = append(... ``` Then the Exclude function would not even be required. :)
if cidr.Contains(neighborIP) {
continue neighbors
}
}
}
for _, pattern := range excludePatterns {
if pattern.MatchString(neighbor.Address) {
continue neighbors
}
}
filteredNeighbors = append(filteredNeighbors, neighbor)
}
return filteredNeighbors, nil
} else {
return neighbors, nil
}
}
func FilterHiddenNeighborsStatus(neighborsStatus api.NeighborsStatus, hiddenNeighbors []string) (api.NeighborsStatus, error) {
if len(hiddenNeighbors) > 0 {
filteredNeighborsStatus := make(api.NeighborsStatus, 0, len(neighborsStatus))
excludeCIDRs, excludeIPs, excludePatterns, err := hiddenNeighborsExcludeLists(hiddenNeighbors)
if err != nil {
return neighborsStatus, err
}
neighbors:
for _, neighborStatus := range neighborsStatus {
neighborIDAsIP := net.ParseIP(neighborStatus.ID)
if neighborIDAsIP != nil {
for _, ip := range excludeIPs {
if ip.Equal(neighborIDAsIP) {
continue neighbors
}
annikahannig commented 2024-02-29 18:05:13 +08:00 (Migrated from github.com)
Review

This should rather be an early return, to reduce nesting.

This should rather be an early return, to reduce nesting.
}
for _, cidr := range excludeCIDRs {
if cidr.Contains(neighborIDAsIP) {
continue neighbors
}
}
}
for _, pattern := range excludePatterns {
if pattern.MatchString(neighborStatus.ID) {
continue neighbors
}
}
filteredNeighborsStatus = append(filteredNeighborsStatus, neighborStatus)
}
return filteredNeighborsStatus, nil
} else {
return neighborsStatus, nil
}
}

View File

@ -54,6 +54,7 @@ type NeighborsStoreBackend interface {
type NeighborsStore struct {
backend NeighborsStoreBackend
sources *SourcesStore
cfg *config.Config
annikahannig commented 2024-02-29 18:16:34 +08:00 (Migrated from github.com)
Review

I think this is an artifact?

I think this is an artifact?
jof commented 2024-03-01 04:26:19 +08:00 (Migrated from github.com)
Review

Indeed, it was.

Indeed, it was.
forceNeighborRefresh bool
}
@ -92,6 +93,7 @@ func NewNeighborsStore(
backend: backend,
sources: sources,
forceNeighborRefresh: forceNeighborRefresh,
cfg: cfg,
}
return store
}