diff --git a/CHANGELOG.md b/CHANGELOG.md index 151b513..66f7d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,45 @@ # Changelog + +## 4.0.1 (2019-03-07) + +* Enhance the neighbors store to perform uncached requests for peer status + on every request. A timeout with fallback to cached data is applied in order + too keep the response times low. +* Add caching to Neighbors() + +## 4.0.0 (2019-02-22) + +Breaking Changes: Birdwatcher 2.0 + +Support for birdwatcher route server API implementation version 2.0.0 and above. +This new implementation of birdwatcher only provides the direct output of the +birdc comands and eliminates complex endpoints that fetch data from multiple +birdc responses. The aggregation of data, based on the particular route server +setup in use is now implemented in Alice-LG. +Therefore the birdwatcher source can be configured with a new config parameter +'type', which specifies a processing strategy for the ingested data which +corresponds to a particular layout of the routing daemon (BIRD) configuration +(e.g. single-table, multi-table or something even more custom). For developers +it is made easy to add new configuration types. + +The neighbor summary has been removed, since much of it's data can be requested +from the new birdwatcher endpoints in alternative ways. + +The config option from birdwatcher "PeerTablePrefix" and "PipeProtocolPrefix" +have been carried over to Alice-LG. These constants may be defined on a +per route server basis and are used to generate the request URLs for the +route server (birdwatcher) API in case of multi-table setup. + +In addition this version contains the following bug-fixes and features: +* Fix a bug in Neighbors(), a peer that is down would cause a runtime error +* Fix the cache, it would still store entries even if disabled +* Fix a bug affecting the cache (subsequent modification of entries) +* Remove additional caches to avoid duplicate caching and save memory +* Save memory by periodically expiring entries with a housekeeping routine +* Change extended communities format to (string, string, string) + ## 3.4.4 (2019-01-29) * Loading indicators in frontend for received routes and filtered routes diff --git a/README.md b/README.md index 3d18ae1..aa44cac 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,10 @@ And checkout the API at: Alice-LG is a BGP looking glass which gets its data from external APIs. Currently Alice-LG supports the following APIs: -- [birdwatcher API](https://github.com/ecix/birdwatcher) for [BIRD](http://bird.network.cz/) +- [birdwatcher API](https://github.com/alice-lg/birdwatcher) for [BIRD](http://bird.network.cz/) -Normally you would first install the [birdwatcher API](https://github.com/ecix/birdwatcher) directly on the machine(s) where you run [BIRD](http://bird.network.cz/) on -and then install Alice-LG on a seperate public facing server and point her to the afore mentioned [birdwatcher API](https://github.com/ecix/birdwatcher). +Normally you would first install the [birdwatcher API](https://github.com/alice-lg/birdwatcher) directly on the machine(s) where you run [BIRD](http://bird.network.cz/) on +and then install Alice-LG on a seperate public facing server and point her to the afore mentioned [birdwatcher API](https://github.com/alice-lg/birdwatcher). This project was a direct result of the [RIPE IXP Tools Hackathon](https://atlas.ripe.net/hackathon/ixp-tools/) just prior to [RIPE73](https://ripe73.ripe.net/) in Madrid, Spain. @@ -76,16 +76,21 @@ You can copy it to any of the following locations: You will have to edit the configuration file as you need to point Alice-LG to the correct [APIs](https://github.com/alice-lg/birdwatcher): ```ini -[source.0] +[source.rs1-example-v4] name = rs1.example.com (IPv4) -[source.0.birdwatcher] +[source.rs1-example-v4.birdwatcher] api = http://rs1.example.com:29184/ # show_last_reboot = true # timezone = UTC +# type = single_table / multi_table +type = multi_table +# not needed for single_table +peer_table_prefix = T +pipe_protocol_prefix = M -[source.1] +[source.rs1-example-v6] name = rs1.example.com (IPv6) -[source.1.birdwatcher] +[source.rs1-example-v6.birdwatcher] api = http://rs1.example.com:29186/ ``` diff --git a/VERSION b/VERSION index f989260..fcdb2e1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.4.4 +4.0.0 diff --git a/backend/api/response_neighbors.go b/backend/api/response_neighbors.go index f170d16..6221a4c 100644 --- a/backend/api/response_neighbors.go +++ b/backend/api/response_neighbors.go @@ -52,3 +52,30 @@ func (self *NeighboursResponse) CacheTtl() time.Duration { } type NeighboursLookupResults map[string]Neighbours + + +type NeighboursStatus []*NeighbourStatus + +type NeighbourStatus struct { + Id string `json:"id"` + State string `json:"state"` + Since time.Duration `json:"uptime"` +} + +// Implement sorting interface for status +func (neighbours NeighboursStatus) Len() int { + return len(neighbours) +} + +func (neighbours NeighboursStatus) Less(i, j int) bool { + return neighbours[i].Id < neighbours[j].Id +} + +func (neighbours NeighboursStatus) Swap(i, j int) { + neighbours[i], neighbours[j] = neighbours[j], neighbours[i] +} + +type NeighboursStatusResponse struct { + Api ApiStatus `json:"api"` + Neighbours NeighboursStatus `json:"neighbours"` +} diff --git a/backend/api/response_test.go b/backend/api/response_test.go index df77615..6e25515 100644 --- a/backend/api/response_test.go +++ b/backend/api/response_test.go @@ -72,7 +72,7 @@ func TestCommunityStringify(t *testing.T) { t.Error("Expected 23:42, got:", com.String()) } - extCom := ExtCommunity{"ro", 42, 123} + extCom := ExtCommunity{"ro", "42", "123"} if extCom.String() != "ro:42:123" { t.Error("Expected ro:42:123, but got:", extCom.String()) } @@ -88,8 +88,8 @@ func TestHasCommunity(t *testing.T) { Community{42, 23}, }, ExtCommunities: []ExtCommunity{ - ExtCommunity{"rt", 23, 42}, - ExtCommunity{"ro", 123, 456}, + ExtCommunity{"rt", "23", "42"}, + ExtCommunity{"ro", "123", "456"}, }, LargeCommunities: []Community{ Community{1000, 23, 42}, @@ -105,11 +105,11 @@ func TestHasCommunity(t *testing.T) { t.Error("Expected community 111:11 to be not present") } - if bgp.HasExtCommunity(ExtCommunity{"ro", 123, 456}) == false { + if bgp.HasExtCommunity(ExtCommunity{"ro", "123", "456"}) == false { t.Error("Expected ro:123:456 in ext community set") } - if bgp.HasExtCommunity(ExtCommunity{"ro", 111, 11}) != false { + if bgp.HasExtCommunity(ExtCommunity{"ro", "111", "11"}) != false { t.Error("Expected ro:111:111 not in ext community set") } @@ -133,9 +133,9 @@ func TestUniqueCommunities(t *testing.T) { func TestUniqueExtCommunities(t *testing.T) { all := ExtCommunities{ - ExtCommunity{"rt", 23, 42}, - ExtCommunity{"ro", 42, 123}, - ExtCommunity{"rt", 23, 42}} + ExtCommunity{"rt", "23", "42"}, + ExtCommunity{"ro", "42", "123"}, + ExtCommunity{"rt", "23", "42"}} unique := all.Unique() if len(unique) != 2 { t.Error("len(unique) should be < len(all)") diff --git a/backend/api/search_filters_parsers.go b/backend/api/search_filters_parsers.go index 25143f0..06dc8e5 100644 --- a/backend/api/search_filters_parsers.go +++ b/backend/api/search_filters_parsers.go @@ -65,15 +65,7 @@ func parseExtCommunityValue(value string) (*SearchFilter, error) { community := make(ExtCommunity, len(components)) for i, c := range components { - if i == 0 { - community[i] = c - } else { - v, err := strconv.Atoi(c) - if err != nil { - return nil, err - } - community[i] = v - } + community[i] = c } return &SearchFilter{ diff --git a/backend/api/search_filters_parsers_test.go b/backend/api/search_filters_parsers_test.go index 1cd2e11..25595b5 100644 --- a/backend/api/search_filters_parsers_test.go +++ b/backend/api/search_filters_parsers_test.go @@ -53,9 +53,9 @@ func TestParseExtCommunityValue(t *testing.T) { com := filter.Value.(ExtCommunity) if com[0].(string) != "rt" && - com[1].(int) != 23 && - com[2].(int) != 42 { - t.Error("Expected community to be: ['rt', 23, 42] but got:", com) + com[1].(string) != "23" && + com[2].(string) != "42" { + t.Error("Expected community to be: ['rt', '23', '42'] but got:", com) } } diff --git a/backend/api/search_filters_test.go b/backend/api/search_filters_test.go index 57a3208..ea5dca9 100644 --- a/backend/api/search_filters_test.go +++ b/backend/api/search_filters_test.go @@ -13,7 +13,7 @@ func makeTestRoute() *Route { Community{111, 11}, }, ExtCommunities: []ExtCommunity{ - ExtCommunity{"ro", 23, 123}, + ExtCommunity{"ro", "23", "123"}, }, LargeCommunities: []Community{ Community{1000, 23, 42}, @@ -32,7 +32,7 @@ func makeTestLookupRoute() *LookupRoute { Community{111, 11}, }, ExtCommunities: []ExtCommunity{ - ExtCommunity{"ro", 23, 123}, + ExtCommunity{"ro", "23", "123"}, }, LargeCommunities: []Community{ Community{1000, 23, 42}, @@ -124,9 +124,9 @@ func TestSearchFilterEqual(t *testing.T) { } // Ext. Communities - a = &SearchFilter{Value: ExtCommunity{"ro", 23, 42}} - b = &SearchFilter{Value: ExtCommunity{"ro", 23, 42}} - c = &SearchFilter{Value: ExtCommunity{"rt", 42, 23}} + a = &SearchFilter{Value: ExtCommunity{"ro", "23", "42"}} + b = &SearchFilter{Value: ExtCommunity{"ro", "23", "42"}} + c = &SearchFilter{Value: ExtCommunity{"rt", "42", "23"}} if a.Equal(b) == false { t.Error("filter[ro:23:42] == filter[ro:23:42] should be true") @@ -293,10 +293,10 @@ func TestSearchFilterCompareRoute(t *testing.T) { } // Ext. Communities - if searchFilterMatchExtCommunity(route, ExtCommunity{"ro", 23, 123}) != true { + if searchFilterMatchExtCommunity(route, ExtCommunity{"ro", "23", "123"}) != true { t.Error("Route should have community ro:23:123") } - if searchFilterMatchExtCommunity(route, ExtCommunity{"rt", 42, 111}) == true { + if searchFilterMatchExtCommunity(route, ExtCommunity{"rt", "42", "111"}) == true { t.Error("Route should not have community rt:42:111") } diff --git a/backend/caches/neighbors.go b/backend/caches/neighbors.go index 04f7223..5ed8d1a 100644 --- a/backend/caches/neighbors.go +++ b/backend/caches/neighbors.go @@ -45,5 +45,9 @@ func (self *NeighborsCache) Get() *api.NeighboursResponse { } func (self *NeighborsCache) Set(response *api.NeighboursResponse) { + if self.disabled { + return + } + self.response = response } diff --git a/backend/caches/routes.go b/backend/caches/routes.go index 598b6d2..f518e11 100644 --- a/backend/caches/routes.go +++ b/backend/caches/routes.go @@ -57,6 +57,10 @@ func (self *RoutesCache) Get(neighborId string) *api.RoutesResponse { } func (self *RoutesCache) Set(neighborId string, response *api.RoutesResponse) { + if self.disabled { + return + } + self.Lock() defer self.Unlock() @@ -70,3 +74,21 @@ func (self *RoutesCache) Set(neighborId string, response *api.RoutesResponse) { self.accessedAt[neighborId] = time.Now() self.responses[neighborId] = response } + +func (self *RoutesCache) Expire() int { + self.Lock() + defer self.Unlock() + + expiredKeys := []string{} + for key, response := range self.responses { + if response.CacheTtl() < 0 { + expiredKeys = append(expiredKeys, key) + } + } + + for _, key := range expiredKeys { + delete(self.responses, key) + } + + return len(expiredKeys) +} diff --git a/backend/config.go b/backend/config.go index 13c50dc..6db03f1 100644 --- a/backend/config.go +++ b/backend/config.go @@ -23,6 +23,12 @@ type ServerConfig struct { NeighboursStoreRefreshInterval int `ini:"neighbours_store_refresh_interval"` RoutesStoreRefreshInterval int `ini:"routes_store_refresh_interval"` Asn int `ini:"asn"` + EnableNeighborsStatusRefresh bool `ini:"enable_neighbors_status_refresh"` +} + +type HousekeepingConfig struct { + Interval int `ini:"interval"` + ForceReleaseMemory bool `ini:"force_release_memory"` } type RejectionsConfig struct { @@ -99,10 +105,11 @@ type SourceConfig struct { } type Config struct { - Server ServerConfig - Ui UiConfig - Sources []*SourceConfig - File string + Server ServerConfig + Housekeeping HousekeepingConfig + Ui UiConfig + Sources []*SourceConfig + File string } // Get source by id @@ -602,6 +609,19 @@ func getSources(config *ini.File) ([]*SourceConfig, error) { // Set backend switch backendType { case SOURCE_BIRDWATCHER: + sourceType := backendConfig.Key("type").MustString("") + peerTablePrefix := backendConfig.Key("peer_table_prefix").MustString("T") + pipeProtocolPrefix := backendConfig.Key("pipe_protocol_prefix").MustString("M") + + if sourceType != "single_table" && + sourceType != "multi_table" { + log.Fatal("Configuration error (birdwatcher source) unknown birdwatcher type:", sourceType) + } + + log.Println("Adding birdwatcher source of type", sourceType, + "with peer_table_prefix", peerTablePrefix, + "and pipe_protocol_prefix", pipeProtocolPrefix) + c := birdwatcher.Config{ Id: config.Id, Name: config.Name, @@ -610,7 +630,12 @@ func getSources(config *ini.File) ([]*SourceConfig, error) { ServerTime: "2006-01-02T15:04:05.999999999Z07:00", ServerTimeShort: "2006-01-02", ServerTimeExt: "Mon, 02 Jan 2006 15:04:05 -0700", + + Type: sourceType, + PeerTablePrefix: peerTablePrefix, + PipeProtocolPrefix: pipeProtocolPrefix, } + backendConfig.MapTo(&c) config.Birdwatcher = c @@ -664,6 +689,9 @@ func loadConfig(file string) (*Config, error) { server := ServerConfig{} parsedConfig.Section("server").MapTo(&server) + housekeeping := HousekeepingConfig{} + parsedConfig.Section("housekeeping").MapTo(&housekeeping) + // Get all sources sources, err := getSources(parsedConfig) if err != nil { @@ -677,10 +705,11 @@ func loadConfig(file string) (*Config, error) { } config := &Config{ - Server: server, - Ui: ui, - Sources: sources, - File: file, + Server: server, + Housekeeping: housekeeping, + Ui: ui, + Sources: sources, + File: file, } return config, nil diff --git a/backend/housekeeping.go b/backend/housekeeping.go new file mode 100644 index 0000000..5416c13 --- /dev/null +++ b/backend/housekeeping.go @@ -0,0 +1,34 @@ +package main + +import ( + "log" + "time" + "runtime/debug" +) + +func Housekeeping(config *Config) { + for { + if config.Housekeeping.Interval > 0 { + time.Sleep(time.Duration(config.Housekeeping.Interval) * time.Minute) + } else { + time.Sleep(5 * time.Minute) + } + + log.Println("Housekeeping started") + + // Expire the caches + log.Println("Expiring caches") + for _, source := range config.Sources { + count := source.getInstance().ExpireCaches() + log.Println("Expired", count, "entries for source", source.Name) + } + + if config.Housekeeping.ForceReleaseMemory { + // Trigger a GC and SCVG run + log.Println("Freeing memory") + debug.FreeOSMemory() + } + + } +} + diff --git a/backend/main.go b/backend/main.go index 8b7b357..5eaed14 100644 --- a/backend/main.go +++ b/backend/main.go @@ -47,6 +47,9 @@ func main() { AliceNeighboursStore.Start() } + // Start the Housekeeping + go Housekeeping(AliceConfig) + // Setup request routing router := httprouter.New() diff --git a/backend/neighbours_store.go b/backend/neighbours_store.go index e344959..f7a7b2e 100644 --- a/backend/neighbours_store.go +++ b/backend/neighbours_store.go @@ -15,10 +15,11 @@ var REGEX_MATCH_ASLOOKUP = regexp.MustCompile(`(?i)^AS(\d+)`) type NeighboursIndex map[string]*api.Neighbour type NeighboursStore struct { - neighboursMap map[string]NeighboursIndex - configMap map[string]*SourceConfig - statusMap map[string]StoreStatus - refreshInterval time.Duration + neighboursMap map[string]NeighboursIndex + configMap map[string]*SourceConfig + statusMap map[string]StoreStatus + refreshInterval time.Duration + refreshNeighborStatus bool sync.RWMutex } @@ -48,11 +49,14 @@ func NewNeighboursStore(config *Config) *NeighboursStore { refreshInterval = time.Duration(5) * time.Minute } + refreshNeighborStatus := config.Server.EnableNeighborsStatusRefresh + store := &NeighboursStore{ neighboursMap: neighboursMap, statusMap: statusMap, configMap: configMap, refreshInterval: refreshInterval, + refreshNeighborStatus: refreshNeighborStatus, } return store } @@ -165,9 +169,32 @@ func (self *NeighboursStore) GetNeighborsAt(sourceId string) api.Neighbours { neighborsIdx := self.neighboursMap[sourceId] self.RUnlock() + var neighborsStatus map[string]api.NeighbourStatus + if self.refreshNeighborStatus { + sourceConfig := self.configMap[sourceId] + source := sourceConfig.getInstance() + + neighborsStatusData, err := source.NeighboursStatus() + if err == nil { + neighborsStatus = make(map[string]api.NeighbourStatus, len(neighborsStatusData.Neighbours)) + + for _, neighbor := range neighborsStatusData.Neighbours { + neighborsStatus[neighbor.Id] = *neighbor + } + } + } + neighbors := make(api.Neighbours, 0, len(neighborsIdx)) for _, neighbor := range neighborsIdx { + if self.refreshNeighborStatus { + if _, ok := neighborsStatus[neighbor.Id]; ok { + self.Lock() + neighbor.State = neighborsStatus[neighbor.Id].State + self.Unlock() + } + } + neighbors = append(neighbors, neighbor) } diff --git a/backend/sources/birdwatcher/client.go b/backend/sources/birdwatcher/client.go index f421d41..eccc908 100644 --- a/backend/sources/birdwatcher/client.go +++ b/backend/sources/birdwatcher/client.go @@ -6,6 +6,7 @@ import ( "encoding/json" "io/ioutil" "net/http" + "time" ) type ClientResponse map[string]interface{} @@ -22,8 +23,8 @@ func NewClient(api string) *Client { } // Make API request, parse response and return map or error -func (self *Client) GetJson(endpoint string) (ClientResponse, error) { - res, err := http.Get(self.Api + endpoint) +func (self *Client) Get(client *http.Client, url string) (ClientResponse, error) { + res, err := client.Get(url) if err != nil { return ClientResponse{}, err } @@ -44,3 +45,19 @@ func (self *Client) GetJson(endpoint string) (ClientResponse, error) { return result, nil } + +// Make API request, parse response and return map or error +func (self *Client) GetJson(endpoint string) (ClientResponse, error) { + client := &http.Client{} + + return self.Get(client, self.Api + endpoint) +} + +// Make API request, parse response and return map or error +func (self *Client) GetJsonTimeout(timeout time.Duration, endpoint string) (ClientResponse, error) { + client := &http.Client{ + Timeout: timeout, + } + + return self.Get(client, self.Api + endpoint) +} diff --git a/backend/sources/birdwatcher/config.go b/backend/sources/birdwatcher/config.go index d6220d2..47bf059 100644 --- a/backend/sources/birdwatcher/config.go +++ b/backend/sources/birdwatcher/config.go @@ -11,5 +11,8 @@ type Config struct { ServerTimeExt string `ini:"servertime_ext"` ShowLastReboot bool `ini:"show_last_reboot"` - DisableNeighborSummary bool `ini:"disable_neighbor_summary"` + Type string `ini:"type"` + PeerTablePrefix string `ini:"peer_table_prefix"` + PipeProtocolPrefix string `ini:"pipe_protocol_prefix"` + NeighborsRefreshTimeout int `ini:"neighbors_refresh_timeout"` } diff --git a/backend/sources/birdwatcher/parsers.go b/backend/sources/birdwatcher/parsers.go index b42ba7b..5e8e81c 100644 --- a/backend/sources/birdwatcher/parsers.go +++ b/backend/sources/birdwatcher/parsers.go @@ -162,6 +162,16 @@ func parseNeighbours(bird ClientResponse, config Config) (api.Neighbours, error) uptime := parseRelativeServerTime(protocol["state_changed"], config) lastError := mustString(protocol["last_error"], "") + routesReceived := float64(0) + if routes != nil { + if _, ok := routes["imported"]; ok { + routesReceived = routesReceived + routes["imported"].(float64) + } + if _, ok := routes["filtered"]; ok { + routesReceived = routesReceived + routes["filtered"].(float64) + } + } + neighbour := &api.Neighbour{ Id: protocolId, @@ -170,11 +180,11 @@ func parseNeighbours(bird ClientResponse, config Config) (api.Neighbours, error) State: mustString(protocol["state"], "unknown"), Description: mustString(protocol["description"], "no description"), //TODO make these changes configurable - RoutesReceived: mustInt(routes["imported"], 0), - RoutesExported: mustInt(routes["exported"], 0), //TODO protocol_exported? - RoutesFiltered: mustInt(routes["filtered"], 0), - RoutesPreferred: mustInt(routes["preferred"], 0), - RoutesAccepted: mustInt(routes["pipe_imported"], mustInt(routes["imported"], 0)), + RoutesReceived: mustInt(routesReceived, 0), + RoutesAccepted: mustInt(routes["imported"], 0), + RoutesFiltered: mustInt(routes["filtered"], 0), + RoutesExported: mustInt(routes["exported"], 0), //TODO protocol_exported? + RoutesPreferred: mustInt(routes["preferred"], 0), Uptime: uptime, LastError: lastError, @@ -190,40 +200,29 @@ func parseNeighbours(bird ClientResponse, config Config) (api.Neighbours, error) return neighbours, nil } -// Get neighbors from summary endpoint -func parseNeighborSummary( - bird ClientResponse, config Config, -) (api.Neighbours, error) { - birdNeighbors := bird["neighbors"].([]interface{}) +// Parse neighbours response +func parseNeighboursShort(bird ClientResponse, config Config) (api.NeighboursStatus, error) { + neighbours := api.NeighboursStatus{} + protocols := bird["protocols"].(map[string]interface{}) - neighbors := make(api.Neighbours, 0, len(birdNeighbors)) + // Iterate over protocols map: + for protocolId, proto := range protocols { + protocol := proto.(map[string]interface{}) - for _, b := range birdNeighbors { - n := b.(map[string]interface{}) + uptime := parseRelativeServerTime(protocol["since"], config) - uptime := parseRelativeServerTime(n["state_changed"], config) - - // Parse neighbor from response - neighbor := &api.Neighbour{ - Id: mustString(n["id"], "unknown"), - Address: mustString(n["neighbor"], "unknown"), - Asn: mustInt(n["asn"], 0), - State: mustString(n["state"], "unknown"), - Uptime: uptime, - Description: mustString(n["description"], "unknown"), - LastError: mustString(n["last_error"], ""), - RoutesReceived: mustInt(n["routes_received"], -1), - RoutesAccepted: mustInt(n["routes_accepted"], -1), - RoutesFiltered: mustInt(n["routes_filtered"], -1), - RoutesExported: mustInt(n["routes_exported"], -1), + neighbour := &api.NeighbourStatus{ + Id: protocolId, + State: mustString(protocol["state"], "unknown"), + Since: uptime, } - neighbors = append(neighbors, neighbor) + neighbours = append(neighbours, neighbour) } - sort.Sort(neighbors) + sort.Sort(neighbours) - return neighbors, nil + return neighbours, nil } // Parse route bgp info @@ -292,8 +291,8 @@ func parseExtBgpCommunities(data interface{}) []api.ExtCommunity { } communities = append(communities, api.ExtCommunity{ cdata[0], - int(cdata[1].(float64)), - int(cdata[2].(float64)), + cdata[1], + cdata[2], }) } diff --git a/backend/sources/birdwatcher/parsers_test.go b/backend/sources/birdwatcher/parsers_test.go index fad0c80..035d3f4 100644 --- a/backend/sources/birdwatcher/parsers_test.go +++ b/backend/sources/birdwatcher/parsers_test.go @@ -92,76 +92,6 @@ func Test_NeighboursParsing(t *testing.T) { } } -func Test_NeighborSummaryParsing(t *testing.T) { - - config := Config{ - Timezone: "UTC", - ServerTimeShort: "2006-01-02 15:04:05"} // Or "" - bird, err := loadTestResponse("../../testdata/api/neighbor_summary.json") - if err != nil { - t.Error(err) - return - } - - neighbors, err := parseNeighborSummary(bird, config) - if err != nil { - t.Error(err) - } - - if len(neighbors) != 2 { - t.Error("There should be two neighbors in the test set, got:", - len(neighbors)) - } - - // Check first, Expected sorted by ASN ascending, ASN 23 should be 1st. - n := neighbors[0] - if n.Asn != 23 { - t.Error("Expected first ASN to be 23, got:", n.Asn) - } - - if n.Id != "R002a_0_1" { - t.Error("Expected ID R002a_0_1, got:", n.Id) - } - - if n.State != "start" { - t.Error("Unexpected state:", n.State) - } - - if n.Description != "Test Peer 2000" { - t.Error("Unexpected description:", n.Description) - } - - // Uptime is relative to the last_change timestamp, - // so the value is shifting. Calculate the expected value first: - lastChange := time.Date(2018, 7, 14, 15, 8, 30, 0, time.UTC) - expectedUptime := time.Since(lastChange) - uptimeDiff := expectedUptime - n.Uptime - - if uptimeDiff > 1*time.Second { - t.Error( - "Unexpected uptime:", n.Uptime, - "diverges more than 1 s from expected value", - ) - } - - if n.RoutesReceived != 154 { - t.Error("Unexpected routes received:", n.RoutesReceived) - } - - if n.RoutesAccepted != 152 { - t.Error("Unexpected routes accepted:", n.RoutesAccepted) - } - - if n.RoutesFiltered != 0 { - t.Error("Unexpected routes filtered:", n.RoutesFiltered) - } - - if n.RoutesExported != 2342 { - t.Error("Unexpected routes exported:", n.RoutesExported) - } - -} - func Test_RoutesParsing(t *testing.T) { config := Config{Timezone: "UTC"} // Or "" bird, _ := parseTestResponse(API_RESPONSE_ROUTES) diff --git a/backend/sources/birdwatcher/source.go b/backend/sources/birdwatcher/source.go index fa85bcc..c4b01af 100644 --- a/backend/sources/birdwatcher/source.go +++ b/backend/sources/birdwatcher/source.go @@ -3,16 +3,18 @@ package birdwatcher import ( "github.com/alice-lg/alice-lg/backend/api" "github.com/alice-lg/alice-lg/backend/caches" + "github.com/alice-lg/alice-lg/backend/sources" - "log" + "fmt" "sort" + "time" ) -const ( - NEIGHBOR_SUMMARY_ENDPOINT = "/neighbors/summary" -) +type Birdwatcher interface { + sources.Source +} -type Birdwatcher struct { +type GenericBirdwatcher struct { config Config client *Client @@ -21,15 +23,13 @@ type Birdwatcher struct { // Caches: Routes routesRequiredCache *caches.RoutesCache - routesReceivedCache *caches.RoutesCache - routesFilteredCache *caches.RoutesCache routesNotExportedCache *caches.RoutesCache // Mutices: routesFetchMutex *LockMap } -func NewBirdwatcher(config Config) *Birdwatcher { +func NewBirdwatcher(config Config) Birdwatcher { client := NewClient(config.Api) // Cache settings: @@ -43,48 +43,172 @@ func NewBirdwatcher(config Config) *Birdwatcher { neighborsCache := caches.NewNeighborsCache(neighborsCacheDisable) routesRequiredCache := caches.NewRoutesCache( routesCacheDisabled, routesCacheMaxSize) - routesReceivedCache := caches.NewRoutesCache( - routesCacheDisabled, routesCacheMaxSize) - routesFilteredCache := caches.NewRoutesCache( - routesCacheDisabled, routesCacheMaxSize) routesNotExportedCache := caches.NewRoutesCache( routesCacheDisabled, routesCacheMaxSize) - // Check if we have a neighbor summary endpoint: - if config.DisableNeighborSummary { - log.Println( - "Config override:", - "Disabled neighbor summary on", config.Name, - ) + var birdwatcher Birdwatcher + + if config.Type == "single_table" { + singleTableBirdwatcher := new(SingleTableBirdwatcher) + + singleTableBirdwatcher.config = config + singleTableBirdwatcher.client = client + + singleTableBirdwatcher.neighborsCache = neighborsCache + + singleTableBirdwatcher.routesRequiredCache = routesRequiredCache + singleTableBirdwatcher.routesNotExportedCache = routesNotExportedCache + + singleTableBirdwatcher.routesFetchMutex = NewLockMap() + + birdwatcher = singleTableBirdwatcher + } else if config.Type == "multi_table" { + multiTableBirdwatcher := new(MultiTableBirdwatcher) + + multiTableBirdwatcher.config = config + multiTableBirdwatcher.client = client + + multiTableBirdwatcher.neighborsCache = neighborsCache + + multiTableBirdwatcher.routesRequiredCache = routesRequiredCache + multiTableBirdwatcher.routesNotExportedCache = routesNotExportedCache + + multiTableBirdwatcher.routesFetchMutex = NewLockMap() + + birdwatcher = multiTableBirdwatcher } - birdwatcher := &Birdwatcher{ - config: config, - client: client, - - neighborsCache: neighborsCache, - - routesRequiredCache: routesRequiredCache, - routesReceivedCache: routesReceivedCache, - routesFilteredCache: routesFilteredCache, - routesNotExportedCache: routesNotExportedCache, - - routesFetchMutex: NewLockMap(), - } return birdwatcher } -func (self *Birdwatcher) Status() (*api.StatusResponse, error) { +func (self *GenericBirdwatcher) filterProtocols(protocols map[string]interface{}, protocol string) map[string]interface{} { + response := make(map[string]interface{}) + response["protocols"] = make(map[string]interface{}) + + for protocolId, protocolData := range protocols { + if protocolData.(map[string]interface{})["bird_protocol"] == protocol { + response["protocols"].(map[string]interface{})[protocolId] = protocolData + } + } + + return response +} + +func (self *GenericBirdwatcher) filterProtocolsBgp(bird ClientResponse) map[string]interface{} { + return self.filterProtocols(bird["protocols"].(map[string]interface{}), "BGP") +} + +func (self *GenericBirdwatcher) filterProtocolsPipe(bird ClientResponse) map[string]interface{} { + return self.filterProtocols(bird["protocols"].(map[string]interface{}), "Pipe") +} + +func (self *GenericBirdwatcher) filterRoutesByPeerOrLearntFrom(routes api.Routes, peer string, learntFrom string) api.Routes { + result_routes := make(api.Routes, 0, len(routes)) + + // Choose routes with next_hop == gateway of this neighbour + for _, route := range routes { + if (route.Gateway == peer) || + (route.Gateway == learntFrom) || + (route.Details["learnt_from"] == peer) { + result_routes = append(result_routes, route) + } + } + + // Sort routes for deterministic ordering + sort.Sort(result_routes) + routes = result_routes + + return routes +} + +func (self *GenericBirdwatcher) filterRoutesByDuplicates(routes api.Routes, filterRoutes api.Routes) api.Routes { + result_routes := make(api.Routes, 0, len(routes)) + + routesMap := make(map[string]*api.Route) // for O(1) access + for _, route := range routes { + routesMap[route.Id] = route + } + + // Remove routes from "routes" that are contained within filterRoutes + for _, filterRoute := range filterRoutes { + if _, ok := routesMap[filterRoute.Id]; ok { + delete(routesMap, filterRoute.Id) + } + } + + for _, route := range routesMap { + result_routes = append(result_routes, route) + } + + // Sort routes for deterministic ordering + sort.Sort(result_routes) + routes = result_routes + + return routes +} + +func (self *GenericBirdwatcher) filterRoutesByNeighborId(routes api.Routes, neighborId string) api.Routes { + result_routes := make(api.Routes, 0, len(routes)) + + // Choose routes with next_hop == gateway of this neighbour + for _, route := range routes { + if route.Details["from_protocol"] == neighborId { + result_routes = append(result_routes, route) + } + } + + // Sort routes for deterministic ordering + sort.Sort(result_routes) + routes = result_routes + + return routes +} + +func (self *GenericBirdwatcher) fetchProtocolsShort() (*api.ApiStatus, map[string]interface{}, error) { + // Query birdwatcher + timeout := 2 * time.Second + if self.config.NeighborsRefreshTimeout > 0 { + timeout = time.Duration(self.config.NeighborsRefreshTimeout) * time.Second + } + bird, err := self.client.GetJsonTimeout(timeout, "/protocols/short?uncached=true") + if err != nil { + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + if _, ok := bird["protocols"]; !ok { + return nil, nil, fmt.Errorf("Failed to fetch protocols") + } + + return &apiStatus, bird, nil +} + +func (self *GenericBirdwatcher) ExpireCaches() int { + count := self.routesRequiredCache.Expire() + count += self.routesNotExportedCache.Expire() + + return count +} + +func (self *GenericBirdwatcher) Status() (*api.StatusResponse, error) { + // Query birdwatcher bird, err := self.client.GetJson("/status") if err != nil { return nil, err } + // Use api status from first request apiStatus, err := parseApiStatus(bird, self.config) if err != nil { return nil, err } + // Parse the status birdStatus, err := parseBirdwatcherStatus(bird, self.config) if err != nil { return nil, err @@ -98,405 +222,30 @@ func (self *Birdwatcher) Status() (*api.StatusResponse, error) { return response, nil } -// Get bird BGP protocols -func (self *Birdwatcher) Neighbours() (*api.NeighboursResponse, error) { - // Check if we hit the cache - response := self.neighborsCache.Get() - if response != nil { - return response, nil - } - - var err error - - if self.config.DisableNeighborSummary { - // Use classic method - response, err = self.bgpProtocolsNeighbors() - } else { - - // First try neighbors summary - response, err = self.summaryNeighbors() - if err != nil { - // Inform user that this did not work - log.Println( - "Could not use neighbors-summary endpoint.", - "If this capability was disabled intentionally, consider setting", - "`disable_neighbor_summary = true` in your `source.X.birdwatcher`", - "section.", - ) - log.Println(err) - - // Try again with classic approach - response, err = self.bgpProtocolsNeighbors() - } - } - - // Handle other errors - if err != nil { - return nil, err - } - - self.neighborsCache.Set(response) - - return response, nil -} - -// Get neighbors from neighbors summary -func (self *Birdwatcher) summaryNeighbors() (*api.NeighboursResponse, error) { +// Get live neighbor status +func (self *GenericBirdwatcher) NeighboursStatus() (*api.NeighboursStatusResponse, error) { // Query birdwatcher - bird, err := self.client.GetJson(NEIGHBOR_SUMMARY_ENDPOINT) + apiStatus, birdProtocols, err := self.fetchProtocolsShort() if err != nil { return nil, err } - apiStatus, err := parseApiStatus(bird, self.config) + // Parse the neighbors short + neighbours, err := parseNeighboursShort(birdProtocols, self.config) if err != nil { return nil, err } - neighbors, err := parseNeighborSummary(bird, self.config) - if err != nil { - return nil, err - } - - response := &api.NeighboursResponse{ - Api: apiStatus, - Neighbours: neighbors, - } - - return response, nil -} - -// Get neighbors from protocols -func (self *Birdwatcher) bgpProtocolsNeighbors() (*api.NeighboursResponse, error) { - - // Query birdwatcher - bird, err := self.client.GetJson("/protocols/bgp") - if err != nil { - return nil, err - } - - apiStatus, err := parseApiStatus(bird, self.config) - if err != nil { - return nil, err - } - - neighbours, err := parseNeighbours(bird, self.config) - if err != nil { - return nil, err - } - - response := &api.NeighboursResponse{ - Api: apiStatus, + response := &api.NeighboursStatusResponse{ + Api: *apiStatus, Neighbours: neighbours, } return response, nil // dereference for now } -// Get filtered and exported routes -func (self *Birdwatcher) Routes(neighbourId string) (*api.RoutesResponse, error) { - // Exported - bird, err := self.client.GetJson("/routes/protocol/" + neighbourId) - if err != nil { - return nil, err - } - - // Use api status from first request - apiStatus, err := parseApiStatus(bird, self.config) - if err != nil { - return nil, err - } - - imported, err := parseRoutes(bird, self.config) - if err != nil { - return nil, err - } - - gateway := "" - learnt_from := "" - if len(imported) > 0 { - // infer next_hop ip address from imported[0] - gateway = imported[0].Gateway - //TODO: change mechanism to infer gateway when state becomes available elsewhere. - learnt_from = mustString(imported[0].Details["learnt_from"], gateway) - // also take learnt_from address into account if present. - // ^ learnt_from is regularly present on routes for remote-triggered - // blackholing or on filtered routes (e.g. next_hop not in AS-Set) - } - - // Optional: Filtered - bird, _ = self.client.GetJson("/routes/filtered/" + neighbourId) - filtered, err := parseRoutes(bird, self.config) - if err != nil { - log.Println("WARNING Could not retrieve filtered routes:", err) - log.Println("Is the 'routes_filtered' module active in birdwatcher?") - } else { // we got a filtered routes response => perform routes deduplication - - result_filtered := make(api.Routes, 0, len(filtered)) - result_imported := make(api.Routes, 0, len(imported)) - - importedMap := make(map[string]*api.Route) // for O(1) access - for _, route := range imported { - importedMap[route.Id] = route - } - // choose routes with next_hop == gateway of this neighbour - for _, route := range filtered { - if (route.Gateway == gateway) || - (route.Gateway == learnt_from) || - (route.Details["learnt_from"] == gateway) { - result_filtered = append(result_filtered, route) - delete(importedMap, route.Id) // remove routes that are filtered on pipe - } else if len(imported) == 0 { // in case there are just filtered routes - result_filtered = append(result_filtered, route) - } - } - sort.Sort(result_filtered) - filtered = result_filtered - // map to slice - for _, route := range importedMap { - result_imported = append(result_imported, route) - } - sort.Sort(result_imported) - imported = result_imported - } - - // Optional: NoExport - bird, _ = self.client.GetJson("/routes/noexport/" + neighbourId) - noexport, err := parseRoutes(bird, self.config) - if err != nil { - log.Println("WARNING Could not retrieve routes not exported:", err) - log.Println("Is the 'routes_noexport' module active in birdwatcher?") - } else { - result_noexport := make(api.Routes, 0, len(noexport)) - // choose routes with next_hop == gateway of this neighbour - for _, route := range noexport { - if (route.Gateway == gateway) || (route.Gateway == learnt_from) { - result_noexport = append(result_noexport, route) - } else if len(imported) == 0 { // in case there are just filtered routes - result_noexport = append(result_noexport, route) - } - } - } - - response := &api.RoutesResponse{ - Api: apiStatus, - Imported: imported, - Filtered: filtered, - NotExported: noexport, - } - - return response, nil -} - -/* -RoutesRequired is a specialized request to fetch: - - - RoutesExported and - - RoutesFiltered - -from Birdwatcher. As the not exported routes can be very many -these are optional and can be loaded on demand using the -RoutesNotExported() API. - -A route deduplication is applied. -*/ - -func (self *Birdwatcher) RoutesRequired( - neighborId string, -) (*api.RoutesResponse, error) { - // Allow only one concurrent request for this neighbor - // to our backend server. - self.routesFetchMutex.Lock(neighborId) - defer self.routesFetchMutex.Unlock(neighborId) - - // Check if we have a cache hit - response := self.routesRequiredCache.Get(neighborId) - if response != nil { - return response, nil - } - - // First: get routes received - bird, err := self.client.GetJson("/routes/protocol/" + neighborId) - if err != nil { - return nil, err - } - - // Use api status from first request - apiStatus, err := parseApiStatus(bird, self.config) - if err != nil { - return nil, err - } - - imported, err := parseRoutes(bird, self.config) - if err != nil { - return nil, err - } - - // Second: get routes filtered - bird, _ = self.client.GetJson("/routes/filtered/" + neighborId) - filtered, err := parseRoutes(bird, self.config) - if err != nil { - log.Println("WARNING Could not retrieve filtered routes:", err) - log.Println("Is the 'routes_filtered' module active in birdwatcher?") - - filtered = api.Routes{} - } - - // Perform route deduplication - importedMap := make(map[string]*api.Route) - resultFiltered := make(api.Routes, 0, len(filtered)) - resultImported := make(api.Routes, 0, len(imported)) - - gateway := "" - learnt_from := "" - if len(imported) > 0 { - // infer next_hop ip address from imported[0] - //TODO: change mechanism to infer gateway when state becomes - // available elsewhere. - gateway = imported[0].Gateway - learnt_from = mustString(imported[0].Details["learnt_from"], gateway) - // also take learnt_from address into account if present. - // ^ learnt_from is regularly present on routes for - // remote-triggered blackholing or on filtered routes - // (e.g. next_hop not in AS-Set) - } - - // Add routes to map - for _, route := range imported { - importedMap[route.Id] = route - } - - // Choose routes with next_hop == gateway of this neighbour - for _, route := range filtered { - if (route.Gateway == gateway) || - (route.Gateway == learnt_from) || - (route.Details["learnt_from"] == gateway) { - - resultFiltered = append(resultFiltered, route) - delete(importedMap, route.Id) // remove routes that are filtered on pipe - } else if len(imported) == 0 { // in case there are just filtered routes - resultFiltered = append(resultFiltered, route) - } - } - - // Map to slice - for _, route := range importedMap { - resultImported = append(resultImported, route) - } - - // Sort routes for deterministic ordering - sort.Sort(resultImported) - sort.Sort(resultFiltered) - - // Make response - response = &api.RoutesResponse{ - Api: apiStatus, - Imported: resultImported, - Filtered: resultFiltered, - } - - // Cache result - self.routesRequiredCache.Set(neighborId, response) - - return response, nil -} - -// Get all received routes -func (self *Birdwatcher) RoutesReceived( - neighborId string, -) (*api.RoutesResponse, error) { - // Check if we have a cache hit - response := self.routesReceivedCache.Get(neighborId) - if response != nil { - return response, nil - } - - // Routes received: Use RoutesRequired() api to apply routes deduplication - // However: Store in separate cache for faster access - routes, err := self.RoutesRequired(neighborId) - if err != nil { - return nil, err - } - - response = &api.RoutesResponse{ - Api: routes.Api, - Imported: routes.Imported, - } - self.routesReceivedCache.Set(neighborId, response) - - return response, nil -} - -// Get all filtered routes -func (self *Birdwatcher) RoutesFiltered( - neighborId string, -) (*api.RoutesResponse, error) { - // Check if we have a cache hit - response := self.routesFilteredCache.Get(neighborId) - if response != nil { - return response, nil - } - - // Routes filtered. Do the same thing as with routes recieved. - routes, err := self.RoutesRequired(neighborId) - if err != nil { - return nil, err - } - - response = &api.RoutesResponse{ - Api: routes.Api, - Filtered: routes.Filtered, - } - - self.routesFilteredCache.Set(neighborId, response) - - return response, nil -} - -// Get all not exported routes -func (self *Birdwatcher) RoutesNotExported( - neighborId string, -) (*api.RoutesResponse, error) { - // Check if we have a cache hit - response := self.routesNotExportedCache.Get(neighborId) - if response != nil { - return response, nil - } - - // Routes received - bird, err := self.client.GetJson("/routes/noexport/" + neighborId) - if err != nil { - log.Println("WARNING Could not retrieve routes not exported:", err) - log.Println("Is the 'routes_noexport' module active in birdwatcher?") - - return nil, err - } - - // Use api status from first request - apiStatus, err := parseApiStatus(bird, self.config) - if err != nil { - return nil, err - } - - routes, err := parseRoutes(bird, self.config) - if err != nil { - return nil, err - } - - response = &api.RoutesResponse{ - Api: apiStatus, - Imported: nil, - Filtered: nil, - NotExported: routes, - } - - self.routesNotExportedCache.Set(neighborId, response) - - return response, nil -} - // Make routes lookup -func (self *Birdwatcher) LookupPrefix(prefix string) (*api.RoutesLookupResponse, error) { +func (self *GenericBirdwatcher) LookupPrefix(prefix string) (*api.RoutesLookupResponse, error) { // Get RS info rs := api.Routeserver{ Id: self.config.Id, @@ -550,12 +299,3 @@ func (self *Birdwatcher) LookupPrefix(prefix string) (*api.RoutesLookupResponse, } return response, nil } - -func (self *Birdwatcher) AllRoutes() (*api.RoutesResponse, error) { - bird, err := self.client.GetJson("/routes/dump") - if err != nil { - return nil, err - } - result, err := parseRoutesDump(bird, self.config) - return result, err -} diff --git a/backend/sources/birdwatcher/source_multitable.go b/backend/sources/birdwatcher/source_multitable.go new file mode 100644 index 0000000..b2b435e --- /dev/null +++ b/backend/sources/birdwatcher/source_multitable.go @@ -0,0 +1,527 @@ +package birdwatcher + +import ( + "github.com/alice-lg/alice-lg/backend/api" + + "strings" + + "fmt" + "sort" + "log" +) + + +type MultiTableBirdwatcher struct { + GenericBirdwatcher +} + + +func (self *MultiTableBirdwatcher) getMasterPipeName(table string) string { + if strings.HasPrefix(table, self.config.PeerTablePrefix) { + return self.config.PipeProtocolPrefix + table[1:] + } else { + return "" + } +} + +func (self *MultiTableBirdwatcher) parseProtocolToTableTree(bird ClientResponse) map[string]interface{} { + protocols := bird["protocols"].(map[string]interface{}) + + response := make(map[string]interface{}) + + for _, protocolData := range protocols { + protocol := protocolData.(map[string]interface{}) + + if protocol["bird_protocol"] == "BGP" { + table := protocol["table"].(string) + neighborAddress := protocol["neighbor_address"].(string) + + if _, ok := response[table]; !ok { + response[table] = make(map[string]interface{}) + } + + if _, ok := response[table].(map[string]interface{})[neighborAddress]; !ok { + response[table].(map[string]interface{})[neighborAddress] = make(map[string]interface{}) + } + + response[table].(map[string]interface{})[neighborAddress] = protocol + } + } + + return response +} + + +func (self *MultiTableBirdwatcher) fetchProtocols() (*api.ApiStatus, map[string]interface{}, error) { + // Query birdwatcher + bird, err := self.client.GetJson("/protocols") + if err != nil { + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + if _, ok := bird["protocols"]; !ok { + return nil, nil, fmt.Errorf("Failed to fetch protocols") + } + + return &apiStatus, bird, nil +} + +func (self *MultiTableBirdwatcher) fetchReceivedRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + _, birdProtocols, err := self.fetchProtocols() + if err != nil { + return nil, nil, err + } + + protocols := birdProtocols["protocols"].(map[string]interface{}) + + if _, ok := protocols[neighborId]; !ok { + return nil, nil, fmt.Errorf("Invalid Neighbor") + } + + peer := protocols[neighborId].(map[string]interface{})["neighbor_address"].(string) + + // Query birdwatcher + bird, err := self.client.GetJson("/routes/peer/" + peer) + if err != nil { + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + // Parse the routes + received, err := parseRoutes(bird, self.config) + if err != nil { + log.Println("WARNING Could not retrieve received routes:", err) + log.Println("Is the 'routes_peer' module active in birdwatcher?") + return &apiStatus, nil, err + } + + return &apiStatus, received, nil +} + +func (self *MultiTableBirdwatcher) fetchFilteredRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + _, birdProtocols, err := self.fetchProtocols() + if err != nil { + return nil, nil, err + } + + protocols := birdProtocols["protocols"].(map[string]interface{}) + + if _, ok := protocols[neighborId]; !ok { + return nil, nil, fmt.Errorf("Invalid Neighbor") + } + + // Stage 1 filters + birdFiltered, err := self.client.GetJson("/routes/filtered/" + neighborId) + if err != nil { + log.Println("WARNING Could not retrieve filtered routes:", err) + log.Println("Is the 'routes_filtered' module active in birdwatcher?") + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(birdFiltered, self.config) + if err != nil { + return nil, nil, err + } + + // Parse the routes + filtered := parseRoutesData(birdFiltered["routes"].([]interface{}), self.config) + + // Stage 2 filters + table := protocols[neighborId].(map[string]interface{})["table"].(string) + pipeName := self.getMasterPipeName(table) + + // If there is no pipe to master, there is nothing left to do + if pipeName == "" { + return &apiStatus, filtered, nil + } + + // Query birdwatcher + birdPipeFiltered, err := self.client.GetJson("/routes/pipe/filtered/?table=" + table + "&pipe=" + pipeName) + if err != nil { + log.Println("WARNING Could not retrieve filtered routes:", err) + log.Println("Is the 'pipe_filtered' module active in birdwatcher?") + return &apiStatus, nil, err + } + + // Parse the routes + pipeFiltered := parseRoutesData(birdPipeFiltered["routes"].([]interface{}), self.config) + + // Sort routes for deterministic ordering + filtered = append(filtered, pipeFiltered...) + sort.Sort(filtered) + + return &apiStatus, filtered, nil +} + +func (self *MultiTableBirdwatcher) fetchNotExportedRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + _, birdProtocols, err := self.fetchProtocols() + if err != nil { + return nil, nil, err + } + + protocols := birdProtocols["protocols"].(map[string]interface{}) + + if _, ok := protocols[neighborId]; !ok { + return nil, nil, fmt.Errorf("Invalid Neighbor") + } + + table := protocols[neighborId].(map[string]interface{})["table"].(string) + pipeName := self.getMasterPipeName(table) + + // Query birdwatcher + bird, err := self.client.GetJson("/routes/noexport/" + pipeName) + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + notExported, err := parseRoutes(bird, self.config) + if err != nil { + log.Println("WARNING Could not retrieve routes not exported:", err) + log.Println("Is the 'routes_noexport' module active in birdwatcher?") + } + + return &apiStatus, notExported, nil +} + +/* +RoutesRequired is a specialized request to fetch: + +- RoutesExported and +- RoutesFiltered + +from Birdwatcher. As the not exported routes can be very many +these are optional and can be loaded on demand using the +RoutesNotExported() API. + +A route deduplication is applied. +*/ +func (self *MultiTableBirdwatcher) fetchRequiredRoutes(neighborId string) (*api.RoutesResponse, error) { + // Allow only one concurrent request for this neighbor + // to our backend server. + self.routesFetchMutex.Lock(neighborId) + defer self.routesFetchMutex.Unlock(neighborId) + + // Check if we have a cache hit + response := self.routesRequiredCache.Get(neighborId) + if response != nil { + return response, nil + } + + // First: get routes received + apiStatus, receivedRoutes, err := self.fetchReceivedRoutes(neighborId) + if err != nil { + return nil, err + } + + // Second: get routes filtered + _, filteredRoutes, err := self.fetchFilteredRoutes(neighborId) + if err != nil { + return nil, err + } + + // Perform route deduplication + importedRoutes := api.Routes{} + if len(receivedRoutes) > 0 { + peer := receivedRoutes[0].Gateway + learntFrom := mustString(receivedRoutes[0].Details["learnt_from"], peer) + + filteredRoutes = self.filterRoutesByPeerOrLearntFrom(filteredRoutes, peer, learntFrom) + importedRoutes = self.filterRoutesByDuplicates(receivedRoutes, filteredRoutes) + } + + response = &api.RoutesResponse{ + Api: *apiStatus, + Imported: importedRoutes, + Filtered: filteredRoutes, + } + + // Cache result + self.routesRequiredCache.Set(neighborId, response) + + return response, nil +} + + +// Get neighbors from protocols +func (self *MultiTableBirdwatcher) Neighbours() (*api.NeighboursResponse, error) { + // Check if we hit the cache + response := self.neighborsCache.Get() + if response != nil { + return response, nil + } + + // Query birdwatcher + apiStatus, birdProtocols, err := self.fetchProtocols() + if err != nil { + return nil, err + } + + // Parse the neighbors + neighbours, err := parseNeighbours(self.filterProtocolsBgp(birdProtocols), self.config) + if err != nil { + return nil, err + } + + pipes := self.filterProtocolsPipe(birdProtocols)["protocols"].(map[string]interface{}) + tree := self.parseProtocolToTableTree(birdProtocols) + + // Now determine the session count for each neighbor and check if the pipe + // did filter anything + filtered := make(map[string]int) + for table, _ := range tree { + allRoutesImported := int64(0) + pipeRoutesImported := int64(0) + + // Sum up all routes from all peers for a table + for _, protocol := range tree[table].(map[string]interface{}) { + // Skip peers that are not up (start/down) + if protocol.(map[string]interface{})["state"].(string) != "up" { + continue + } + allRoutesImported += int64(protocol.(map[string]interface{})["routes"].(map[string]interface{})["imported"].(float64)) + + pipeName := self.getMasterPipeName(table) + + if _, ok := pipes[pipeName]; ok { + if _, ok := pipes[pipeName].(map[string]interface{})["routes"].(map[string]interface{})["imported"]; ok { + pipeRoutesImported = int64(pipes[pipeName].(map[string]interface{})["routes"].(map[string]interface{})["imported"].(float64)) + } else { + continue + } + } else { + continue + } + } + + // If no routes were imported, there is nothing left to filter + if allRoutesImported == 0 { + continue + } + + // If the pipe did not filter anything, there is nothing left to do + if pipeRoutesImported == allRoutesImported { + continue + } + + if len(tree[table].(map[string]interface{})) == 1 { + // Single router + for _, protocol := range tree[table].(map[string]interface{}) { + filtered[protocol.(map[string]interface{})["protocol"].(string)] = int(allRoutesImported-pipeRoutesImported) + } + } else { + // Multiple routers + if pipeRoutesImported == 0 { + // 0 is a special condition, which means that the pipe did filter ALL routes of + // all peers. Therefore we already know the amount of filtered routes and don't have + // to query birdwatcher again. + for _, protocol := range tree[table].(map[string]interface{}) { + // Skip peers that are not up (start/down) + if protocol.(map[string]interface{})["state"].(string) != "up" { + continue + } + filtered[protocol.(map[string]interface{})["protocol"].(string)] = int(protocol.(map[string]interface{})["routes"].(map[string]interface{})["imported"].(float64)) + } + } else { + // Otherwise the pipe did import at least some routes which means that + // we have to query birdwatcher to get the count for each peer. + for neighborAddress, protocol := range tree[table].(map[string]interface{}) { + table := protocol.(map[string]interface{})["table"].(string) + pipe := self.getMasterPipeName(table) + + count, err := self.client.GetJson("/routes/pipe/filtered/count?table=" + table + "&pipe=" + pipe + "&address=" + neighborAddress) + if err != nil { + log.Println("WARNING Could not retrieve filtered routes count:", err) + log.Println("Is the 'pipe_filtered_count' module active in birdwatcher?") + return nil, err + } + + if _, ok := count["routes"]; ok { + filtered[protocol.(map[string]interface{})["protocol"].(string)] = int(count["routes"].(float64)) + } + } + } + } + } + + // Update the results with the information about filtered routes from the pipe + for _, neighbor := range neighbours { + if pipeRoutesFiltered, ok := filtered[neighbor.Id]; ok { + neighbor.RoutesAccepted -= pipeRoutesFiltered + neighbor.RoutesFiltered += pipeRoutesFiltered + } + } + + response = &api.NeighboursResponse{ + Api: *apiStatus, + Neighbours: neighbours, + } + + // Cache result + self.neighborsCache.Set(response) + + return response, nil // dereference for now +} + +// Get filtered and exported routes +func (self *MultiTableBirdwatcher) Routes(neighbourId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + // Fetch required routes first (received and filtered) + // However: Store in separate cache for faster access + required, err := self.fetchRequiredRoutes(neighbourId) + if err != nil { + return nil, err + } + + // Optional: NoExport + _, notExported, err := self.fetchNotExportedRoutes(neighbourId) + if err != nil { + return nil, err + } + + response.Api = required.Api + response.Imported = required.Imported + response.Filtered = required.Filtered + response.NotExported = notExported + + return response, nil +} + +// Get all received routes +func (self *MultiTableBirdwatcher) RoutesReceived(neighborId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + + // Check if we have a cache hit + cachedRoutes := self.routesRequiredCache.Get(neighborId) + if cachedRoutes != nil { + response.Api = cachedRoutes.Api + response.Imported = cachedRoutes.Imported + return response, nil + } + + // Fetch required routes first (received and filtered) + routes, err := self.fetchRequiredRoutes(neighborId) + if err != nil { + return nil, err + } + + response.Api = routes.Api + response.Imported = routes.Imported + + return response, nil +} + +// Get all filtered routes +func (self *MultiTableBirdwatcher) RoutesFiltered(neighborId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + + // Check if we have a cache hit + cachedRoutes := self.routesRequiredCache.Get(neighborId) + if cachedRoutes != nil { + response.Api = cachedRoutes.Api + response.Filtered = cachedRoutes.Filtered + return response, nil + } + + // Fetch required routes first (received and filtered) + routes, err := self.fetchRequiredRoutes(neighborId) + if err != nil { + return nil, err + } + + response.Api = routes.Api + response.Filtered = routes.Filtered + + return response, nil +} + +// Get all not exported routes +func (self *MultiTableBirdwatcher) RoutesNotExported(neighborId string) (*api.RoutesResponse, error) { + // Check if we have a cache hit + response := self.routesNotExportedCache.Get(neighborId) + if response != nil { + return response, nil + } + + // Fetch not exported routes + apiStatus, routes, err := self.fetchNotExportedRoutes(neighborId) + if err != nil { + return nil, err + } + + response = &api.RoutesResponse{ + Api: *apiStatus, + NotExported: routes, + } + + // Cache result + self.routesNotExportedCache.Set(neighborId, response) + + return response, nil +} + +func (self *MultiTableBirdwatcher) AllRoutes() (*api.RoutesResponse, error) { + // Query birdwatcher + _, birdProtocols, err := self.fetchProtocols() + if err != nil { + return nil, err + } + + // Fetch received routes first + birdImported, err := self.client.GetJson("/routes/table/master") + if err != nil { + return nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(birdImported, self.config) + if err != nil { + return nil, err + } + + response := &api.RoutesResponse{ + Api: apiStatus, + } + + // Parse the routes + imported := parseRoutesData(birdImported["routes"].([]interface{}), self.config) + // Sort routes for deterministic ordering + sort.Sort(imported) + response.Imported = imported + + // Iterate over all the protocols and fetch the filtered routes for everyone + protocolsBgp := self.filterProtocolsBgp(birdProtocols) + for protocolId, protocolsData := range protocolsBgp["protocols"].(map[string]interface{}) { + peer := protocolsData.(map[string]interface{})["neighbor_address"].(string) + learntFrom := mustString(protocolsData.(map[string]interface{})["learnt_from"], peer) + + // Fetch filtered routes + _, filtered, err := self.fetchFilteredRoutes(protocolId) + if err != nil { + continue + } + + // Perform route deduplication + filtered = self.filterRoutesByPeerOrLearntFrom(filtered, peer, learntFrom) + response.Filtered = append(response.Filtered, filtered...) + } + + return response, nil +} diff --git a/backend/sources/birdwatcher/source_singletable.go b/backend/sources/birdwatcher/source_singletable.go new file mode 100644 index 0000000..631b828 --- /dev/null +++ b/backend/sources/birdwatcher/source_singletable.go @@ -0,0 +1,315 @@ +package birdwatcher + +import ( + "github.com/alice-lg/alice-lg/backend/api" + + "log" + "sort" +) + + +type SingleTableBirdwatcher struct { + GenericBirdwatcher +} + + +func (self *SingleTableBirdwatcher) fetchReceivedRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + bird, err := self.client.GetJson("/routes/protocol/" + neighborId) + if err != nil { + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + // Parse the routes + received, err := parseRoutes(bird, self.config) + if err != nil { + log.Println("WARNING Could not retrieve received routes:", err) + log.Println("Is the 'routes_protocol' module active in birdwatcher?") + return &apiStatus, nil, err + } + + return &apiStatus, received, nil +} + +func (self *SingleTableBirdwatcher) fetchFilteredRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + bird, err := self.client.GetJson("/routes/filtered/" + neighborId) + if err != nil { + return nil, nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + // Parse the routes + filtered, err := parseRoutes(bird, self.config) + if err != nil { + log.Println("WARNING Could not retrieve filtered routes:", err) + log.Println("Is the 'routes_filtered' module active in birdwatcher?") + return &apiStatus, nil, err + } + + return &apiStatus, filtered, nil +} + +func (self *SingleTableBirdwatcher) fetchNotExportedRoutes(neighborId string) (*api.ApiStatus, api.Routes, error) { + // Query birdwatcher + bird, err := self.client.GetJson("/routes/noexport/" + neighborId) + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, nil, err + } + + // Parse the routes + notExported, err := parseRoutes(bird, self.config) + if err != nil { + log.Println("WARNING Could not retrieve routes not exported:", err) + log.Println("Is the 'routes_noexport' module active in birdwatcher?") + } + + return &apiStatus, notExported, nil +} + +/* +RoutesRequired is a specialized request to fetch: + +- RoutesExported and +- RoutesFiltered + +from Birdwatcher. As the not exported routes can be very many +these are optional and can be loaded on demand using the +RoutesNotExported() API. + +A route deduplication is applied. +*/ +func (self *SingleTableBirdwatcher) fetchRequiredRoutes(neighborId string) (*api.RoutesResponse, error) { + // Allow only one concurrent request for this neighbor + // to our backend server. + self.routesFetchMutex.Lock(neighborId) + defer self.routesFetchMutex.Unlock(neighborId) + + // Check if we have a cache hit + response := self.routesRequiredCache.Get(neighborId) + if response != nil { + return response, nil + } + + // First: get routes received + apiStatus, receivedRoutes, err := self.fetchReceivedRoutes(neighborId) + if err != nil { + return nil, err + } + + // Second: get routes filtered + _, filteredRoutes, err := self.fetchFilteredRoutes(neighborId) + if err != nil { + return nil, err + } + + // Perform route deduplication + importedRoutes := api.Routes{} + if len(receivedRoutes) > 0 { + peer := receivedRoutes[0].Gateway + learntFrom := mustString(receivedRoutes[0].Details["learnt_from"], peer) + + filteredRoutes = self.filterRoutesByPeerOrLearntFrom(filteredRoutes, peer, learntFrom) + importedRoutes = self.filterRoutesByDuplicates(receivedRoutes, filteredRoutes) + } + + response = &api.RoutesResponse{ + Api: *apiStatus, + Imported: importedRoutes, + Filtered: filteredRoutes, + } + + // Cache result + self.routesRequiredCache.Set(neighborId, response) + + return response, nil +} + + +// Get neighbors from protocols +func (self *SingleTableBirdwatcher) Neighbours() (*api.NeighboursResponse, error) { + // Check if we hit the cache + response := self.neighborsCache.Get() + if response != nil { + return response, nil + } + + // Query birdwatcher + bird, err := self.client.GetJson("/protocols/bgp") + if err != nil { + return nil, err + } + + // Use api status from first request + apiStatus, err := parseApiStatus(bird, self.config) + if err != nil { + return nil, err + } + + // Parse the neighbors + neighbours, err := parseNeighbours(bird, self.config) + if err != nil { + return nil, err + } + + response = &api.NeighboursResponse{ + Api: apiStatus, + Neighbours: neighbours, + } + + // Cache result + self.neighborsCache.Set(response) + + return response, nil // dereference for now +} + +// Get filtered and exported routes +func (self *SingleTableBirdwatcher) Routes(neighbourId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + + // Fetch required routes first (received and filtered) + required, err := self.fetchRequiredRoutes(neighbourId) + if err != nil { + return nil, err + } + + // Optional: NoExport + _, notExported, err := self.fetchNotExportedRoutes(neighbourId) + if err != nil { + return nil, err + } + + response.Api = required.Api + response.Imported = required.Imported + response.Filtered = required.Filtered + response.NotExported = notExported + + return response, nil +} + +// Get all received routes +func (self *SingleTableBirdwatcher) RoutesReceived(neighborId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + + // Check if we hit the cache + cachedRoutes := self.routesRequiredCache.Get(neighborId) + if cachedRoutes != nil { + response.Api = cachedRoutes.Api + response.Imported = cachedRoutes.Imported + return response, nil + } + + // Fetch required routes first (received and filtered) + // However: Store in separate cache for faster access + routes, err := self.fetchRequiredRoutes(neighborId) + if err != nil { + return nil, err + } + + response.Api = routes.Api + response.Imported = routes.Imported + + return response, nil +} + +// Get all filtered routes +func (self *SingleTableBirdwatcher) RoutesFiltered(neighborId string) (*api.RoutesResponse, error) { + response := &api.RoutesResponse{} + + // Check if we hit the cache + cachedRoutes := self.routesRequiredCache.Get(neighborId) + if cachedRoutes != nil { + response.Api = cachedRoutes.Api + response.Filtered = cachedRoutes.Filtered + return response, nil + } + + // Fetch required routes first (received and filtered) + // However: Store in separate cache for faster access + routes, err := self.fetchRequiredRoutes(neighborId) + if err != nil { + return nil, err + } + + response.Api = routes.Api + response.Filtered = routes.Filtered + + return response, nil +} + +// Get all not exported routes +func (self *SingleTableBirdwatcher) RoutesNotExported(neighborId string) (*api.RoutesResponse, error) { + // Check if we hit the cache + response := self.routesNotExportedCache.Get(neighborId) + if response != nil { + return response, nil + } + + // Fetch not exported routes + apiStatus, routes, err := self.fetchNotExportedRoutes(neighborId) + if err != nil { + return nil, err + } + + response = &api.RoutesResponse{ + Api: *apiStatus, + NotExported: routes, + } + + // Cache result + self.routesNotExportedCache.Set(neighborId, response) + + return response, nil +} + +func (self *SingleTableBirdwatcher) AllRoutes() (*api.RoutesResponse, error) { + // First fetch all routes from the master table + birdImported, err := self.client.GetJson("/routes/table/master") + if err != nil { + return nil, err + } + + // Then fetch all filtered routes from the master table + birdFiltered, err := self.client.GetJson("/routes/table/master/filtered") + if err != nil { + return nil, err + } + + // Use api status from second request + apiStatus, err := parseApiStatus(birdFiltered, self.config) + if err != nil { + return nil, err + } + + response := &api.RoutesResponse{ + Api: apiStatus, + } + + // Parse the routes + imported := parseRoutesData(birdImported["routes"].([]interface{}), self.config) + // Sort routes for deterministic ordering + sort.Sort(imported) + response.Imported = imported + + // Parse the routes + filtered := parseRoutesData(birdFiltered["routes"].([]interface{}), self.config) + // Sort routes for deterministic ordering + sort.Sort(filtered) + response.Filtered = filtered + + return response, nil +} diff --git a/backend/sources/source.go b/backend/sources/source.go index 822ef4e..165917c 100644 --- a/backend/sources/source.go +++ b/backend/sources/source.go @@ -5,8 +5,10 @@ import ( ) type Source interface { + ExpireCaches() int Status() (*api.StatusResponse, error) Neighbours() (*api.NeighboursResponse, error) + NeighboursStatus() (*api.NeighboursStatusResponse, error) Routes(neighbourId string) (*api.RoutesResponse, error) RoutesReceived(neighbourId string) (*api.RoutesResponse, error) RoutesFiltered(neighbourId string) (*api.RoutesResponse, error) diff --git a/client/components/filters/encoding.jsx b/client/components/filters/encoding.jsx index 6762b32..28c84df 100644 --- a/client/components/filters/encoding.jsx +++ b/client/components/filters/encoding.jsx @@ -38,8 +38,7 @@ function _decodeCommunity(community) { } function _decodeExtCommunity(community) { - const parts = community.split(":"); - return [parts[0]].concat(parts.slice(1).map((p) => parseInt(p, 10))); + return community.split(":"); } export function decodeFiltersCommunities(params) { diff --git a/etc/alice-lg/alice.example.conf b/etc/alice-lg/alice.example.conf index 5e67438..7e90231 100644 --- a/etc/alice-lg/alice.example.conf +++ b/etc/alice-lg/alice.example.conf @@ -5,12 +5,20 @@ [server] # configures the built-in webserver and provides global application settings listen_http = 127.0.0.1:7340 -enable_prefix_lookup = true # enable the prefix-lookup endpoint / the global search feature +enable_prefix_lookup = true +# Try to refresh the neighbor status on every request to /neighbors +enable_neighbors_status_refresh = false asn = 9033 # this ASN is used as a fallback value in the RPKI feature and for route # filtering evaluation with large BGP communities +[housekeeping] +# Interval for the housekeeping routine in minutes +interval = 5 +# Try to release memory via a forced GC/SCVG run on every housekeeping run +force_release_memory = true + [theme] path = /path/to/my/alice/theme/files # Optional: @@ -140,6 +148,12 @@ blackholes = 10.23.6.666, 10.23.6.665 [source.rs0-example-v4.birdwatcher] api = http://rs1.example.com:29184/ +# single_table / multi_table +type = multi_table +peer_table_prefix = T +pipe_protocol_prefix = M +# Timeout in seconds to wait for the status data (only required if enable_neighbors_status_refresh is true) +neighbors_refresh_timeout = 2 # Optional: show_last_reboot = true @@ -150,8 +164,12 @@ name = rs1.example.com (IPv6) [source.rs1-example-v6.birdwatcher] timezone = Europe/Brussels api = http://rs1.example.com:29186/ - -# disable_neighbor_summary = true +# single_table / multi_table +type = multi_table +peer_table_prefix = T +pipe_protocol_prefix = M +# Timeout in seconds to wait for the status data (only required if enable_neighbors_status_refresh is true) +neighbors_refresh_timeout = 2 # Optional: Examples for time format # Please see https://golang.org/pkg/time/#pkg-constants for an