diff --git a/pkg/pools/communities.go b/pkg/pools/communities.go index 3486207..daa71a6 100644 --- a/pkg/pools/communities.go +++ b/pkg/pools/communities.go @@ -7,20 +7,22 @@ import ( "github.com/alice-lg/alice-lg/pkg/api" ) -// Communities is a pool for deduplicating a list of BGP communities +// CommunitiesPool is for deduplicating a list of BGP communities // (Large and default. The ext communities representation right now // makes problems and need to be fixed. TODO.) -type Communities struct { - communities *IntList - root *Node +type CommunitiesPool struct { + communitiesRoot *Node + root *Node sync.Mutex } -// NewCommunities creates a new pool for lists +// NewCommunitiesPool creates a new pool for lists // of BGP communities. -func NewCommunities() *Communities { - return &Communities{ - communities: NewIntList(), +func NewCommunitiesPool() *CommunitiesPool { + return &CommunitiesPool{ + communitiesRoot: &Node{ + ptr: []int{}, + }, root: &Node{ ptr: []api.Community{}, }, @@ -28,18 +30,19 @@ func NewCommunities() *Communities { } // Acquire a list of bgp communities -func (p *Communities) Acquire(communities []api.Community) []api.Community { +func (p *CommunitiesPool) Acquire(communities []api.Community) []api.Community { + p.Lock() + defer p.Unlock() + // Make identification list by using the pointer address + // of the deduplicated community as ID ids := make([]int, len(communities)) for i, comm := range communities { - commPtr := p.communities.Acquire(comm) + commPtr := p.communitiesRoot.traverse(comm, comm) addr := reflect.ValueOf(commPtr).UnsafePointer() ids[i] = int(uintptr(addr)) } - p.Lock() - defer p.Unlock() if len(ids) == 0 { return p.root.ptr.([]api.Community) } - return p.root.traverse(communities, ids).([]api.Community) } diff --git a/pkg/pools/communities_test.go b/pkg/pools/communities_test.go index a02239f..7d2bbbd 100644 --- a/pkg/pools/communities_test.go +++ b/pkg/pools/communities_test.go @@ -24,7 +24,7 @@ func TestAcquireCommunities(t *testing.T) { {2341, 1, 1}, } - p := NewCommunities() + p := NewCommunitiesPool() pc1 := p.Acquire(c1) pc2 := p.Acquire(c2) diff --git a/pkg/pools/lists.go b/pkg/pools/lists.go index 536da70..897b125 100644 --- a/pkg/pools/lists.go +++ b/pkg/pools/lists.go @@ -46,18 +46,18 @@ func (n *Node) traverse(list interface{}, tail []int) interface{} { return child.traverse(list, tail) } -// A IntList pool can be used to deduplicate -// lists of integers. Like an AS path. +// A IntListPool can be used to deduplicate +// lists of integers. Like an AS path or BGP communities. // // A Tree datastructure is used. -type IntList struct { +type IntListPool struct { root *Node sync.Mutex } -// NewIntList creates a new int list pool -func NewIntList() *IntList { - return &IntList{ +// NewIntListPool creates a new int list pool +func NewIntListPool() *IntListPool { + return &IntListPool{ root: &Node{ ptr: []int{}, }, @@ -65,7 +65,7 @@ func NewIntList() *IntList { } // Acquire int list from pool -func (p *IntList) Acquire(list []int) []int { +func (p *IntListPool) Acquire(list []int) []int { p.Lock() defer p.Unlock() @@ -75,19 +75,19 @@ func (p *IntList) Acquire(list []int) []int { return p.root.traverse(list, list).([]int) } -// A StringList pool can be used for deduplicating lists +// A StringListPool can be used for deduplicating lists // of strings. (This is a variant of an int list, as string // values are converted to int. -type StringList struct { +type StringListPool struct { root *Node values map[string]int head int sync.Mutex } -// NewStringList creates a new string list. -func NewStringList() *StringList { - return &StringList{ +// NewStringListPool creates a new string list. +func NewStringListPool() *StringListPool { + return &StringListPool{ head: 1, values: map[string]int{}, root: &Node{ @@ -97,7 +97,7 @@ func NewStringList() *StringList { } // Acquire the string list pointer from the pool -func (p *StringList) Acquire(list []string) []string { +func (p *StringListPool) Acquire(list []string) []string { if len(list) == 0 { return p.root.ptr.([]string) // root } diff --git a/pkg/pools/lists_test.go b/pkg/pools/lists_test.go index fda7b58..78cda5c 100644 --- a/pkg/pools/lists_test.go +++ b/pkg/pools/lists_test.go @@ -12,7 +12,7 @@ func TestAcquireIntList(t *testing.T) { b := []int{23, 42, 1337, 65535, 1} c := []int{23, 42, 1338, 65535, 2} - p := NewIntList() + p := NewIntListPool() r1 := p.Acquire(a) p.Acquire(c) @@ -49,7 +49,7 @@ func TestAcquireStringList(t *testing.T) { w := []string{"foo", "bar", "bgp"} e := []string{"foo", "bpf"} - p2 := NewStringList() + p2 := NewStringListPool() x1 := p2.Acquire(q) x2 := p2.Acquire(w) x3 := p2.Acquire(e) diff --git a/pkg/pools/pools.go b/pkg/pools/pools.go index 6ab9786..95590be 100644 --- a/pkg/pools/pools.go +++ b/pkg/pools/pools.go @@ -8,43 +8,51 @@ import "log" // and are defined per intended usage // Neighbors stores neighbor IDs -var Neighbors *String +var Neighbors *StringPool // Networks4 stores network ip v4 addresses -var Networks4 *String +var Networks4 *StringPool // Networks6 stores network ip v6 addresses -var Networks6 *String +var Networks6 *StringPool // Interfaces stores interfaces like: eth0, bond0 etc... -var Interfaces *String +var Interfaces *StringPool // Gateways4 store ip v4 gateway addresses -var Gateways4 *String +var Gateways4 *StringPool // Gateways6 store ip v6 gateway addresses -var Gateways6 *String +var Gateways6 *StringPool // Origins is a store for 'IGP' -var Origins *String +var Origins *StringPool // ASPaths stores lists of ASNs -var ASPaths *IntList +var ASPaths *IntListPool // Types stores a list of types (['BGP', 'univ']) -var Types *StringList +var Types *StringListPool + +// Communities store a list of BGP communities +var Communities *CommunitiesPool + +// LargeCommunities store a list of large BGP communities +var LargeCommunities *CommunitiesPool // Initialize global pools func init() { log.Println("initializing memory pools") - Neighbors = NewString() - Networks4 = NewString() - Networks6 = NewString() - Interfaces = NewString() - Gateways4 = NewString() - Gateways6 = NewString() - Origins = NewString() - ASPaths = NewIntList() - Types = NewStringList() + Neighbors = NewStringPool() + Networks4 = NewStringPool() + Networks6 = NewStringPool() + Interfaces = NewStringPool() + Gateways4 = NewStringPool() + Gateways6 = NewStringPool() + Origins = NewStringPool() + ASPaths = NewIntListPool() + Types = NewStringListPool() + Communities = NewCommunitiesPool() + LargeCommunities = NewCommunitiesPool() } diff --git a/pkg/pools/string.go b/pkg/pools/string.go index cbac878..4457f6b 100644 --- a/pkg/pools/string.go +++ b/pkg/pools/string.go @@ -2,9 +2,9 @@ package pools import "sync" -// String is a pool for strings. +// StringPool is a pool for strings. // This will most likely be a pool for IP addresses. -type String struct { +type StringPool struct { values map[string]*string counter map[string]uint @@ -13,16 +13,16 @@ type String struct { sync.Mutex } -// NewString creates a new string pool -func NewString() *String { - return &String{ +// NewStringPool creates a new string pool +func NewStringPool() *StringPool { + return &StringPool{ values: map[string]*string{}, counter: map[string]uint{}, } } // Acquire a pointer to a string value -func (p *String) Acquire(s string) *string { +func (p *StringPool) Acquire(s string) *string { p.Lock() defer p.Unlock() // Deduplicate value @@ -37,7 +37,7 @@ func (p *String) Acquire(s string) *string { // GarbageCollect releases all values, which have not been seen // again. -func (p *String) GarbageCollect() uint { +func (p *StringPool) GarbageCollect() uint { p.Lock() defer p.Unlock() var released uint = 0 diff --git a/pkg/pools/string_test.go b/pkg/pools/string_test.go index b404541..3b65dbd 100644 --- a/pkg/pools/string_test.go +++ b/pkg/pools/string_test.go @@ -6,7 +6,7 @@ import ( ) func TestAcquireString(t *testing.T) { - p := NewString() + p := NewStringPool() s1 := p.Acquire("hello") s2 := p.Acquire("hello") s3 := p.Acquire("world") @@ -24,7 +24,7 @@ func TestAcquireString(t *testing.T) { } func TestGarbageCollectString(t *testing.T) { - p := NewString() + p := NewStringPool() // Gen 1 p.Acquire("hello") diff --git a/pkg/sources/birdwatcher/parsers.go b/pkg/sources/birdwatcher/parsers.go index 6e3e0b4..ab2386e 100644 --- a/pkg/sources/birdwatcher/parsers.go +++ b/pkg/sources/birdwatcher/parsers.go @@ -247,6 +247,13 @@ func parseRouteBgpInfo(data interface{}) *api.BGPInfo { localPref, _ := strconv.Atoi(decoders.String(bgpData["local_pref"], "0")) med, _ := strconv.Atoi(decoders.String(bgpData["med"], "0")) + // Testing and benchmarks show: Deduplicating communities has + // quite a performance impact: + // Without using pools, parsing 600000 routes + // takes roughly 16 seconds, with pools for strings + // and AS paths: 18 seconds. + // With communities: 46 seconds. This is quite long. + bgp := &api.BGPInfo{ Origin: pools.Origins.Acquire( decoders.String(bgpData["origin"], "unknown")), @@ -255,9 +262,9 @@ func parseRouteBgpInfo(data interface{}) *api.BGPInfo { decoders.String(bgpData["next_hop"], "unknown")), LocalPref: localPref, Med: med, - Communities: communities, + Communities: pools.Communities.Acquire(communities), ExtCommunities: extCommunities, - LargeCommunities: largeCommunities, + LargeCommunities: pools.LargeCommunities.Acquire(largeCommunities), } return bgp }