use communities pools

pkg/pools/communities.go

@@ -7,20 +7,22 @@ import (
 	"github.com/alice-lg/alice-lg/pkg/api"
 )
 
-// Communities is a pool for deduplicating a list of BGP communities
+// CommunitiesPool is for deduplicating a list of BGP communities
 // (Large and default. The ext communities representation right now
 // makes problems and need to be fixed. TODO.)
-type Communities struct {
+type CommunitiesPool struct {
-	communities *IntList
+	communitiesRoot *Node
 	root            *Node
 	sync.Mutex
 }
 
-// NewCommunities creates a new pool for lists
+// NewCommunitiesPool creates a new pool for lists
 // of BGP communities.
-func NewCommunities() *Communities {
+func NewCommunitiesPool() *CommunitiesPool {
-	return &Communities{
+	return &CommunitiesPool{
-		communities: NewIntList(),
+		communitiesRoot: &Node{
+			ptr: []int{},
+		},
 		root: &Node{
 			ptr: []api.Community{},
 		},
@@ -28,18 +30,19 @@ func NewCommunities() *Communities {
 }
 
 // Acquire a list of bgp communities
-func (p *Communities) Acquire(communities []api.Community) []api.Community {
+func (p *CommunitiesPool) Acquire(communities []api.Community) []api.Community {
+	p.Lock()
+	defer p.Unlock()
+	// Make identification list by using the pointer address
+	// of the deduplicated community as ID
 	ids := make([]int, len(communities))
 	for i, comm := range communities {
-		commPtr := p.communities.Acquire(comm)
+		commPtr := p.communitiesRoot.traverse(comm, comm)
 		addr := reflect.ValueOf(commPtr).UnsafePointer()
 		ids[i] = int(uintptr(addr))
 	}
-	p.Lock()
-	defer p.Unlock()
 	if len(ids) == 0 {
 		return p.root.ptr.([]api.Community)
 	}
-
 	return p.root.traverse(communities, ids).([]api.Community)
 }

pkg/pools/communities_test.go

@@ -24,7 +24,7 @@ func TestAcquireCommunities(t *testing.T) {
 		{2341, 1, 1},
 	}
 
-	p := NewCommunities()
+	p := NewCommunitiesPool()
 
 	pc1 := p.Acquire(c1)
 	pc2 := p.Acquire(c2)

pkg/pools/lists.go

@@ -46,18 +46,18 @@ func (n *Node) traverse(list interface{}, tail []int) interface{} {
 	return child.traverse(list, tail)
 }
 
-// A IntList pool can be used to deduplicate
+// A IntListPool can be used to deduplicate
-// lists of integers. Like an AS path.
+// lists of integers. Like an AS path or BGP communities.
 //
 // A Tree datastructure is used.
-type IntList struct {
+type IntListPool struct {
 	root *Node
 	sync.Mutex
 }
 
-// NewIntList creates a new int list pool
+// NewIntListPool creates a new int list pool
-func NewIntList() *IntList {
+func NewIntListPool() *IntListPool {
-	return &IntList{
+	return &IntListPool{
 		root: &Node{
 			ptr: []int{},
 		},
@@ -65,7 +65,7 @@ func NewIntList() *IntList {
 }
 
 // Acquire int list from pool
-func (p *IntList) Acquire(list []int) []int {
+func (p *IntListPool) Acquire(list []int) []int {
 	p.Lock()
 	defer p.Unlock()
 
@@ -75,19 +75,19 @@ func (p *IntList) Acquire(list []int) []int {
 	return p.root.traverse(list, list).([]int)
 }
 
-// A StringList pool can be used for deduplicating lists
+// A StringListPool can be used for deduplicating lists
 // of strings. (This is a variant of an int list, as string
 // values are converted to int.
-type StringList struct {
+type StringListPool struct {
 	root   *Node
 	values map[string]int
 	head   int
 	sync.Mutex
 }
 
-// NewStringList creates a new string list.
+// NewStringListPool creates a new string list.
-func NewStringList() *StringList {
+func NewStringListPool() *StringListPool {
-	return &StringList{
+	return &StringListPool{
 		head:   1,
 		values: map[string]int{},
 		root: &Node{
@@ -97,7 +97,7 @@ func NewStringList() *StringList {
 }
 
 // Acquire the string list pointer from the pool
-func (p *StringList) Acquire(list []string) []string {
+func (p *StringListPool) Acquire(list []string) []string {
 	if len(list) == 0 {
 		return p.root.ptr.([]string) // root
 	}

pkg/pools/lists_test.go

@@ -12,7 +12,7 @@ func TestAcquireIntList(t *testing.T) {
 	b := []int{23, 42, 1337, 65535, 1}
 	c := []int{23, 42, 1338, 65535, 2}
 
-	p := NewIntList()
+	p := NewIntListPool()
 
 	r1 := p.Acquire(a)
 	p.Acquire(c)
@@ -49,7 +49,7 @@ func TestAcquireStringList(t *testing.T) {
 	w := []string{"foo", "bar", "bgp"}
 	e := []string{"foo", "bpf"}
 
-	p2 := NewStringList()
+	p2 := NewStringListPool()
 	x1 := p2.Acquire(q)
 	x2 := p2.Acquire(w)
 	x3 := p2.Acquire(e)

pkg/pools/pools.go

@@ -8,43 +8,51 @@ import "log"
 // and are defined per intended usage
 
 // Neighbors stores neighbor IDs
-var Neighbors *String
+var Neighbors *StringPool
 
 // Networks4 stores network ip v4 addresses
-var Networks4 *String
+var Networks4 *StringPool
 
 // Networks6 stores network ip v6 addresses
-var Networks6 *String
+var Networks6 *StringPool
 
 // Interfaces stores interfaces like: eth0, bond0 etc...
-var Interfaces *String
+var Interfaces *StringPool
 
 // Gateways4 store ip v4 gateway addresses
-var Gateways4 *String
+var Gateways4 *StringPool
 
 // Gateways6 store ip v6 gateway addresses
-var Gateways6 *String
+var Gateways6 *StringPool
 
 // Origins is a store for 'IGP'
-var Origins *String
+var Origins *StringPool
 
 // ASPaths stores lists of ASNs
-var ASPaths *IntList
+var ASPaths *IntListPool
 
 // Types stores a list of types (['BGP', 'univ'])
-var Types *StringList
+var Types *StringListPool
+
+// Communities store a list of BGP communities
+var Communities *CommunitiesPool
+
+// LargeCommunities store a list of large BGP communities
+var LargeCommunities *CommunitiesPool
 
 // Initialize global pools
 func init() {
 	log.Println("initializing memory pools")
 
-	Neighbors = NewString()
+	Neighbors = NewStringPool()
-	Networks4 = NewString()
+	Networks4 = NewStringPool()
-	Networks6 = NewString()
+	Networks6 = NewStringPool()
-	Interfaces = NewString()
+	Interfaces = NewStringPool()
-	Gateways4 = NewString()
+	Gateways4 = NewStringPool()
-	Gateways6 = NewString()
+	Gateways6 = NewStringPool()
-	Origins = NewString()
+	Origins = NewStringPool()
-	ASPaths = NewIntList()
+	ASPaths = NewIntListPool()
-	Types = NewStringList()
+	Types = NewStringListPool()
+	Communities = NewCommunitiesPool()
+	LargeCommunities = NewCommunitiesPool()
 }

pkg/pools/string.go

@@ -2,9 +2,9 @@ package pools
 
 import "sync"
 
-// String is a pool for strings.
+// StringPool is a pool for strings.
 // This will most likely be a pool for IP addresses.
-type String struct {
+type StringPool struct {
 	values map[string]*string
 
 	counter map[string]uint
@@ -13,16 +13,16 @@ type String struct {
 	sync.Mutex
 }
 
-// NewString creates a new string pool
+// NewStringPool creates a new string pool
-func NewString() *String {
+func NewStringPool() *StringPool {
-	return &String{
+	return &StringPool{
 		values:  map[string]*string{},
 		counter: map[string]uint{},
 	}
 }
 
 // Acquire a pointer to a string value
-func (p *String) Acquire(s string) *string {
+func (p *StringPool) Acquire(s string) *string {
 	p.Lock()
 	defer p.Unlock()
 	// Deduplicate value
@@ -37,7 +37,7 @@ func (p *String) Acquire(s string) *string {
 
 // GarbageCollect releases all values, which have not been seen
 // again.
-func (p *String) GarbageCollect() uint {
+func (p *StringPool) GarbageCollect() uint {
 	p.Lock()
 	defer p.Unlock()
 	var released uint = 0

pkg/pools/string_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestAcquireString(t *testing.T) {
-	p := NewString()
+	p := NewStringPool()
 	s1 := p.Acquire("hello")
 	s2 := p.Acquire("hello")
 	s3 := p.Acquire("world")
@@ -24,7 +24,7 @@ func TestAcquireString(t *testing.T) {
 }
 
 func TestGarbageCollectString(t *testing.T) {
-	p := NewString()
+	p := NewStringPool()
 
 	// Gen 1
 	p.Acquire("hello")

pkg/sources/birdwatcher/parsers.go

@@ -247,6 +247,13 @@ func parseRouteBgpInfo(data interface{}) *api.BGPInfo {
 	localPref, _ := strconv.Atoi(decoders.String(bgpData["local_pref"], "0"))
 	med, _ := strconv.Atoi(decoders.String(bgpData["med"], "0"))
 
+	// Testing and benchmarks show: Deduplicating communities has
+	// quite a performance impact:
+	// Without using pools, parsing 600000 routes
+	// takes roughly 16 seconds, with pools for strings
+	// and AS paths: 18 seconds.
+	// With communities: 46 seconds. This is quite long.
+
 	bgp := &api.BGPInfo{
 		Origin: pools.Origins.Acquire(
 			decoders.String(bgpData["origin"], "unknown")),
@@ -255,9 +262,9 @@ func parseRouteBgpInfo(data interface{}) *api.BGPInfo {
 			decoders.String(bgpData["next_hop"], "unknown")),
 		LocalPref:        localPref,
 		Med:              med,
-		Communities:      communities,
+		Communities:      pools.Communities.Acquire(communities),
 		ExtCommunities:   extCommunities,
-		LargeCommunities: largeCommunities,
+		LargeCommunities: pools.LargeCommunities.Acquire(largeCommunities),
 	}
 	return bgp
 }