for large result sets, do not calculate communities filters

pkg/api/response_routes.go

@@ -129,8 +129,9 @@ type PaginatedResponse struct {
 
 // FilteredResponse includes filters applied and available
 type FilteredResponse struct {
-	FiltersAvailable *SearchFilters `json:"filters_available"`
-	FiltersApplied   *SearchFilters `json:"filters_applied"`
+	FiltersAvailable    *SearchFilters `json:"filters_available"`
+	FiltersApplied      *SearchFilters `json:"filters_applied"`
+	FiltersNotAvailable []string       `json:"filters_not_available"`
 }
 
 const (

pkg/api/search_filters.go

@@ -120,6 +120,8 @@ func (f *SearchFilter) Equal(other *SearchFilter) bool {
 		cmp = searchFilterCmpInt
 	case string:
 		cmp = searchFilterCmpString
+	case *string:
+		cmp = searchFilterCmpString
 	}
 
 	if cmp == nil {
@@ -388,25 +390,26 @@ func (s *SearchFilters) GetGroupByKey(key string) *SearchFilterGroup {
 	return nil
 }
 
-// UpdateFromLookupRoute updates a filter
-// and its counters.
-//
-// Update filter struct to include route:
-//   - Extract ASN, source, bgp communities,
-//   - Find Filter in group, increment result count if required.
-func (s *SearchFilters) UpdateFromLookupRoute(r *LookupRoute) {
+// UpdateSourcesFromLookupRoute updates the source filter
+func (s *SearchFilters) UpdateSourcesFromLookupRoute(r *LookupRoute) {
 	// Add source
 	s.GetGroupByKey(SearchKeySources).AddFilter(&SearchFilter{
 		Name:  r.RouteServer.Name,
 		Value: r.RouteServer.ID,
 	})
+}
 
+// UpdateASNSFromLookupRoute updates the ASN filter
+func (s *SearchFilters) UpdateASNSFromLookupRoute(r *LookupRoute) {
 	// Add ASN from neighbor
 	s.GetGroupByKey(SearchKeyASNS).AddFilter(&SearchFilter{
 		Name:  r.Neighbor.Description,
 		Value: r.Neighbor.ASN,
 	})
+}
 
+// UpdateCommunitiesFromLookupRoute updates the communities filter
+func (s *SearchFilters) UpdateCommunitiesFromLookupRoute(r *LookupRoute) {
 	// Add communities
 	communities := s.GetGroupByKey(SearchKeyCommunities)
 	for _, c := range r.Route.BGP.Communities {
@@ -431,6 +434,18 @@ func (s *SearchFilters) UpdateFromLookupRoute(r *LookupRoute) {
 	}
 }
 
+// UpdateFromLookupRoute updates a filter
+// and its counters.
+//
+// Update filter struct to include route:
+//   - Extract ASN, source, bgp communities,
+//   - Find Filter in group, increment result count if required.
+func (s *SearchFilters) UpdateFromLookupRoute(r *LookupRoute) {
+	s.UpdateSourcesFromLookupRoute(r)
+	s.UpdateASNSFromLookupRoute(r)
+	s.UpdateCommunitiesFromLookupRoute(r)
+}
+
 // UpdateFromRoute updates a search filter, however as
 // information of the route server or neighbor is not
 // present, as this is not a lookup route, only
@@ -593,6 +608,13 @@ func (s *SearchFilters) MergeProperties(other *SearchFilters) {
 	}
 }
 
+// HasGroup checks if a group with a given key exists
+// and filters are present.
+func (s *SearchFilters) HasGroup(key string) bool {
+	group := s.GetGroupByKey(key)
+	return len(group.Filters) > 0
+}
+
 // A NeighborFilter includes only a name and ASN.
 // We are using a slightly simpler solution for
 // neighbor queries.

pkg/api/search_filters_test.go

@@ -660,3 +660,23 @@ func TestNeighborFilterFromQuery(t *testing.T) {
 		t.Error("Unexpected name:", filter.name)
 	}
 }
+
+func TestSearchFiltersHasKey(t *testing.T) {
+	// Sources filter present
+	query := "asn=2342&sources=foo"
+	values, _ := url.ParseQuery(query)
+	filters, _ := FiltersFromQuery(values)
+
+	if !filters.HasGroup(SearchKeySources) {
+		t.Error("sources should be filtered")
+	}
+
+	// Check without sources present
+	query = "asn=2342"
+	values, _ = url.ParseQuery(query)
+	filters, _ = FiltersFromQuery(values)
+
+	if filters.HasGroup(SearchKeySources) {
+		t.Error("sources should not be filtered")
+	}
+}

pkg/http/api_endpoints_search.go

@@ -78,6 +78,20 @@ func (s *Server) apiLookupPrefixGlobal(
 	imported := make(api.LookupRoutes, 0, totalResults)
 	filtered := make(api.LookupRoutes, 0, totalResults)
 
+	// TODO: Make configurable
+	communityFilterCutoff := 100000
+	canFilterCommunities := totalResults <= communityFilterCutoff
+
+	// In case there is a source filter applied, we can filter communities
+	if filtersApplied.HasGroup(api.SearchKeySources) {
+		canFilterCommunities = true
+	}
+
+	filtersNotAvailable := []string{}
+	if !canFilterCommunities {
+		filtersNotAvailable = append(filtersNotAvailable, api.SearchKeyCommunities)
+	}
+
 	// Now, as we have allocated even more space process routes by, splitting,
 	// filtering and updating the available filters...
 	filtersAvailable := api.NewSearchFilters()
@@ -94,7 +108,14 @@ func (s *Server) apiLookupPrefixGlobal(
 			imported = append(imported, r)
 		}
 
-		filtersAvailable.UpdateFromLookupRoute(r)
+		// Update available filters for sources and asns,
+		// conditionally for communities.
+		filtersAvailable.UpdateSourcesFromLookupRoute(r)
+		filtersAvailable.UpdateASNSFromLookupRoute(r)
+
+		if canFilterCommunities {
+			filtersAvailable.UpdateCommunitiesFromLookupRoute(r)
+		}
 	}
 
 	// Remove applied filters from available
@@ -148,8 +169,9 @@ func (s *Server) apiLookupPrefixGlobal(
 			Pagination: paginationFiltered,
 		},
 		FilteredResponse: api.FilteredResponse{
-			FiltersAvailable: filtersAvailable,
-			FiltersApplied:   filtersApplied,
+			FiltersAvailable:    filtersAvailable,
+			FiltersNotAvailable: filtersNotAvailable,
+			FiltersApplied:      filtersApplied,
 		},
 	}