From 2cf3ad6b5963ac5082b0b2f5880855630515b5d8 Mon Sep 17 00:00:00 2001 From: Annika Hannig Date: Thu, 13 Jan 2022 15:59:57 +0100 Subject: [PATCH] improved prefix query validation --- db/shell.sh | 4 +- pkg/http/api_endpoints_search.go | 9 +- pkg/http/api_validators.go | 21 ++- .../backends/postgres/neighbors_backend.go | 34 +++- .../postgres/neighbors_backend_test.go | 10 +- pkg/store/backends/postgres/routes_backend.go | 173 ++++++++++++++++++ pkg/store/backends/postgres/schema.sql | 11 +- 7 files changed, 239 insertions(+), 23 deletions(-) create mode 100644 pkg/store/backends/postgres/routes_backend.go diff --git a/db/shell.sh b/db/shell.sh index 688ca9d..bb49cc2 100755 --- a/db/shell.sh +++ b/db/shell.sh @@ -4,7 +4,7 @@ # @author : annika # @file : init # @created : Tuesday Jan 11, 2022 15:35:20 CET -# @description : Initialize the database +# @description : Start `psql` as database shell ###################################################################### if [ -z $PSQL ]; then @@ -55,5 +55,5 @@ if [ $OPT_TESTING -eq 1 ]; then export PGDATABASE=$NAME fi -psql +exec psql diff --git a/pkg/http/api_endpoints_search.go b/pkg/http/api_endpoints_search.go index 0ae3a67..a6869e5 100644 --- a/pkg/http/api_endpoints_search.go +++ b/pkg/http/api_endpoints_search.go @@ -25,11 +25,6 @@ func (s *Server) apiLookupPrefixGlobal( return nil, err } - q, err = validatePrefixQuery(q) - if err != nil { - return nil, err - } - // Check what we want to query // Prefix -> fetch prefix // _ -> fetch neighbors and routes @@ -47,6 +42,10 @@ func (s *Server) apiLookupPrefixGlobal( // Perform query var routes api.LookupRoutes if lookupPrefix { + q, err = validatePrefixQuery(q) + if err != nil { + return nil, err + } routes, err = s.routesStore.LookupPrefix(ctx, q) if err != nil { return nil, err diff --git a/pkg/http/api_validators.go b/pkg/http/api_validators.go index 5c4a265..0660ec4 100644 --- a/pkg/http/api_validators.go +++ b/pkg/http/api_validators.go @@ -1,11 +1,24 @@ package http import ( + "errors" "fmt" + "strings" "net/http" ) +var ( + // ErrQueryTooShort will be returned when the query + // is less than 2 characters. + ErrQueryTooShort = errors.New("query too short") + + // ErrQueryIncomplete will be returned when the + // prefix query lacks a : or . + ErrQueryIncomplete = errors.New( + "prefix query must contain at least on '.' or ':'") +) + // Helper: Validate source Id func validateSourceID(id string) (string, error) { if len(id) > 42 { @@ -34,11 +47,15 @@ func validateQueryString(req *http.Request, key string) (string, error) { return value, nil } -// Helper: Validate prefix query +// Helper: Validate prefix query. It should contain +// at least one dot or : func validatePrefixQuery(value string) (string, error) { // We should at least provide 2 chars if len(value) < 2 { - return "", fmt.Errorf("query too short") + return "", ErrQueryTooShort + } + if !strings.Contains(value, ":") && !strings.Contains(value, ".") { + return "", ErrQueryIncomplete } return value, nil } diff --git a/pkg/store/backends/postgres/neighbors_backend.go b/pkg/store/backends/postgres/neighbors_backend.go index 38f8d96..2191a15 100644 --- a/pkg/store/backends/postgres/neighbors_backend.go +++ b/pkg/store/backends/postgres/neighbors_backend.go @@ -32,33 +32,55 @@ func (b *NeighborsBackend) SetNeighbors( sourceID string, neighbors api.Neighbors, ) error { + // Clear current neighbors + now := time.Now().UTC() for _, n := range neighbors { - if err := b.persistNeighbor(ctx, sourceID, n); err != nil { + if err := b.persist(ctx, sourceID, n, now); err != nil { return err } } + // Remove old neighbors + if err := b.deleteStale(ctx, sourceID, now); err != nil { + return err + } return nil } -// Private persistNeighbor saves a neighbor to the database -func (b *NeighborsBackend) persistNeighbor( +// Private persist saves a neighbor to the database +func (b *NeighborsBackend) persist( ctx context.Context, sourceID string, neighbor *api.Neighbor, + now time.Time, ) error { - now := time.Now().UTC() qry := ` INSERT INTO neighbors ( id, rs_id, neighbor, updated_at ) VALUES ( $1, $2, $3, $4 ) ON CONFLICT ON CONSTRAINT neighbors_pkey DO UPDATE - SET neighbor = EXCLUDED.neighbor, - updated_at = EXCLUDED.updated_at + SET neighbor = EXCLUDED.neighbor, + updated_at = EXCLUDED.updated_at ` _, err := b.pool.Exec(ctx, qry, neighbor.ID, sourceID, neighbor, now) return err } +// Private deleteStale removes all neighbors not inserted or +// updated at a specific time. +func (b *NeighborsBackend) deleteStale( + ctx context.Context, + sourceID string, + t time.Time, +) error { + qry := ` + DELETE FROM neighbors + WHERE rs_id = $1 + AND updated_at <> $2 + ` + _, err := b.pool.Exec(ctx, qry, sourceID, t) + return err +} + // Private queryNeighborsAt selects all neighbors // for a given sourceID func (b *NeighborsBackend) queryNeighborsAt( diff --git a/pkg/store/backends/postgres/neighbors_backend_test.go b/pkg/store/backends/postgres/neighbors_backend_test.go index 627268d..5c78386 100644 --- a/pkg/store/backends/postgres/neighbors_backend_test.go +++ b/pkg/store/backends/postgres/neighbors_backend_test.go @@ -3,6 +3,7 @@ package postgres import ( "context" "testing" + "time" "github.com/alice-lg/alice-lg/pkg/api" ) @@ -14,24 +15,25 @@ func TestPersistNeighborLookup(t *testing.T) { ID: "n2342", Address: "test123", } - if err := b.persistNeighbor(context.Background(), "rs1", n); err != nil { + now := time.Now().UTC() + if err := b.persist(context.Background(), "rs1", n, now); err != nil { t.Fatal(err) } // make an update n.Address = "test234" - if err := b.persistNeighbor(context.Background(), "rs1", n); err != nil { + if err := b.persist(context.Background(), "rs1", n, now); err != nil { t.Fatal(err) } // Add a second n.ID = "foo23" - if err := b.persistNeighbor(context.Background(), "rs1", n); err != nil { + if err := b.persist(context.Background(), "rs1", n, now); err != nil { t.Fatal(err) } // Add to different rs - if err := b.persistNeighbor(context.Background(), "rs2", n); err != nil { + if err := b.persist(context.Background(), "rs2", n, now); err != nil { t.Fatal(err) } diff --git a/pkg/store/backends/postgres/routes_backend.go b/pkg/store/backends/postgres/routes_backend.go new file mode 100644 index 0000000..79f9d44 --- /dev/null +++ b/pkg/store/backends/postgres/routes_backend.go @@ -0,0 +1,173 @@ +package postgres + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/alice-lg/alice-lg/pkg/api" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" +) + +// RoutesBackend implements a postgres store for routes. +type RoutesBackend struct { + pool *pgxpool.Pool +} + +// NewRoutesBackend creates a new instance with a postgres +// connection pool. +func NewRoutesBackend(pool *pgxpool.Pool) *RoutesBackend { + return &RoutesBackend{ + pool: pool, + } +} + +// SetRoutes implements the RoutesStoreBackend interface +// function for setting all routes of a source identified +// by ID. +func (b *RoutesBackend) SetRoutes( + ctx context.Context, + sourceID string, + routes api.LookupRoutes, +) error { + now := time.Now().UTC() + for _, r := range routes { + if err := b.persist(ctx, sourceID, r, now); err != nil { + return err + } + } + if err := b.deleteStale(ctx, sourceID, now); err != nil { + return err + } + return nil +} + +// Private persist route in database +func (b *RoutesBackend) persist( + ctx context.Context, + sourceID string, + route *api.LookupRoute, + now time.Time, +) error { + qry := ` + INSERT INTO routes ( + id, + rs_id, + neighbor_id, + network, + route, + updated_at + ) VALUES ( + $1, $2, $3, $4, $5, $6 + ) + ON CONFLICT ON CONSTRAINT routes_pkey DO UPDATE + SET route = EXCLUDED.route, + network = EXCLUDED.network, + neighbor_id = EXCLUDED.neighbor_id, + updated_at = EXCLUDED.updated_at + ` + _, err := b.pool.Exec( + ctx, + qry, + route.Route.ID, + sourceID, + route.Neighbor.ID, + route.Route.Network, + route, + now) + return err +} + +// Private deleteStale removes all routes not inserted or +// updated at a specific time. +func (b *RoutesBackend) deleteStale( + ctx context.Context, + sourceID string, + t time.Time, +) error { + qry := ` + DELETE FROM routes + WHERE rs_id = $1 + AND updated_at <> $2 + ` + _, err := b.pool.Exec(ctx, qry, sourceID, t) + return err +} + +// Private queryCountByState will query routes and filter +// by state +func (b *RoutesBackend) queryCountByState( + ctx context.Context, + sourceID string, + state string, +) pgx.Row { + qry := `SELECT COUNT(1) FROM routes + WHERE rs_id = $1 AND route->'state' = $2` + return b.pool.QueryRow(ctx, qry, sourceID, state) +} + +// CountRoutesAt returns the number of filtered and imported +// routes and implements the RoutesStoreBackend interface. +func (b *RoutesBackend) CountRoutesAt( + ctx context.Context, + sourceID string, +) (uint, uint, error) { + var ( + imported uint + filtered uint + ) + err := b.queryCountByState(ctx, sourceID, api.RouteStateFiltered). + Scan(&filtered) + if err != nil { + return 0, 0, err + } + err = b.queryCountByState(ctx, sourceID, api.RouteStateImported). + Scan(&imported) + if err != nil { + return 0, 0, err + } + return imported, filtered, nil +} + +// FindByNeighbors will return the prefixes for a +// list of neighbors identified by ID. +func (b *RoutesBackend) FindByNeighbors( + ctx context.Context, + neighborIDs []interface{}, +) (api.LookupRoutes, error) { + vars := make([]string, 0, len(neighborIDs)) + for n := range neighborIDs { + vars = append(vars, fmt.Sprintf("$%d", n+1)) + } + listQry := strings.Join(vars, ",") + qry := ` + SELECT route + FROM routes + WHERE neighbor_id IN (` + listQry + `)` + + rows, err := b.pool.Query(ctx, qry, neighborIDs...) + if err != nil { + return nil, err + } + cmd := rows.CommandTag() + results := make(api.LookupRoutes, 0, cmd.RowsAffected()) + for rows.Next() { + route := &api.LookupRoute{} + if err := rows.Scan(&route); err != nil { + return nil, err + } + results = append(results, route) + } + return results, nil +} + +// FindByPrefix will return the prefixes matching a pattern +func (b *RoutesBackend) FindByPrefix( + ctx context.Context, + prefix string, +) (api.LookupRoutes, error) { + // We are searching route.Network + return nil, nil +} diff --git a/pkg/store/backends/postgres/schema.sql b/pkg/store/backends/postgres/schema.sql index 85d3bc5..41024ad 100644 --- a/pkg/store/backends/postgres/schema.sql +++ b/pkg/store/backends/postgres/schema.sql @@ -31,7 +31,9 @@ CREATE TABLE neighbors ( ); CREATE INDEX idx_neighbors_rs_id - ON neighbors USING HASH (rs_id); + ON neighbors USING HASH (rs_id); +CREATE INDEX idx_neighbors_updated_at + ON neighbors ( updated_at ); -- Routes CREATE TABLE routes ( @@ -40,7 +42,7 @@ CREATE TABLE routes ( neighbor_id VARCHAR(255) NOT NULL, -- Indexed attributes - network cidr NOT NULL, + network VARCHAR(50) NOT NULL, -- JSON serialized route route jsonb NOT NULL, @@ -54,8 +56,9 @@ CREATE TABLE routes ( REFERENCES neighbors(rs_id, id) ON DELETE CASCADE ); -CREATE INDEX idx_routes_network ON routes ( network ); - +CREATE INDEX idx_routes_network ON routes ( network ); +CREATE INDEX idx_neighbor_id ON routes ( neighbor_id ); +CREATE INDEX idx_routes_updated_at ON routes ( updated_at ); -- The meta table stores information about the schema -- like when it was migrated and the current revision.