From 2c7fab0a38a733e8fa95f197620d63d218f1ba22 Mon Sep 17 00:00:00 2001 From: Annika Hannig Date: Wed, 12 Jan 2022 11:50:51 +0100 Subject: [PATCH] initial database connection and migration --- db/.gitignore | 1 + db/compose.yml | 13 +++ db/init.sh | 70 ++++++++++++ pkg/store/backends/postgres/connection.go | 5 - .../backends/postgres/connection_test.go | 46 ++++++++ pkg/store/backends/postgres/manager.go | 102 ++++++++++++++++++ .../backends/postgres/neighbors_backend.go | 33 ++++++ pkg/store/backends/postgres/schema.sql | 11 +- 8 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 db/.gitignore create mode 100644 db/compose.yml create mode 100755 db/init.sh create mode 100644 pkg/store/backends/postgres/connection_test.go create mode 100644 pkg/store/backends/postgres/manager.go create mode 100644 pkg/store/backends/postgres/neighbors_backend.go diff --git a/db/.gitignore b/db/.gitignore new file mode 100644 index 0000000..c10c784 --- /dev/null +++ b/db/.gitignore @@ -0,0 +1 @@ +_data/ diff --git a/db/compose.yml b/db/compose.yml new file mode 100644 index 0000000..0eb7c97 --- /dev/null +++ b/db/compose.yml @@ -0,0 +1,13 @@ + +version: "3" + +services: + + postgres: + image: postgres:11 + ports: + - 5432:5432 + environment: + POSTGRES_PASSWORD: postgres + volumes: + - ./_data:/var/lib/postgresql/data diff --git a/db/init.sh b/db/init.sh new file mode 100755 index 0000000..26c519c --- /dev/null +++ b/db/init.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env sh + +###################################################################### +# @author : annika +# @file : init +# @created : Tuesday Jan 11, 2022 15:35:20 CET +# @description : Initialize the database +###################################################################### + +if [ -z $PSQL ]; then + PSQL="psql" +fi + +if [ -z $PGHOST ]; then + export PGHOST="localhost" +fi + +if [ -z $PGPORT ]; then + export PGPORT="5432" +fi + +if [ -z $PGDATABASE ]; then + export PGDATABASE="alice" +fi + +if [ -z $PGUSER ]; then + export PGUSER="postgres" +fi + +if [ -z $PGPASSWORD ]; then + export PGPASSWORD="postgres" +fi + +## Commandline opts: +OPT_USAGE=0 +OPT_TESTING=0 +OPT_CLEAR=0 + +while [ $# -gt 0 ]; do + case "$1" in + -h) OPT_USAGE=1 ;; + -t) OPT_TESTING=1 ;; + -c) OPT_CLEAR=1 ;; + esac + shift +done + +if [ $OPT_USAGE -eq 1 ]; then + echo "Options:" + echo " -h Show this helpful text" + echo " -c Drop and create the database" + echo " -t Make a test database" + exit +fi + +if [ $OPT_TESTING -eq 1 ]; then + echo "++ using test database" + NAME="${PGDATABASE}_test" + export PGDATABASE=$NAME +fi + +if [ $OPT_CLEAR -eq 1 ]; then + echo "++ clearing database" + $PSQL template1 -c "DROP DATABASE $PGDATABASE" + $PSQL template1 -c "CREATE DATABASE $PGDATABASE" +fi +###################################################################### + + + diff --git a/pkg/store/backends/postgres/connection.go b/pkg/store/backends/postgres/connection.go index 1c9b021..3f3d78b 100644 --- a/pkg/store/backends/postgres/connection.go +++ b/pkg/store/backends/postgres/connection.go @@ -10,11 +10,6 @@ import ( ) var ( - // ErrNotInitialized will be returned if the - // database pool is accessed before initializing using - // Connect. - ErrNotInitialized = errors.New("store not initialized, pool not ready") - // ErrMaxConnsUnconfigured will be returned, if the // the maximum connections are zero. ErrMaxConnsUnconfigured = errors.New("MaxConns not configured") diff --git a/pkg/store/backends/postgres/connection_test.go b/pkg/store/backends/postgres/connection_test.go new file mode 100644 index 0000000..6def9c7 --- /dev/null +++ b/pkg/store/backends/postgres/connection_test.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/jackc/pgx/v4/pgxpool" +) + +// ConnectTest uses defaults for the connection +// environment variable is not set. +func ConnectTest() *pgxpool.Pool { + ctx := context.Background() + url := os.Getenv("ALICE_TEST_DB_URL") + if url == "" { + url = "postgres://postgres:postgres@localhost:5432/alice_test" + } + p, err := Connect(ctx, &ConnectOpts{ + URL: url, + MinConns: 2, + MaxConns: 16}) + if err != nil { + panic(err) + } + + m := NewManager(p) + err = m.Initialize(ctx) + if err != nil { + panic(err) + } + + return p +} + +func TestInitialize(t *testing.T) { + p := ConnectTest() + m := NewManager(p) + s := m.Status(context.Background()) + if s.Error != nil { + t.Error(s.Error) + } + if s.Migrated == false { + t.Error("schema is not migrated, current:", s.SchemaVersion) + } +} diff --git a/pkg/store/backends/postgres/manager.go b/pkg/store/backends/postgres/manager.go new file mode 100644 index 0000000..f16da76 --- /dev/null +++ b/pkg/store/backends/postgres/manager.go @@ -0,0 +1,102 @@ +package postgres + +import ( + "context" + _ "embed" // embed schema + "log" + "time" + + "github.com/jackc/pgx/v4/pgxpool" +) + +// Include the schema through embedding +//go:embed schema.sql +var schema string + +// CurrentSchemaVersion is the current version of the schema +const CurrentSchemaVersion = 1 + +var ( +// ErrNotInitialized is returned when the database +// schema is not migrated yet +) + +// Status is the database / store status +type Status struct { + Migrated bool `json:"migrated"` + SchemaVersion int `json:"schema_version"` + SchemaAppliedAt time.Time `json:"schema_applied_at"` + Error error `json:"error"` +} + +// Log writes the status into the log +func (s *Status) Log() { + log.Println( + "Database migrated:", s.Migrated) + log.Println( + "Schema version:", s.SchemaVersion, + "applied at:", s.SchemaAppliedAt) +} + +// The Manager supervises the database. It can migrate the +// schema and retrieve a status. +type Manager struct { + pool *pgxpool.Pool +} + +// NewManager creates a new database manager +func NewManager(pool *pgxpool.Pool) *Manager { + return &Manager{ + pool: pool, + } +} + +// Start the background jobs for database management +func (m *Manager) Start(ctx context.Context) { + m.Status(ctx).Log() +} + +// Status retrieves the current schema version +// and checks if migrated. In case an error occures, +// it will be included in the result. +func (m *Manager) Status(ctx context.Context) *Status { + status := &Status{} + qry := ` + SELECT version, applied_at FROM __meta__ + ORDER BY version DESC + LIMIT 1 + ` + err := m.pool.QueryRow(ctx, qry).Scan( + &status.SchemaVersion, + &status.SchemaAppliedAt, + ) + if err != nil { + status.Error = err + return status + } + + // Is the database migrated? + status.Migrated = CurrentSchemaVersion == status.SchemaVersion + + return status +} + +// Migrate applies the database intialisation script if required. +func (m *Manager) Migrate(ctx context.Context) error { + s := m.Status(ctx) + if s.Migrated { + return nil + } + + return m.Initialize(ctx) +} + +// Initialize will apply the database schema. This will clear the +// database. However for now we treat the state as disposable. +func (m *Manager) Initialize(ctx context.Context) error { + _, err := m.pool.Exec(ctx, schema) + if err != nil { + return err + } + return nil +} diff --git a/pkg/store/backends/postgres/neighbors_backend.go b/pkg/store/backends/postgres/neighbors_backend.go new file mode 100644 index 0000000..0d4e8fd --- /dev/null +++ b/pkg/store/backends/postgres/neighbors_backend.go @@ -0,0 +1,33 @@ +package postgres + +import ( + "context" + + "github.com/alice-lg/alice-lg/pkg/api" + "github.com/jackc/pgx/v4/pgxpool" +) + +// NeighborsBackend implements a neighbors store +// using a postgres database +type NeighborsBackend struct { + pool *pgxpool.Pool +} + +// NewNeighborsBackend initializes the backend +// with a pool. +func NewNeighborsBackend(pool *pgxpool.Pool) *NeighborsBackend { + b := &NeighborsBackend{ + pool: pool, + } + return b +} + +// SetNeighbors updates the current neighbors of +// a route server identified by sourceID +func (b *NeighborsBackend) SetNeighbors( + ctx context.Context, + sourceID string, + neighbors api.Neighbors, +) error { + return nil +} diff --git a/pkg/store/backends/postgres/schema.sql b/pkg/store/backends/postgres/schema.sql index f64f6b4..b9dab6c 100644 --- a/pkg/store/backends/postgres/schema.sql +++ b/pkg/store/backends/postgres/schema.sql @@ -5,9 +5,14 @@ -- ---------------------- -- -- %% Author: annika --- %% Description: Create alice-lg db schema. +-- %% Description: Apply alice-lg db schema. -- +-- Clear state +DROP TABLE IF EXISTS routes; +DROP TABLE IF EXISTS neighbors; +DROP TABLE IF EXISTS __meta__; + -- Neighbors CREATE TABLE neighbors ( id VARCHAR(255) NOT NULL PRIMARY KEY, @@ -19,7 +24,7 @@ CREATE TABLE neighbors ( neighbor jsonb NOT NULL, -- Timestamps - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX idx_neighbors_rs_id @@ -39,7 +44,7 @@ CREATE TABLE routes ( route jsonb NOT NULL, -- Timestamps - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX idx_routes_network ON routes ( network );