major refactor: sig ch, remove conf settings, jetstream pub, ci

* This is a major refactor and includes general improvements around

- context cancellation
- build settings
- jetstream pub sub
- logging
- docker builds
- conf loading

cmd/init.go

@@ -1,93 +0,0 @@
-package main
-
-import (
-	"strings"
-	"time"
-
-	"github.com/grassrootseconomics/cic-chain-events/internal/events"
-	"github.com/grassrootseconomics/cic-chain-events/internal/store"
-	"github.com/grassrootseconomics/cic-chain-events/pkg/fetch"
-	"github.com/jackc/pgx/v5"
-	"github.com/knadh/goyesql/v2"
-	"github.com/knadh/koanf"
-	"github.com/knadh/koanf/parsers/toml"
-	"github.com/knadh/koanf/providers/env"
-	"github.com/knadh/koanf/providers/file"
-	"github.com/zerodha/logf"
-)
-
-func initLogger(debug bool) logf.Logger {
-	loggOpts := logf.Opts{
-		EnableColor: true,
-	}
-
-	if debug {
-		loggOpts.Level = logf.DebugLevel
-		loggOpts.EnableCaller = true
-	}
-
-	return logf.New(loggOpts)
-}
-
-func initConfig(configFilePath string) *koanf.Koanf {
-	var (
-		ko = koanf.New(".")
-	)
-
-	confFile := file.Provider(configFilePath)
-	if err := ko.Load(confFile, toml.Parser()); err != nil {
-		lo.Fatal("could not load config file", "error", err)
-	}
-
-	if err := ko.Load(env.Provider("", ".", func(s string) string {
-		return strings.ReplaceAll(strings.ToLower(
-			strings.TrimPrefix(s, "")), "_", ".")
-	}), nil); err != nil {
-		lo.Fatal("could not override config from env vars", "error", err)
-	}
-
-	return ko
-}
-
-func initQueries(queriesPath string) goyesql.Queries {
-	queries, err := goyesql.ParseFile(queriesPath)
-	if err != nil {
-		lo.Fatal("could not load queries file", "error", err)
-	}
-
-	return queries
-}
-
-func initPgStore() (store.Store[pgx.Rows], error) {
-	pgStore, err := store.NewPostgresStore(store.PostgresStoreOpts{
-		DSN:               ko.MustString("postgres.dsn"),
-		InitialLowerBound: uint64(ko.MustInt64("syncer.initial_lower_bound")),
-		Logg:              lo,
-		Queries:           q,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return pgStore, nil
-}
-
-func initFetcher() fetch.Fetch {
-	return fetch.NewGraphqlFetcher(fetch.GraphqlOpts{
-		GraphqlEndpoint: ko.MustString("chain.graphql_endpoint"),
-	})
-}
-
-func initJetStream() (events.EventEmitter, error) {
-	jsEmitter, err := events.NewJetStreamEventEmitter(events.JetStreamOpts{
-		ServerUrl:       ko.MustString("jetstream.endpoint"),
-		PersistDuration: time.Duration(ko.MustInt("jetstream.persist_duration_hours")) * time.Hour,
-		DedupDuration:   time.Duration(ko.MustInt("jetstream.dedup_duration_hours")) * time.Hour,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return jsEmitter, nil
-}

cmd/service/filters.go

@@ -4,8 +4,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/grassrootseconomics/cic-chain-events/internal/events"
 	"github.com/grassrootseconomics/cic-chain-events/internal/filter"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
 )
 
 var (
@@ -29,25 +29,25 @@ func initAddressFilter() filter.Filter {
 	})
 }
 
-func initTransferFilter(eventEmitter events.EventEmitter) filter.Filter {
+func initTransferFilter(pub *pub.Pub) filter.Filter {
 	return filter.NewTransferFilter(filter.TransferFilterOpts{
-		EventEmitter: eventEmitter,
-		Logg:         lo,
+		Pub:  pub,
+		Logg: lo,
 	})
 
 }
 
-func initGasGiftFilter(eventEmitter events.EventEmitter) filter.Filter {
+func initGasGiftFilter(pub *pub.Pub) filter.Filter {
 	return filter.NewGasFilter(filter.GasFilterOpts{
-		EventEmitter:  eventEmitter,
+		Pub:           pub,
 		Logg:          lo,
 		SystemAddress: systemAddress,
 	})
 }
 
-func initRegisterFilter(eventEmitter events.EventEmitter) filter.Filter {
+func initRegisterFilter(pub *pub.Pub) filter.Filter {
 	return filter.NewRegisterFilter(filter.RegisterFilterOpts{
-		EventEmitter: eventEmitter,
-		Logg:         lo,
+		Pub:  pub,
+		Logg: lo,
 	})
 }

cmd/service/init.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/alitto/pond"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pool"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
+	"github.com/grassrootseconomics/cic-chain-events/internal/store"
+	"github.com/grassrootseconomics/cic-chain-events/pkg/fetch"
+	"github.com/jackc/pgx/v5"
+	"github.com/knadh/goyesql/v2"
+	"github.com/knadh/koanf/parsers/toml"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+	"github.com/knadh/koanf/v2"
+	"github.com/nats-io/nats.go"
+	"github.com/zerodha/logf"
+)
+
+func initLogger() logf.Logger {
+	loggOpts := logf.Opts{}
+
+	if debugFlag {
+		loggOpts.EnableColor = true
+		loggOpts.EnableColor = true
+		loggOpts.Level = logf.DebugLevel
+	}
+
+	return logf.New(loggOpts)
+}
+
+func initConfig() *koanf.Koanf {
+	var (
+		ko = koanf.New(".")
+	)
+
+	confFile := file.Provider(confFlag)
+	if err := ko.Load(confFile, toml.Parser()); err != nil {
+		lo.Fatal("init: could not load config file", "error", err)
+	}
+
+	if err := ko.Load(env.Provider("EVENTS_", ".", func(s string) string {
+		return strings.ReplaceAll(strings.ToLower(
+			strings.TrimPrefix(s, "EVENTS_")), "__", ".")
+	}), nil); err != nil {
+		lo.Fatal("init: could not override config from env vars", "error", err)
+	}
+
+	if debugFlag {
+		ko.Print()
+	}
+
+	return ko
+}
+
+func initQueries(queriesPath string) goyesql.Queries {
+	queries, err := goyesql.ParseFile(queriesPath)
+	if err != nil {
+		lo.Fatal("init: could not load queries file", "error", err)
+	}
+
+	return queries
+}
+
+func initPgStore(migrationsPath string, queries goyesql.Queries) store.Store[pgx.Rows] {
+	pgStore, err := store.NewPostgresStore(store.PostgresStoreOpts{
+		MigrationsFolderPath: migrationsPath,
+		DSN:                  ko.MustString("postgres.dsn"),
+		InitialLowerBound:    uint64(ko.MustInt64("syncer.initial_lower_bound")),
+		Logg:                 lo,
+		Queries:              queries,
+	})
+	if err != nil {
+		lo.Fatal("init: critical error loading chain provider", "error", err)
+	}
+
+	return pgStore
+}
+
+func initFetcher() fetch.Fetch {
+	return fetch.NewGraphqlFetcher(fetch.GraphqlOpts{
+		GraphqlEndpoint: ko.MustString("chain.graphql_endpoint"),
+	})
+}
+
+func initJanitorWorkerPool(ctx context.Context) *pond.WorkerPool {
+	return pool.NewPool(ctx, pool.Opts{
+		Concurrency: ko.MustInt("syncer.janitor_concurrency"),
+		QueueSize:   ko.MustInt("syncer.janitor_queue_size"),
+	})
+}
+
+func initHeadSyncerWorkerPool(ctx context.Context) *pond.WorkerPool {
+	return pool.NewPool(ctx, pool.Opts{
+		Concurrency: 1,
+		QueueSize:   1,
+	})
+}
+
+func initJetStream() (*nats.Conn, nats.JetStreamContext) {
+	natsConn, err := nats.Connect(ko.MustString("jetstream.endpoint"))
+	if err != nil {
+		lo.Fatal("init: critical error connecting to NATS", "error", err)
+	}
+
+	js, err := natsConn.JetStream()
+	if err != nil {
+		lo.Fatal("init: bad JetStream opts", "error", err)
+
+	}
+
+	return natsConn, js
+}
+
+func initPub(natsConn *nats.Conn, jsCtx nats.JetStreamContext) *pub.Pub {
+	pub, err := pub.NewPub(pub.PubOpts{
+		DedupDuration:   time.Duration(ko.MustInt("jetstream.dedup_duration_hrs")) * time.Hour,
+		JsCtx:           jsCtx,
+		NatsConn:        natsConn,
+		PersistDuration: time.Duration(ko.MustInt("jetstream.persist_duration_hrs")) * time.Hour,
+	})
+	if err != nil {
+		lo.Fatal("init: critical error bootstrapping pub", "error", err)
+	}
+
+	return pub
+}

cmd/service/main.go

@@ -3,26 +3,34 @@ package main
 import (
 	"context"
 	"flag"
-	"os"
-	"os/signal"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/grassrootseconomics/cic-chain-events/internal/filter"
 	"github.com/grassrootseconomics/cic-chain-events/internal/pipeline"
-	"github.com/grassrootseconomics/cic-chain-events/internal/pool"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
 	"github.com/grassrootseconomics/cic-chain-events/internal/syncer"
 	"github.com/knadh/goyesql/v2"
-	"github.com/knadh/koanf"
+	"github.com/knadh/koanf/v2"
+	"github.com/labstack/echo/v4"
 	"github.com/zerodha/logf"
 )
 
+type (
+	internalServiceContainer struct {
+		apiService *echo.Echo
+		pub        *pub.Pub
+	}
+)
+
 var (
-	confFlag    string
-	debugFlag   bool
-	queriesFlag string
+	build string
+
+	confFlag             string
+	debugFlag            bool
+	migrationsFolderFlag string
+	queriesFlag          string
 
 	ko *koanf.Koanf
 	lo logf.Logger
@@ -31,61 +39,51 @@ var (
 
 func init() {
 	flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
-	flag.BoolVar(&debugFlag, "log", true, "Enable debug logging")
+	flag.BoolVar(&debugFlag, "debug", false, "Enable debug logging")
+	flag.StringVar(&migrationsFolderFlag, "migrations", "migrations/", "Migrations folder location")
 	flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location")
 	flag.Parse()
 
-	lo = initLogger(debugFlag)
-	ko = initConfig(confFlag)
-	q = initQueries(queriesFlag)
+	lo = initLogger()
+	ko = initConfig()
 }
 
 func main() {
-	syncerStats := &syncer.Stats{}
-	wg := &sync.WaitGroup{}
-	apiServer := initApiServer()
-
-	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
-	defer stop()
-
-	janitorWorkerPool := pool.NewPool(ctx, pool.Opts{
-		Concurrency: ko.MustInt("syncer.janitor_concurrency"),
-		QueueSize:   ko.MustInt("syncer.janitor_queue_size"),
-	})
-
-	pgStore, err := initPgStore()
-	if err != nil {
-		lo.Fatal("main: critical error loading pg store", "error", err)
-	}
-
-	jsCtx, err := initJetStream()
-	if err != nil {
-		lo.Fatal("main: critical error loading jetstream context", "error", err)
-	}
+	lo.Info("main: starting cic-chain-events", "build", build)
 
+	parsedQueries := initQueries(queriesFlag)
 	graphqlFetcher := initFetcher()
+	pgStore := initPgStore(migrationsFolderFlag, parsedQueries)
+	natsConn, jsCtx := initJetStream()
+	jsPub := initPub(natsConn, jsCtx)
 
 	pipeline := pipeline.NewPipeline(pipeline.PipelineOpts{
 		BlockFetcher: graphqlFetcher,
 		Filters: []filter.Filter{
 			initAddressFilter(),
-			initGasGiftFilter(jsCtx),
-			initTransferFilter(jsCtx),
-			initRegisterFilter(jsCtx),
+			initGasGiftFilter(jsPub),
+			initTransferFilter(jsPub),
+			initRegisterFilter(jsPub),
 		},
 		Logg:  lo,
 		Store: pgStore,
 	})
 
-	headSyncerWorker := pool.NewPool(ctx, pool.Opts{
-		Concurrency: 1,
-		QueueSize:   1,
-	})
+	internalServices := &internalServiceContainer{
+		pub: jsPub,
+	}
+	syncerStats := &syncer.Stats{}
+	wg := &sync.WaitGroup{}
+
+	signalCh, closeCh := createSigChannel()
+	defer closeCh()
+
+	ctx, cancel := context.WithCancel(context.Background())
 
 	headSyncer, err := syncer.NewHeadSyncer(syncer.HeadSyncerOpts{
 		Logg:       lo,
 		Pipeline:   pipeline,
-		Pool:       headSyncerWorker,
+		Pool:       initHeadSyncerWorkerPool(ctx),
 		Stats:      syncerStats,
 		WsEndpoint: ko.MustString("chain.ws_endpoint"),
 	})
@@ -97,7 +95,7 @@ func main() {
 		BatchSize:     uint64(ko.MustInt64("syncer.janitor_queue_size")),
 		Logg:          lo,
 		Pipeline:      pipeline,
-		Pool:          janitorWorkerPool,
+		Pool:          initJanitorWorkerPool(ctx),
 		Stats:         syncerStats,
 		Store:         pgStore,
 		SweepInterval: time.Second * time.Duration(ko.MustInt64("syncer.janitor_sweep_interval")),
@@ -107,6 +105,7 @@ func main() {
 	go func() {
 		defer wg.Done()
 		if err := headSyncer.Start(ctx); err != nil {
+			lo.Info("main: starting head syncer")
 			lo.Fatal("main: critical error starting head syncer", "error", err)
 		}
 	}()
@@ -114,16 +113,19 @@ func main() {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		lo.Info("main: starting janitor")
 		if err := janitor.Start(ctx); err != nil {
 			lo.Fatal("main: critical error starting janitor", "error", err)
 		}
 	}()
 
+	internalServices.apiService = initApiServer()
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		lo.Info("starting API server")
-		if err := apiServer.Start(ko.MustString("api.address")); err != nil {
+		host := ko.MustString("service.address")
+		lo.Info("main: starting API server", "host", host)
+		if err := internalServices.apiService.Start(host); err != nil {
 			if strings.Contains(err.Error(), "Server closed") {
 				lo.Info("main: shutting down server")
 			} else {
@@ -132,11 +134,9 @@ func main() {
 		}
 	}()
 
-	<-ctx.Done()
-
-	if err := apiServer.Shutdown(ctx); err != nil {
-		lo.Error("main: could not gracefully shutdown api server", "err", err)
-	}
+	lo.Info("main: graceful shutdown triggered", "signal", <-signalCh)
+	cancel()
+	startGracefulShutdown(context.Background(), internalServices)
 
 	wg.Wait()
 }

cmd/service/utils.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+func createSigChannel() (chan os.Signal, func()) {
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
+
+	return signalCh, func() {
+		close(signalCh)
+	}
+}
+
+func startGracefulShutdown(ctx context.Context, internalServices *internalServiceContainer) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	internalServices.pub.Close()
+
+	if err := internalServices.apiService.Shutdown(ctx); err != nil {
+		lo.Fatal("Could not gracefully shutdown api server", "err", err)
+	}
+}