major refactor: sig ch, remove conf settings, jetstream pub, ci

* This is a major refactor and includes general improvements around

- context cancellation
- build settings
- jetstream pub sub
- logging
- docker builds
- conf loading

cmd/service/api.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/labstack/echo/v4"
+)
+
+func initApiServer() *echo.Echo {
+	server := echo.New()
+	server.HideBanner = true
+	server.HidePort = true
+
+	if ko.Bool("metrics.go_process") {
+		server.GET("/metrics", func(c echo.Context) error {
+			metrics.WritePrometheus(c.Response(), true)
+			return nil
+		})
+	}
+
+	return server
+}

cmd/service/filters.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/grassrootseconomics/cic-chain-events/internal/filter"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
+)
+
+var (
+	systemAddress = strings.ToLower("0x3D85285e39f05773aC92EAD27CB50a4385A529E4")
+)
+
+func initAddressFilter() filter.Filter {
+	// TODO: Bootstrap addresses from smart contract
+	// TODO: Add route to update cache
+	cache := &sync.Map{}
+
+	// Example bootstrap addresses
+	cache.Store(strings.ToLower("0xB92463E2262E700e29c16416270c9Fdfa17934D7"), "TRNVoucher")
+	cache.Store(strings.ToLower("0xf2a1fc19Ad275A0EAe3445798761FeD1Eea725d5"), "GasFaucet")
+	cache.Store(strings.ToLower("0x1e041282695C66944BfC53cabce947cf35CEaf87"), "AddressIndex")
+
+	return filter.NewAddressFilter(filter.AddressFilterOpts{
+		Cache:         cache,
+		Logg:          lo,
+		SystemAddress: systemAddress,
+	})
+}
+
+func initTransferFilter(pub *pub.Pub) filter.Filter {
+	return filter.NewTransferFilter(filter.TransferFilterOpts{
+		Pub:  pub,
+		Logg: lo,
+	})
+
+}
+
+func initGasGiftFilter(pub *pub.Pub) filter.Filter {
+	return filter.NewGasFilter(filter.GasFilterOpts{
+		Pub:           pub,
+		Logg:          lo,
+		SystemAddress: systemAddress,
+	})
+}
+
+func initRegisterFilter(pub *pub.Pub) filter.Filter {
+	return filter.NewRegisterFilter(filter.RegisterFilterOpts{
+		Pub:  pub,
+		Logg: lo,
+	})
+}

cmd/service/init.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/alitto/pond"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pool"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
+	"github.com/grassrootseconomics/cic-chain-events/internal/store"
+	"github.com/grassrootseconomics/cic-chain-events/pkg/fetch"
+	"github.com/jackc/pgx/v5"
+	"github.com/knadh/goyesql/v2"
+	"github.com/knadh/koanf/parsers/toml"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+	"github.com/knadh/koanf/v2"
+	"github.com/nats-io/nats.go"
+	"github.com/zerodha/logf"
+)
+
+func initLogger() logf.Logger {
+	loggOpts := logf.Opts{}
+
+	if debugFlag {
+		loggOpts.EnableColor = true
+		loggOpts.EnableColor = true
+		loggOpts.Level = logf.DebugLevel
+	}
+
+	return logf.New(loggOpts)
+}
+
+func initConfig() *koanf.Koanf {
+	var (
+		ko = koanf.New(".")
+	)
+
+	confFile := file.Provider(confFlag)
+	if err := ko.Load(confFile, toml.Parser()); err != nil {
+		lo.Fatal("init: could not load config file", "error", err)
+	}
+
+	if err := ko.Load(env.Provider("EVENTS_", ".", func(s string) string {
+		return strings.ReplaceAll(strings.ToLower(
+			strings.TrimPrefix(s, "EVENTS_")), "__", ".")
+	}), nil); err != nil {
+		lo.Fatal("init: could not override config from env vars", "error", err)
+	}
+
+	if debugFlag {
+		ko.Print()
+	}
+
+	return ko
+}
+
+func initQueries(queriesPath string) goyesql.Queries {
+	queries, err := goyesql.ParseFile(queriesPath)
+	if err != nil {
+		lo.Fatal("init: could not load queries file", "error", err)
+	}
+
+	return queries
+}
+
+func initPgStore(migrationsPath string, queries goyesql.Queries) store.Store[pgx.Rows] {
+	pgStore, err := store.NewPostgresStore(store.PostgresStoreOpts{
+		MigrationsFolderPath: migrationsPath,
+		DSN:                  ko.MustString("postgres.dsn"),
+		InitialLowerBound:    uint64(ko.MustInt64("syncer.initial_lower_bound")),
+		Logg:                 lo,
+		Queries:              queries,
+	})
+	if err != nil {
+		lo.Fatal("init: critical error loading chain provider", "error", err)
+	}
+
+	return pgStore
+}
+
+func initFetcher() fetch.Fetch {
+	return fetch.NewGraphqlFetcher(fetch.GraphqlOpts{
+		GraphqlEndpoint: ko.MustString("chain.graphql_endpoint"),
+	})
+}
+
+func initJanitorWorkerPool(ctx context.Context) *pond.WorkerPool {
+	return pool.NewPool(ctx, pool.Opts{
+		Concurrency: ko.MustInt("syncer.janitor_concurrency"),
+		QueueSize:   ko.MustInt("syncer.janitor_queue_size"),
+	})
+}
+
+func initHeadSyncerWorkerPool(ctx context.Context) *pond.WorkerPool {
+	return pool.NewPool(ctx, pool.Opts{
+		Concurrency: 1,
+		QueueSize:   1,
+	})
+}
+
+func initJetStream() (*nats.Conn, nats.JetStreamContext) {
+	natsConn, err := nats.Connect(ko.MustString("jetstream.endpoint"))
+	if err != nil {
+		lo.Fatal("init: critical error connecting to NATS", "error", err)
+	}
+
+	js, err := natsConn.JetStream()
+	if err != nil {
+		lo.Fatal("init: bad JetStream opts", "error", err)
+
+	}
+
+	return natsConn, js
+}
+
+func initPub(natsConn *nats.Conn, jsCtx nats.JetStreamContext) *pub.Pub {
+	pub, err := pub.NewPub(pub.PubOpts{
+		DedupDuration:   time.Duration(ko.MustInt("jetstream.dedup_duration_hrs")) * time.Hour,
+		JsCtx:           jsCtx,
+		NatsConn:        natsConn,
+		PersistDuration: time.Duration(ko.MustInt("jetstream.persist_duration_hrs")) * time.Hour,
+	})
+	if err != nil {
+		lo.Fatal("init: critical error bootstrapping pub", "error", err)
+	}
+
+	return pub
+}

cmd/service/main.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/grassrootseconomics/cic-chain-events/internal/filter"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pipeline"
+	"github.com/grassrootseconomics/cic-chain-events/internal/pub"
+	"github.com/grassrootseconomics/cic-chain-events/internal/syncer"
+	"github.com/knadh/goyesql/v2"
+	"github.com/knadh/koanf/v2"
+	"github.com/labstack/echo/v4"
+	"github.com/zerodha/logf"
+)
+
+type (
+	internalServiceContainer struct {
+		apiService *echo.Echo
+		pub        *pub.Pub
+	}
+)
+
+var (
+	build string
+
+	confFlag             string
+	debugFlag            bool
+	migrationsFolderFlag string
+	queriesFlag          string
+
+	ko *koanf.Koanf
+	lo logf.Logger
+	q  goyesql.Queries
+)
+
+func init() {
+	flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
+	flag.BoolVar(&debugFlag, "debug", false, "Enable debug logging")
+	flag.StringVar(&migrationsFolderFlag, "migrations", "migrations/", "Migrations folder location")
+	flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location")
+	flag.Parse()
+
+	lo = initLogger()
+	ko = initConfig()
+}
+
+func main() {
+	lo.Info("main: starting cic-chain-events", "build", build)
+
+	parsedQueries := initQueries(queriesFlag)
+	graphqlFetcher := initFetcher()
+	pgStore := initPgStore(migrationsFolderFlag, parsedQueries)
+	natsConn, jsCtx := initJetStream()
+	jsPub := initPub(natsConn, jsCtx)
+
+	pipeline := pipeline.NewPipeline(pipeline.PipelineOpts{
+		BlockFetcher: graphqlFetcher,
+		Filters: []filter.Filter{
+			initAddressFilter(),
+			initGasGiftFilter(jsPub),
+			initTransferFilter(jsPub),
+			initRegisterFilter(jsPub),
+		},
+		Logg:  lo,
+		Store: pgStore,
+	})
+
+	internalServices := &internalServiceContainer{
+		pub: jsPub,
+	}
+	syncerStats := &syncer.Stats{}
+	wg := &sync.WaitGroup{}
+
+	signalCh, closeCh := createSigChannel()
+	defer closeCh()
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	headSyncer, err := syncer.NewHeadSyncer(syncer.HeadSyncerOpts{
+		Logg:       lo,
+		Pipeline:   pipeline,
+		Pool:       initHeadSyncerWorkerPool(ctx),
+		Stats:      syncerStats,
+		WsEndpoint: ko.MustString("chain.ws_endpoint"),
+	})
+	if err != nil {
+		lo.Fatal("main: crticial error loading head syncer", "error", err)
+	}
+
+	janitor := syncer.NewJanitor(syncer.JanitorOpts{
+		BatchSize:     uint64(ko.MustInt64("syncer.janitor_queue_size")),
+		Logg:          lo,
+		Pipeline:      pipeline,
+		Pool:          initJanitorWorkerPool(ctx),
+		Stats:         syncerStats,
+		Store:         pgStore,
+		SweepInterval: time.Second * time.Duration(ko.MustInt64("syncer.janitor_sweep_interval")),
+	})
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := headSyncer.Start(ctx); err != nil {
+			lo.Info("main: starting head syncer")
+			lo.Fatal("main: critical error starting head syncer", "error", err)
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		lo.Info("main: starting janitor")
+		if err := janitor.Start(ctx); err != nil {
+			lo.Fatal("main: critical error starting janitor", "error", err)
+		}
+	}()
+
+	internalServices.apiService = initApiServer()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		host := ko.MustString("service.address")
+		lo.Info("main: starting API server", "host", host)
+		if err := internalServices.apiService.Start(host); err != nil {
+			if strings.Contains(err.Error(), "Server closed") {
+				lo.Info("main: shutting down server")
+			} else {
+				lo.Fatal("main: critical error shutting down server", "err", err)
+			}
+		}
+	}()
+
+	lo.Info("main: graceful shutdown triggered", "signal", <-signalCh)
+	cancel()
+	startGracefulShutdown(context.Background(), internalServices)
+
+	wg.Wait()
+}

cmd/service/utils.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+func createSigChannel() (chan os.Signal, func()) {
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
+
+	return signalCh, func() {
+		close(signalCh)
+	}
+}
+
+func startGracefulShutdown(ctx context.Context, internalServices *internalServiceContainer) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	internalServices.pub.Close()
+
+	if err := internalServices.apiService.Shutdown(ctx); err != nil {
+		lo.Fatal("Could not gracefully shutdown api server", "err", err)
+	}
+}