cic-chain-events/internal/syncer/janitor.go

133 lines
3.0 KiB
Go
Raw Normal View History

2023-01-05 12:45:09 +01:00
package syncer
import (
"context"
"time"
"github.com/alitto/pond"
"github.com/grassrootseconomics/cic-chain-events/internal/pipeline"
"github.com/grassrootseconomics/cic-chain-events/internal/store"
"github.com/jackc/pgx/v5"
"github.com/zerodha/logf"
)
const (
headBlockLag = 5
)
type (
JanitorOpts struct {
BatchSize uint64
Logg logf.Logger
Pipeline *pipeline.Pipeline
Pool *pond.WorkerPool
Stats *Stats
Store store.Store[pgx.Rows]
SweepInterval time.Duration
}
2023-01-05 12:45:09 +01:00
Janitor struct {
batchSize uint64
pipeline *pipeline.Pipeline
logg logf.Logger
pool *pond.WorkerPool
stats *Stats
store store.Store[pgx.Rows]
sweepInterval time.Duration
}
)
2023-01-05 12:45:09 +01:00
func NewJanitor(o JanitorOpts) *Janitor {
return &Janitor{
batchSize: o.BatchSize,
logg: o.Logg,
pipeline: o.Pipeline,
pool: o.Pool,
stats: o.Stats,
store: o.Store,
sweepInterval: o.SweepInterval,
}
}
func (j *Janitor) Start(ctx context.Context) error {
ticker := time.NewTicker(j.sweepInterval)
2023-01-05 12:45:09 +01:00
for {
select {
case <-ctx.Done():
j.logg.Info("janitor: shutdown signal received")
return nil
case <-ticker.C:
ctx, cancel := context.WithTimeout(context.Background(), jobTimeout)
defer cancel()
j.logg.Debug("janitor: starting sweep")
if err := j.QueueMissingBlocks(ctx); err != nil {
j.logg.Error("janitor: queue missing blocks error", "error", err)
2023-01-05 12:45:09 +01:00
}
}
}
}
// QueueMissingBlocks searches for missing block and queues the block for processing.
// It will run twice for a given search range and only after, raise the lower bound.
func (j *Janitor) QueueMissingBlocks(ctx context.Context) error {
2023-01-05 12:45:09 +01:00
if j.stats.GetHeadCursor() == 0 {
j.logg.Warn("janitor: (skipping) awaiting head synchronization")
2023-01-05 12:45:09 +01:00
return nil
}
if j.pool.WaitingTasks() != 0 {
j.logg.Debug("janitor: (skipping) queue has pending jobs", "pending_jobs", j.pool.WaitingTasks())
2023-01-05 12:45:09 +01:00
return nil
}
lowerBound, upperBound, err := j.store.GetSearchBounds(
ctx,
2023-01-05 12:45:09 +01:00
j.batchSize,
j.stats.GetHeadCursor(),
headBlockLag,
2023-01-05 12:45:09 +01:00
)
if err != nil {
return err
}
rows, err := j.store.GetMissingBlocks(ctx, lowerBound, upperBound)
2023-01-05 12:45:09 +01:00
if err != nil {
return err
}
missingBlockCountReport := j.stats.GetHeadCursor() - lowerBound
if missingBlockCountReport > 10 {
2023-03-09 09:33:09 +01:00
j.logg.Info("janitor: missing blocks", "count", missingBlockCountReport)
}
2023-01-05 12:45:09 +01:00
rowsProcessed := 0
for rows.Next() {
var blockNumber uint64
if err := rows.Scan(&blockNumber); err != nil {
2023-01-05 12:45:09 +01:00
return err
}
j.pool.Submit(func() {
if err := j.pipeline.Run(ctx, blockNumber); err != nil {
j.logg.Error("janitor: pipeline run error", "error", err)
2023-01-05 12:45:09 +01:00
}
})
rowsProcessed++
}
j.logg.Debug("janitor: missing blocks to be processed", "count", rowsProcessed)
2023-01-05 12:45:09 +01:00
if rowsProcessed == 0 {
j.logg.Info("janitor: no gap! rasing lower bound", "new_lower_bound", upperBound)
2023-01-05 12:45:09 +01:00
j.stats.UpdateLowerBound(upperBound)
j.store.SetSearchLowerBound(ctx, upperBound)
2023-01-05 12:45:09 +01:00
}
if rows.Err() != nil {
return err
}
return nil
}