feat (breaking): switch to self contained auto bootstrapper

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.3-bookworm as build
+FROM golang:1.23.0-bookworm as build
 
 ENV CGO_ENABLED=1
 

README.md

@@ -2,21 +2,18 @@
 
 ![GitHub Tag](https://img.shields.io/github/v/tag/grassrootseconomics/eth-indexer)
 
-A lightweight Postgres chain indexer designed to couple with
-[eth-tracker](https://github.com/grassrootseconomics/eth-tracker) to index all
-relevant GE related blockchain data on any EVM chain.
+A lightweight Postgres chain indexer designed to couple with [eth-tracker](https://github.com/grassrootseconomics/eth-tracker) to index all relevant GE related blockchain data on any EVM chain.
 
 ## Getting Started
 
 ### Prerequisites
 
-- Git
-- Docker
-- Postgres server
-- Access to a `eth-tracker` instance
+* Git
+* Docker
+* Postgres server
+* Access to a `eth-tracker` instance
 
-See [docker-compose.yaml](dev/docker-compose.yaml) for an example on how to run
-and deploy a single instance.
+See [docker-compose.yaml](dev/docker-compose.yaml) for an example on how to run and deploy a single instance.
 
 ### 1. Build the Docker image
 
@@ -37,9 +34,7 @@ For an example, see `dev/docker-compose.postgres.yaml`.
 
 ### 3. Update config values
 
-See `.env.example` on how to override default values defined in `config.toml`
-using env variables. Alternatively, mount your own config.toml either during
-build time or Docker runtime.
+See `.env.example` on how to override default values defined in `config.toml` using env variables. Alternatively, mount your own config.toml either during build time or Docker runtime.
 
 ```bash
 # Override only specific config values
@@ -49,10 +44,10 @@ mv .env.example .env
 
 Special env variables:
 
-- DEV=*
+* DEV=*
+
+Refer to [`config.toml`](config.toml) to understand different config value settings.
 
-Refer to [`config.toml`](config.toml) to understand different config value
-settings.
 
 ### 4. Run the indexer
 
@@ -63,4 +58,4 @@ docker compose up
 
 ## License
 
-[AGPL-3.0](LICENSE).
+[AGPL-3.0](LICENSE).

cmd/service/main.go

@@ -12,12 +12,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/api"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/cache"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/handler"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/store"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/sub"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/util"
+	"github.com/grassrootseconomics/eth-indexer/internal/api"
+	"github.com/grassrootseconomics/eth-indexer/internal/cache"
+	"github.com/grassrootseconomics/eth-indexer/internal/handler"
+	"github.com/grassrootseconomics/eth-indexer/internal/store"
+	"github.com/grassrootseconomics/eth-indexer/internal/sub"
+	"github.com/grassrootseconomics/eth-indexer/internal/util"
 	"github.com/grassrootseconomics/ethutils"
 	"github.com/knadh/koanf/v2"
 )

cmd/service/router.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/handler"
-	"github.com/grassrootseconomics/eth-indexer/v2/pkg/router"
+	"github.com/grassrootseconomics/eth-indexer/internal/handler"
+	"github.com/grassrootseconomics/eth-indexer/pkg/router"
 )
 
 func bootstrapRouter(handlerContainer *handler.Handler) *router.Router {
@@ -20,7 +20,7 @@ func bootstrapRouter(handlerContainer *handler.Handler) *router.Router {
 	)
 	router.RegisterRoute(
 		"TRACKER.TOKEN_BURN",
-		handlerContainer.IndexTokenBurn,
+		handlerContainer.IndexTokenMint,
 		handlerContainer.AddToken,
 	)
 	router.RegisterRoute(
@@ -38,15 +38,6 @@ func bootstrapRouter(handlerContainer *handler.Handler) *router.Router {
 		handlerContainer.IndexFaucetGive,
 		handlerContainer.FaucetHealthCheck,
 	)
-	router.RegisterRoute(
-		"TRACKER.OWNERSHIP_TRANSFERRED",
-		handlerContainer.IndexOwnershipChange,
-	)
-
-	router.RegisterRoute(
-		"TRACKER.INDEX_REMOVE",
-		handlerContainer.IndexRemove,
-	)
 
 	return router
 }

config.toml

@@ -5,12 +5,16 @@ go_process = true
 address = ":5002"
 
 [postgres]
-dsn = "postgres://postgres:postgres@127.0.0.1:5433/chain_data"
+dsn = "postgres://postgres:postgres@127.0.0.1:5433/ge_celo_data"
 
 [jetstream]
 endpoint = "nats://127.0.0.1:4222"
-id = "eth-indexer-1"
+id = "celo-indexer-1"
 
 [chain]
 rpc_endpoint = "http://localhost:8545"
 chainid = 1337
+
+[bootstrap]
+# This will bootstrap the cache on which addresses to track
+ge_registries = ["0xE979a64D375F5D363d7cecF3c93B9aFD40Ba9f55"]

dev/docker-compose.yaml

@@ -16,8 +16,8 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5   
-  eth-indexer:
-    image: ghcr.io/grassrootseconomics/eth-indexer:latest
+  celo-indexer:
+    image: celo-indexer:latest
     restart: unless-stopped
     depends_on:
       postgres:

dev/init_db.sql

@@ -1 +1 @@
-CREATE DATABASE chain_data;
+CREATE DATABASE ge_celo_data;

go.mod

@@ -1,14 +1,14 @@
-module github.com/grassrootseconomics/eth-indexer/v2
+module github.com/grassrootseconomics/eth-indexer
 
-go 1.23.3
+go 1.23.0
 
 require (
 	github.com/VictoriaMetrics/metrics v1.35.1
 	github.com/ethereum/go-ethereum v1.14.8
 	github.com/go-chi/chi/v5 v5.1.0
-	github.com/grassrootseconomics/eth-tracker v1.3.0-rc
+	github.com/grassrootseconomics/eth-tracker v1.2.2-rc
 	github.com/grassrootseconomics/ethutils v1.3.0
-	github.com/jackc/pgx/v5 v5.7.1
+	github.com/jackc/pgx/v5 v5.6.0
 	github.com/jackc/tern/v2 v2.2.3
 	github.com/kamikazechaser/common v0.2.0
 	github.com/knadh/goyesql/v2 v2.2.0
@@ -45,8 +45,8 @@ require (
 	github.com/holiman/uint256 v1.3.1 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/lmittmann/tint v1.0.4 // indirect

go.sum

@@ -94,8 +94,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grassrootseconomics/eth-tracker v1.3.0-rc h1:iYe2rwCBrU5O8x0+HSJRjcPT1h68k/uGd3i/cJJQuTQ=
-github.com/grassrootseconomics/eth-tracker v1.3.0-rc/go.mod h1:rLXM5u8FDHnMEdah8ACgo/wfawu4o2sljHGkky2rQKE=
+github.com/grassrootseconomics/eth-tracker v1.2.2-rc h1:71iSlRXMl9fgVUBNuMopGINiVy0Z9Lyx+rps7PTc8OY=
+github.com/grassrootseconomics/eth-tracker v1.2.2-rc/go.mod h1:rLXM5u8FDHnMEdah8ACgo/wfawu4o2sljHGkky2rQKE=
 github.com/grassrootseconomics/ethutils v1.3.0 h1:0uX9HG7EujqoNyueYN2gB40zki50AIdxuNLmU0FZroU=
 github.com/grassrootseconomics/ethutils v1.3.0/go.mod h1:Wuv1VEZrkLIXqTSEYI3Nh9HG/ZHOUQ+U+xvWJ8QtjgQ=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
@@ -106,12 +106,12 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
-github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
-github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
-github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
-github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
+github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
+github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jackc/tern/v2 v2.2.3 h1:UWD24+m3zP7eRSlX9vYg2tb6Bf0V161IdOuo4YWWyd4=
 github.com/jackc/tern/v2 v2.2.3/go.mod h1:EStqJVUowhII9OpCTcZISE1BfpGlwE4oq0oQtHAGuuI=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=

internal/handler/add_contract.go

@@ -47,7 +47,7 @@ func (h *Handler) AddToken(ctx context.Context, event event.Event) error {
 
 	if err := h.chainProvider.Client.CallCtx(
 		ctx,
-		eth.CallFunc(contractAddress, sinkAddressGetter).Returns(&sinkAddress),
+		eth.CallFunc(contractAddress, decimalsGetter).Returns(&tokenDecimals),
 	); err != nil {
 		// This will most likely revert if the contract does not have a sinkAddress
 		// Instead of handling the error we just ignore it and set the value to 0

internal/handler/handler.go

@@ -3,8 +3,8 @@ package handler
 import (
 	"log/slog"
 
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/cache"
-	"github.com/grassrootseconomics/eth-indexer/v2/internal/store"
+	"github.com/grassrootseconomics/eth-indexer/internal/cache"
+	"github.com/grassrootseconomics/eth-indexer/internal/store"
 	"github.com/grassrootseconomics/ethutils"
 )
 

internal/handler/index_remove.go

@@ -1,11 +0,0 @@
-package handler
-
-import (
-	"context"
-
-	"github.com/grassrootseconomics/eth-tracker/pkg/event"
-)
-
-func (h *Handler) IndexRemove(ctx context.Context, event event.Event) error {
-	return h.store.RemoveContractAddress(ctx, event)
-}

internal/handler/ownership.go

@@ -1,11 +0,0 @@
-package handler
-
-import (
-	"context"
-
-	"github.com/grassrootseconomics/eth-tracker/pkg/event"
-)
-
-func (h *Handler) IndexOwnershipChange(ctx context.Context, event event.Event) error {
-	return h.store.InsertOwnershipChange(ctx, event)
-}

internal/store/pg.go

@@ -29,18 +29,15 @@ type (
 	}
 
 	queries struct {
-		InsertTx              string `query:"insert-tx"`
-		InsertTokenTransfer   string `query:"insert-token-transfer"`
-		InsertTokenMint       string `query:"insert-token-mint"`
-		InsertTokenBurn       string `query:"insert-token-burn"`
-		InsertFaucetGive      string `query:"insert-faucet-give"`
-		InsertPoolSwap        string `query:"insert-pool-swap"`
-		InsertPoolDeposit     string `query:"insert-pool-deposit"`
-		InsertOwnershipChange string `query:"insert-ownership-change"`
-		InsertToken           string `query:"insert-token"`
-		InsertPool            string `query:"insert-pool"`
-		RemovePool            string `query:"remove-pool"`
-		RemoveToken           string `query:"remove-token"`
+		InsertTx            string `query:"insert-tx"`
+		InsertTokenTransfer string `query:"insert-token-transfer"`
+		InsertTokenMint     string `query:"insert-token-mint"`
+		InsertTokenBurn     string `query:"insert-token-burn"`
+		InsertFaucetGive    string `query:"insert-faucet-give"`
+		InsertPoolSwap      string `query:"insert-pool-swap"`
+		InsertPoolDeposit   string `query:"insert-pool-deposit"`
+		InsertToken         string `query:"insert-token"`
+		InsertPool          string `query:"insert-pool"`
 	}
 )
 
@@ -202,25 +199,6 @@ func (pg *Pg) InsertPoolDeposit(ctx context.Context, eventPayload event.Event) e
 	})
 }
 
-func (pg *Pg) InsertOwnershipChange(ctx context.Context, eventPayload event.Event) error {
-	return pg.executeTransaction(ctx, func(tx pgx.Tx) error {
-		txID, err := pg.insertTx(ctx, tx, eventPayload)
-		if err != nil {
-			return err
-		}
-
-		_, err = tx.Exec(
-			ctx,
-			pg.queries.InsertOwnershipChange,
-			txID,
-			eventPayload.Payload["previousOwner"].(string),
-			eventPayload.Payload["newOwner"].(string),
-			eventPayload.ContractAddress,
-		)
-		return err
-	})
-}
-
 func (pg *Pg) InsertToken(ctx context.Context, contractAddress string, name string, symbol string, decimals uint8, sinkAddress string) error {
 	return pg.executeTransaction(ctx, func(tx pgx.Tx) error {
 		_, err := tx.Exec(
@@ -249,30 +227,6 @@ func (pg *Pg) InsertPool(ctx context.Context, contractAddress string, name strin
 	})
 }
 
-func (pg *Pg) RemoveContractAddress(ctx context.Context, eventPayload event.Event) error {
-	return pg.executeTransaction(ctx, func(tx pgx.Tx) error {
-		_, err := tx.Exec(
-			ctx,
-			pg.queries.RemovePool,
-			eventPayload.Payload["address"].(string),
-		)
-		if err != nil {
-			return err
-		}
-
-		_, err = tx.Exec(
-			ctx,
-			pg.queries.RemoveToken,
-			eventPayload.Payload["address"].(string),
-		)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-}
-
 func (pg *Pg) insertTx(ctx context.Context, tx pgx.Tx, eventPayload event.Event) (int, error) {
 	var txID int
 	if err := tx.QueryRow(

internal/store/store.go

@@ -15,10 +15,8 @@ type (
 		InsertFaucetGive(context.Context, event.Event) error
 		InsertPoolSwap(context.Context, event.Event) error
 		InsertPoolDeposit(context.Context, event.Event) error
-		InsertOwnershipChange(context.Context, event.Event) error
 		InsertToken(context.Context, string, string, string, uint8, string) error
 		InsertPool(context.Context, string, string, string) error
-		RemoveContractAddress(context.Context, event.Event) error
 		Pool() *pgxpool.Pool
 		Close()
 	}

internal/sub/jetstream.go

@@ -6,7 +6,7 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/grassrootseconomics/eth-indexer/v2/pkg/router"
+	"github.com/grassrootseconomics/eth-indexer/pkg/router"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )

migrations/002_track_ownership_change.sql

@@ -1,7 +0,0 @@
-CREATE TABLE IF NOT EXISTS ownership_change (
-  id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-  tx_id INT REFERENCES tx(id),
-  previous_owner VARCHAR(42) NOT NULL DEFAULT '0x0000000000000000000000000000000000000000',
-  new_owner VARCHAR(42) NOT NULL DEFAULT '0x0000000000000000000000000000000000000000',
-  contract_address VARCHAR(42) NOT NULL DEFAULT '0x0000000000000000000000000000000000000000'
-);

migrations/003_add_removed_status.sql

@@ -1,5 +0,0 @@
-ALTER TABLE tokens
-ADD COLUMN removed BOOLEAN NOT NULL DEFAULT false;
-
-ALTER TABLE pools
-ADD COLUMN removed BOOLEAN NOT NULL DEFAULT false;

queries.sql

@@ -104,18 +104,6 @@ INSERT INTO pool_deposit(
     contract_address
 ) VALUES($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING
 
---name: insert-ownership-change
--- $1: tx_id
--- $2: previous_owner
--- $3: new_owner
--- $4: contract_address
-INSERT INTO ownership_change(
-    tx_id,
-    previous_owner,
-    new_owner,
-    contract_address
-) VALUES($1, $2, $3, $4) ON CONFLICT DO NOTHING
-
 --name: insert-token
 -- $1: contract_address
 -- $2: token_name
@@ -134,16 +122,8 @@ INSERT INTO tokens(
 -- $1: contract_address
 -- $2: pool_name
 -- $3: pool_symbol
-INSERT INTO pools(
+INSERT INTO tokens(
 	contract_address,
     pool_name,
     pool_symbol
 ) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING
-
---name: remove-pool
--- $1: contract_address
-UPDATE pools SET removed = true WHERE contract_address = $1
-
---name: remove-token
--- $1: contract_address
-UPDATE tokens SET removed = true WHERE contract_address = $1