docs: add jetstream configs and instructions

* add Makefile to replace bash script

.gitignore

@@ -1,3 +1,4 @@
 covprofile
 .env
 .env.test
+cic-chain-events

Makefile

@@ -0,0 +1,17 @@
+BIN := cic-chain-events
+
+.PHONY: build
+
+build:
+	CGO_ENABLED=1 go build -v -ldflags="-s -w" -o ${BIN} cmd/*.go
+
+run:
+	CGO_ENABLED=1 go run -ldflags="-s -w" cmd/*.go
+
+mod:
+	go mod tidy
+	go mod verify
+
+test:
+	source .env.test
+	go test -v -covermode atomic -coverprofile=covprofile ./internal/...

docker-compose.yaml

@@ -17,6 +17,17 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+  nats:
+    image: nats:2.9
+    restart: unless-stopped
+    command: "-js -sd /nats/data"
+    volumes:
+      - cic-indexer-nats:/nats/data
+    ports:
+      - '4222:4222'
+      - '8222:8222'
 volumes:
   cic-indexer-pg:
     driver: local
+  cic-indexer-nats:
+    driver: local    

docs/jetsream.md

@@ -0,0 +1,56 @@
+## NATS JetStream
+
+### Server setup
+
+- Enable `-js -sd`
+
+## Stream setup
+
+```go
+_, err = js.AddStream(&nats.StreamConfig{
+	Name:       streamName,
+    // Remove from JS once Acked (Should not be used with 1 consumer which acts as a  relayer e.g. Benthos).
+	// Retention:  nats.WorkQueuePolicy,
+    // MaxAge allows us to replay it within 48 hrs
+    MaxAge:     time.Hour * 48,
+	Storage:    nats.FileStorage,
+	Subjects:   []string{streamSubjects},
+    // Sliding window dedup period.
+	Duplicates: time.Minute * 5,
+})
+```
+
+## Producer
+
+```go
+// nats.MsgId is the unique identifier for dedup
+ctx.Publish("*", []byte("*"), nats.MsgId("*"))
+```
+
+## Consumer setup
+
+- Explicit ACK
+- Durable
+- Deliver: all
+
+### Benthos example
+
+```toml
+input:
+  label: jetstream
+  nats_jetstream:
+    urls:
+      - nats://127.0.0.1:4222
+    subject: "*"
+    durable: benthos
+    deliver: all
+  output:
+  stdout:
+    codec: lines
+```
+
+### Replay example
+
+```bash
+nats sub "*" --all --start-sequence=$N
+```

run_tests.sh

@@ -1,4 +0,0 @@
-#! /bin/bash
-set -e
-
-source .env.test && go test -v -covermode atomic -coverprofile=covprofile ./internal/...