From 128b15407acb376413b668359d3d4da1cb761765 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Wed, 11 May 2022 16:57:56 +0300 Subject: [PATCH] server: init api server - echo (https://echo.labstack.com/) as web framework - carbon for auto date-range parsing - add dashboard sql queries --- cmd/init.go | 3 ++ cmd/main.go | 22 ++++++++++++++- cmd/server.go | 18 ++++++++++++ config.toml | 3 ++ go.mod | 8 ++++++ go.sum | 16 +++++++++++ internal/dashboard/api.go | 29 +++++++++++++++++++ internal/dashboard/charts.go | 54 ++++++++++++++++++++++++++++++++++++ internal/dashboard/util.go | 26 +++++++++++++++++ queries/dashboard.sql | 26 +++++++++++++++++ 10 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 cmd/server.go create mode 100644 internal/dashboard/api.go create mode 100644 internal/dashboard/charts.go create mode 100644 internal/dashboard/util.go diff --git a/cmd/init.go b/cmd/init.go index d504d5c..9a13432 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -20,6 +20,9 @@ type config struct { Postgres string `koanf:"postgres"` Redis string `koanf:"redis"` } + Server struct { + Address string `koanf:"address"` + } Chain struct { RpcProvider string `koanf:"rpc"` TokenRegistry string `koanf:"index"` diff --git a/cmd/main.go b/cmd/main.go index 88fbf1f..88e7da4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "context" cic_net "github.com/grassrootseconomics/cic-go/net" "github.com/jackc/pgx/v4/pgxpool" "github.com/knadh/koanf" @@ -10,6 +11,8 @@ import ( "golang.org/x/sys/unix" "os" "os/signal" + "strings" + "time" ) var ( @@ -65,6 +68,17 @@ func main() { } }() + server := initHTTPServer() + go func() { + if err := server.Start(conf.Server.Address); err != nil { + if strings.Contains(err.Error(), "Server closed") { + log.Info().Msg("shutting down server") + } else { + log.Fatal().Err(err).Msg("could not start server") + } + } + }() + sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP) for { @@ -78,5 +92,11 @@ func main() { } processor.Shutdown() - log.Info().Msg("gracefully shutdown processor and scheduler") + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err = server.Shutdown(ctx) + if err != nil { + log.Fatal().Err(err).Msg("could not shut down server") + } + log.Info().Msg("gracefully shutdown processor, scheduler and server") } diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 0000000..0cf1b7f --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,18 @@ +package main + +import ( + "cic-dw/internal/dashboard" + "github.com/labstack/echo/v4" +) + +func initHTTPServer() *echo.Echo { + server := echo.New() + server.HideBanner = true + + // TODO: Remove after stable release + server.Debug = true + + dashboard.InitDashboardApi(server, db, preparedQueries.dashboard) + + return server +} diff --git a/config.toml b/config.toml index 252d949..f615ca2 100644 --- a/config.toml +++ b/config.toml @@ -2,6 +2,9 @@ postgres = "postgresql://postgres:postgres@127.0.0.1:5432/cic_dw" redis = "redis://127.0.0.1:6379/0" +[server] +address = ":3000" + [chain] index = "0x5A1EB529438D8b3cA943A45a48744f4c73d1f098" rpc = "http://127.0.0.1:8545" diff --git a/go.mod b/go.mod index a2c7d92..f4c91df 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-redis/redis/v8 v8.11.2 // indirect github.com/go-stack/stack v1.8.0 // indirect + github.com/golang-module/carbon/v2 v2.1.6 // indirect github.com/golang/protobuf v1.4.3 // indirect github.com/google/uuid v1.2.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect @@ -37,6 +38,10 @@ require ( github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.11.0 // indirect github.com/jackc/puddle v1.2.1 // indirect + github.com/labstack/echo/v4 v4.7.2 // indirect + github.com/labstack/gommon v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.11 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -47,7 +52,10 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.1 // indirect golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/protobuf v1.25.0 // indirect diff --git a/go.sum b/go.sum index 3b04071..c7d920e 100644 --- a/go.sum +++ b/go.sum @@ -167,6 +167,8 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-module/carbon/v2 v2.1.6 h1:PL7wjvWG+NfUL2hIYOUX68yMi0A9rrCVlJxozldJ1lU= +github.com/golang-module/carbon/v2 v2.1.6/go.mod h1:NF5unWf838+pyRY0o+qZdIwBMkFf7w0hmLIguLiEpzU= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -366,7 +368,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/echo/v4 v4.7.2 h1:Kv2/p8OaQ+M6Ex4eGimg9b9e6icoxA42JSlOR3msKtI= +github.com/labstack/echo/v4 v4.7.2/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o= +github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -386,6 +392,8 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -393,6 +401,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -533,8 +543,10 @@ github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefld github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -633,6 +645,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -690,9 +703,12 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/internal/dashboard/api.go b/internal/dashboard/api.go new file mode 100644 index 0000000..a763929 --- /dev/null +++ b/internal/dashboard/api.go @@ -0,0 +1,29 @@ +package dashboard + +import ( + "github.com/jackc/pgx/v4/pgxpool" + "github.com/labstack/echo/v4" + "github.com/nleof/goyesql" +) + +type api struct { + db *pgxpool.Pool + q goyesql.Queries +} + +func InitDashboardApi(e *echo.Echo, db *pgxpool.Pool, queries goyesql.Queries) { + g := e.Group("/dashboard") + + g.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("api", &api{ + db: db, + q: queries, + }) + return next(c) + } + }) + + g.GET("/new-registrations", handleNewRegistrations) + g.GET("/transactions-count", handleTransactionsCount) +} diff --git a/internal/dashboard/charts.go b/internal/dashboard/charts.go new file mode 100644 index 0000000..781bbc8 --- /dev/null +++ b/internal/dashboard/charts.go @@ -0,0 +1,54 @@ +package dashboard + +import ( + "context" + "github.com/georgysavva/scany/pgxscan" + "github.com/labstack/echo/v4" + "net/http" + "time" +) + +type lineChartRes struct { + X time.Time `db:"x" json:"x"` + Y int `db:"y" json:"y"` +} + +func handleNewRegistrations(c echo.Context) error { + var ( + api = c.Get("api").(*api) + data []lineChartRes + ) + + from, to := parseDateRange(c.QueryParams()) + + rows, err := api.db.Query(context.Background(), api.q["new-user-registrations"], from, to) + if err != nil { + return err + } + + if err := pgxscan.ScanAll(&data, rows); err != nil { + return err + } + + return c.JSON(http.StatusOK, data) +} + +func handleTransactionsCount(c echo.Context) error { + var ( + api = c.Get("api").(*api) + data []lineChartRes + ) + + from, to := parseDateRange(c.QueryParams()) + + rows, err := api.db.Query(context.Background(), api.q["transactions-count"], from, to) + if err != nil { + return err + } + + if err := pgxscan.ScanAll(&data, rows); err != nil { + return err + } + + return c.JSON(http.StatusOK, data) +} diff --git a/internal/dashboard/util.go b/internal/dashboard/util.go new file mode 100644 index 0000000..410d157 --- /dev/null +++ b/internal/dashboard/util.go @@ -0,0 +1,26 @@ +package dashboard + +import ( + "github.com/golang-module/carbon/v2" + "net/url" +) + +func parseDateRange(q url.Values) (string, string) { + var from, to string + + qFrom := q.Get("from") + qTo := q.Get("to") + + parseFrom := carbon.Parse(qFrom) + parseTo := carbon.Parse(qTo) + + if parseFrom.Error != nil || parseTo.Error != nil || qFrom == "" || qTo == "" { + from = carbon.Now().StartOfMonth().ToDateString() + to = carbon.Now().EndOfMonth().ToDateString() + } else { + from = parseFrom.ToDateString() + to = parseTo.ToDateString() + } + + return from, to +} diff --git a/queries/dashboard.sql b/queries/dashboard.sql index e69de29..cce11aa 100644 --- a/queries/dashboard.sql +++ b/queries/dashboard.sql @@ -0,0 +1,26 @@ +-- name: new-user-registrations +-- This query generates a date range and left joins the users table to include days with no registrations +-- Produces x, y results for displaying on a line chart +WITH date_range AS ( + SELECT day::date FROM generate_series($1, $2, INTERVAL '1 day') day +) + +SELECT date_range.day AS x, COUNT(users.id) AS y +FROM date_range +LEFT JOIN users ON date_range.day = CAST(users.date_registered AS date) +GROUP BY date_range.day +ORDER BY date_range.day; + +-- name: transactions-count +-- This query generates a date range and left joins the transactions table to include days with no transactions +-- Produces x, y results for displaying on a line chart +WITH date_range AS ( + SELECT day::date FROM generate_series($1, $2, INTERVAL '1 day') day +) + +SELECT date_range.day AS x, COUNT(transactions.id) AS y +FROM date_range +LEFT JOIN transactions ON date_range.day = CAST(transactions.date_block AS date) +GROUP BY date_range.day +ORDER BY date_range.day; +