From f791b871728a30355f4f72789ca0c9a29fba76a6 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Sat, 14 Jan 2023 09:12:34 +0000 Subject: [PATCH] update: add filters for transfer events and tests --- Makefile | 1 + cmd/filters.go | 5 +- cmd/main.go | 2 +- go.mod | 22 +++- go.sum | 30 ++++-- internal/fetch/graphql_test.go | 24 ++--- internal/filter/address_filter.go | 21 ++-- internal/filter/address_filter_test.go | 77 ++++++++++++++ internal/filter/decode_filter.go | 76 ++++++++++++++ internal/filter/decode_filter_test.go | 77 ++++++++++++++ internal/filter/noop_filter.go | 27 ----- internal/filter/transfer_filter.go | 35 ------- internal/pipeline/pipeline_test.go | 138 ------------------------- 13 files changed, 302 insertions(+), 233 deletions(-) create mode 100644 internal/filter/address_filter_test.go create mode 100644 internal/filter/decode_filter.go create mode 100644 internal/filter/decode_filter_test.go delete mode 100644 internal/filter/noop_filter.go delete mode 100644 internal/filter/transfer_filter.go delete mode 100644 internal/pipeline/pipeline_test.go diff --git a/Makefile b/Makefile index 0ae19ef..2ab9342 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +SHELL := /bin/bash BIN := cic-chain-events .PHONY: build diff --git a/cmd/filters.go b/cmd/filters.go index 088c126..1fedbea 100644 --- a/cmd/filters.go +++ b/cmd/filters.go @@ -10,9 +10,8 @@ func initAddressFilter() filter.Filter { }) } -func initTransferFilter() filter.Filter { - return filter.NewTransferFilter(filter.TransferFilterOpts{ +func initDecodeFilter() filter.Filter { + return filter.NewDecodeFilter(filter.DecodeFilterOpts{ Logg: lo, }) } - diff --git a/cmd/main.go b/cmd/main.go index e58cce5..e2b4db0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,7 +61,7 @@ func main() { BlockFetcher: graphqlFetcher, Filters: []filter.Filter{ initAddressFilter(), - initTransferFilter(), + initDecodeFilter(), }, Logg: lo, Store: pgStore, diff --git a/go.mod b/go.mod index b7cfc58..9b13901 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/alitto/pond v1.8.2 github.com/celo-org/celo-blockchain v1.6.1 github.com/goccy/go-json v0.10.0 + github.com/grassrootseconomics/cic-celo-sdk v0.3.1 github.com/jackc/pgx/v5 v5.2.0 github.com/knadh/goyesql/v2 v2.2.0 github.com/knadh/koanf v1.4.5 @@ -16,28 +17,45 @@ require ( ) require ( + filippo.io/edwards25519 v1.0.0-alpha.2 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/celo-org/celo-bls-go v0.2.4 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect + github.com/deckarep/golang-set v1.8.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-ole/go-ole v1.2.5 // indirect github.com/go-stack/stack v1.8.0 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grassrootseconomics/w3-celo-patch v0.1.0 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.1 // indirect + github.com/huin/goupnp v1.0.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/puddle/v2 v2.1.2 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/labstack/gommon v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/onsi/gomega v1.10.1 // indirect github.com/pelletier/go-toml v1.7.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/tsdb v0.7.1 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -51,6 +69,8 @@ require ( golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.2.0 // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index efe6a8a..c1649e4 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alitto/pond v1.8.2 h1:k0k3GIE7CFLW/kyMJj5DDKLFg1VH09l8skZqg/yJNng= github.com/alitto/pond v1.8.2/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -105,7 +106,6 @@ github.com/celo-org/celo-bls-go v0.2.4 h1:V1y92kM5IRJWQZ6DCwqiKLW7swmUA5y/dPJ9Yb github.com/celo-org/celo-bls-go v0.2.4/go.mod h1:eXUCLXu5F1yfd3M+3VaUk5ZUXaA0sLK2rWdLC1Cfaqo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -127,8 +127,9 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -162,11 +163,13 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -226,6 +229,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -234,8 +238,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -243,6 +247,10 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grassrootseconomics/cic-celo-sdk v0.3.1 h1:SzmMFrqxSIdgePqwbUdoS3PNP82MFnlOecycVk2ZYWg= +github.com/grassrootseconomics/cic-celo-sdk v0.3.1/go.mod h1:EiR6d03GYu6jlVKNL1MbTAw/bqAW2WP3J/lkrZxPMdU= +github.com/grassrootseconomics/w3-celo-patch v0.1.0 h1:0fev2hYkGEyFX2D4oUG8yy4jXhtHv7qUtLLboXL5ycw= +github.com/grassrootseconomics/w3-celo-patch v0.1.0/go.mod h1:JtkXc+yDUiQQJdhYTqddZI/itdYGHY7H8PNZzBo4hCk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= @@ -289,11 +297,13 @@ github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2 github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o= +github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -316,8 +326,9 @@ github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8= github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk= github.com/jackc/puddle/v2 v2.1.2 h1:0f7vaaXINONKTsxYDn4otOAiJanX/BMeAtY//BXqzlg= github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -433,6 +444,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -441,6 +453,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -777,8 +790,9 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -849,6 +863,7 @@ gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= @@ -865,6 +880,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/fetch/graphql_test.go b/internal/fetch/graphql_test.go index 251734d..fcfda5b 100644 --- a/internal/fetch/graphql_test.go +++ b/internal/fetch/graphql_test.go @@ -12,28 +12,28 @@ var ( graphqlEndpoint = os.Getenv("TEST_GRAPHQL_ENDPOINT") ) -type itGraphqlTest struct { +type GraphQlTestSuite struct { suite.Suite - graphqlFetcher Fetch + fetch Fetch } -func TestPipelineSuite(t *testing.T) { - suite.Run(t, new(itGraphqlTest)) -} - -func (s *itGraphqlTest) SetupSuite() { - s.graphqlFetcher = NewGraphqlFetcher(GraphqlOpts{ +func (s *GraphQlTestSuite) SetupSuite() { + s.fetch = NewGraphqlFetcher(GraphqlOpts{ GraphqlEndpoint: graphqlEndpoint, }) } -func (s *itGraphqlTest) Test_E2E_Fetch_Existing_Block() { - resp, err := s.graphqlFetcher.Block(context.Background(), 14974600) +func (s *GraphQlTestSuite) Test_E2E_Fetch_Existing_Block() { + resp, err := s.fetch.Block(context.Background(), 14974600) s.NoError(err) s.Len(resp.Data.Block.Transactions, 3) } -func (s *itGraphqlTest) Test_E2E_Fetch_Non_Existing_Block() { - _, err := s.graphqlFetcher.Block(context.Background(), 14974600000) +func (s *GraphQlTestSuite) Test_E2E_Fetch_Non_Existing_Block() { + _, err := s.fetch.Block(context.Background(), 14974600000) s.Error(err) } + +func TestGraphQlSuite(t *testing.T) { + suite.Run(t, new(GraphQlTestSuite)) +} diff --git a/internal/filter/address_filter.go b/internal/filter/address_filter.go index 2d04eac..172d721 100644 --- a/internal/filter/address_filter.go +++ b/internal/filter/address_filter.go @@ -2,31 +2,34 @@ package filter import ( "context" + "sync" + celo "github.com/grassrootseconomics/cic-celo-sdk" "github.com/grassrootseconomics/cic-chain-events/internal/fetch" "github.com/zerodha/logf" ) -const ( - cUSD = "0x765de816845861e75a25fca122bb6898b8b1282a" -) - type AddressFilterOpts struct { - Logg logf.Logger + Cache *sync.Map + CeloProvider *celo.Provider + Logg logf.Logger } type AddressFilter struct { - logg logf.Logger + cache *sync.Map + logg logf.Logger } func NewAddressFilter(o AddressFilterOpts) Filter { + // TODO: Bootstrap addresses from registry smart contract return &AddressFilter{ - logg: o.Logg, + cache: o.Cache, + logg: o.Logg, } } -func (*AddressFilter) Execute(_ context.Context, transaction fetch.Transaction) (bool, error) { - if transaction.To.Address == cUSD { +func (f *AddressFilter) Execute(_ context.Context, transaction fetch.Transaction) (bool, error) { + if _, found := f.cache.Load(transaction.To.Address); found { return true, nil } diff --git a/internal/filter/address_filter_test.go b/internal/filter/address_filter_test.go new file mode 100644 index 0000000..e8273b1 --- /dev/null +++ b/internal/filter/address_filter_test.go @@ -0,0 +1,77 @@ +package filter + +import ( + "context" + "sync" + "testing" + + "github.com/grassrootseconomics/cic-chain-events/internal/fetch" + "github.com/stretchr/testify/suite" + "github.com/zerodha/logf" +) + +type AddressFilterSuite struct { + suite.Suite + filter Filter +} + +func (s *AddressFilterSuite) SetupSuite() { + addressCache := &sync.Map{} + + addressCache.Store("0x6914ba1c49d3c3f32a9e65a0661d7656cb292e9f", "") + + logg := logf.New( + logf.Opts{ + Level: logf.DebugLevel, + }, + ) + + s.filter = NewAddressFilter(AddressFilterOpts{ + Cache: addressCache, + Logg: logg, + }) +} + +func (s *AddressFilterSuite) TestAddresses() { + type testCase struct { + transactionData fetch.Transaction + want bool + wantErr bool + } + + // Generated with eth-encode + tests := []testCase{ + { + transactionData: fetch.Transaction{ + To: struct { + Address string "json:\"address\"" + }{ + Address: "0x6914ba1c49d3c3f32a9e65a0661d7656cb292e9f", + }, + }, + want: true, + wantErr: false, + }, + { + transactionData: fetch.Transaction{ + To: struct { + Address string "json:\"address\"" + }{ + Address: "0x6914ba1c49d3c3f32a9e65a0661d7656cb292e9x", + }, + }, + want: false, + wantErr: false, + }, + } + + for _, test := range tests { + next, err := s.filter.Execute(context.Background(), test.transactionData) + s.NoError(err) + s.Equal(test.want, next) + } +} + +func TestAddressFilterSuite(t *testing.T) { + suite.Run(t, new(AddressFilterSuite)) +} diff --git a/internal/filter/decode_filter.go b/internal/filter/decode_filter.go new file mode 100644 index 0000000..d96bafa --- /dev/null +++ b/internal/filter/decode_filter.go @@ -0,0 +1,76 @@ +package filter + +import ( + "context" + "math/big" + + "github.com/celo-org/celo-blockchain/common" + "github.com/grassrootseconomics/cic-chain-events/internal/fetch" + "github.com/grassrootseconomics/w3-celo-patch" + "github.com/zerodha/logf" +) + +var ( + transferSig = w3.MustNewFunc("transfer(address, uint256)", "bool") + transferFromSig = w3.MustNewFunc("transferFrom(address, address, uint256)", "bool") + mintToSig = w3.MustNewFunc("mintTo(address, uint256)", "bool") +) + +type DecodeFilterOpts struct { + Logg logf.Logger +} + +type DecodeFilter struct { + logg logf.Logger +} + +func NewDecodeFilter(o DecodeFilterOpts) Filter { + return &DecodeFilter{ + logg: o.Logg, + } +} + +func (f *DecodeFilter) Execute(_ context.Context, transaction fetch.Transaction) (bool, error) { + switch transaction.InputData[:10] { + case "0xa9059cbb": + var ( + to common.Address + value big.Int + ) + + if err := transferSig.DecodeArgs(w3.B(transaction.InputData), &to, &value); err != nil { + return false, err + } + + f.logg.Debug("transfer", "to", to.Hex(), "value", value.String()) + return true, nil + case "0x23b872dd": + var ( + from common.Address + to common.Address + value big.Int + ) + + if err := transferFromSig.DecodeArgs(w3.B(transaction.InputData), &from, &to, &value); err != nil { + return false, err + } + + f.logg.Debug("transferFrom", "from", from.Hex(), "to", to.Hex(), "value", value.String()) + return true, nil + case "0x449a52f8": + var ( + to common.Address + value big.Int + ) + + if err := mintToSig.DecodeArgs(w3.B(transaction.InputData), &to, &value); err != nil { + return false, err + } + + f.logg.Debug("mintTo", "to", to.Hex(), "value", value.String()) + return true, nil + default: + f.logg.Debug("unknownSignature", "inpuData", transaction.InputData) + return false, nil + } +} diff --git a/internal/filter/decode_filter_test.go b/internal/filter/decode_filter_test.go new file mode 100644 index 0000000..c7f48ba --- /dev/null +++ b/internal/filter/decode_filter_test.go @@ -0,0 +1,77 @@ +package filter + +import ( + "context" + "testing" + + "github.com/grassrootseconomics/cic-chain-events/internal/fetch" + "github.com/stretchr/testify/suite" + "github.com/zerodha/logf" +) + +type DecodeFilterSuite struct { + suite.Suite + filter Filter +} + +func (s *DecodeFilterSuite) SetupSuite() { + logg := logf.New( + logf.Opts{ + Level: logf.DebugLevel, + }, + ) + + s.filter = NewDecodeFilter(DecodeFilterOpts{ + Logg: logg, + }) +} + +func (s *DecodeFilterSuite) TestTranfserInputs() { + type testCase struct { + transactionData fetch.Transaction + want bool + wantErr bool + } + + // Generated with eth-encode + tests := []testCase{ + { + transactionData: fetch.Transaction{ + InputData: "0xa9059cbb000000000000000000000000000000000000000000000000000000000000dEaD00000000000000000000000000000000000000000000000000000000000003e8", + }, + want: true, + wantErr: false, + }, + { + transactionData: fetch.Transaction{ + InputData: "0x23b872dd000000000000000000000000000000000000000000000000000000000000dEaD000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000003e8", + }, + want: true, + wantErr: false, + }, + { + transactionData: fetch.Transaction{ + InputData: "0x449a52f8000000000000000000000000000000000000000000000000000000000000dEaD00000000000000000000000000000000000000000000000000000000000003e8", + }, + want: true, + wantErr: false, + }, + { + transactionData: fetch.Transaction{ + InputData: "0x8d72ec9d000000000000000000000000000000000000000000000000000000000000dEaD00000000000000000000000000000000000000000000000000000000000003e8", + }, + want: false, + wantErr: false, + }, + } + + for _, test := range tests { + next, err := s.filter.Execute(context.Background(), test.transactionData) + s.NoError(err) + s.Equal(test.want, next) + } +} + +func TestDecodeFilterSuite(t *testing.T) { + suite.Run(t, new(DecodeFilterSuite)) +} diff --git a/internal/filter/noop_filter.go b/internal/filter/noop_filter.go deleted file mode 100644 index e7ff2d5..0000000 --- a/internal/filter/noop_filter.go +++ /dev/null @@ -1,27 +0,0 @@ -package filter - -import ( - "context" - - "github.com/grassrootseconomics/cic-chain-events/internal/fetch" - "github.com/zerodha/logf" -) - -type NoopFilterOpts struct { - Logg logf.Logger -} - -type NoopFilter struct { - logg logf.Logger -} - -func NewNoopFilter(o NoopFilterOpts) Filter { - return &NoopFilter{ - logg: o.Logg, - } -} - -func (f *NoopFilter) Execute(_ context.Context, transaction fetch.Transaction) (bool, error) { - f.logg.Debug("noop filter", "block", transaction.Block.Number, "index", transaction.Index) - return true, nil -} diff --git a/internal/filter/transfer_filter.go b/internal/filter/transfer_filter.go deleted file mode 100644 index e76e625..0000000 --- a/internal/filter/transfer_filter.go +++ /dev/null @@ -1,35 +0,0 @@ -package filter - -import ( - "context" - - "github.com/grassrootseconomics/cic-chain-events/internal/fetch" - "github.com/zerodha/logf" -) - -type TransferFilterOpts struct { - Logg logf.Logger -} - -type TransferFilter struct { - logg logf.Logger -} - -func NewTransferFilter(o TransferFilterOpts) Filter { - return &TransferFilter{ - logg: o.Logg, - } -} - -func (f *TransferFilter) Execute(_ context.Context, transaction fetch.Transaction) (bool, error) { - switch transaction.InputData[:10] { - case "0xa9059cbb": - f.logg.Info("cUSD transfer", "block", transaction.Block.Number, "index", transaction.Index) - case "0x23b872dd": - f.logg.Info("cUSD transferFrom", "block", transaction.Block.Number, "index", transaction.Index) - default: - f.logg.Info("cUSD otherMethod", "block", transaction.Block.Number, "index", transaction.Index) - } - - return true, nil -} diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go deleted file mode 100644 index 0db04cb..0000000 --- a/internal/pipeline/pipeline_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "os" - "testing" - - "github.com/grassrootseconomics/cic-chain-events/internal/fetch" - "github.com/grassrootseconomics/cic-chain-events/internal/filter" - "github.com/stretchr/testify/suite" - "github.com/zerodha/logf" -) - -var ( - graphqlEndpoint = os.Getenv("TEST_GRAPHQL_ENDPOINT") -) - -type itPipelineTest struct { - suite.Suite - errorPipeline *Pipeline - normalPipeline *Pipeline - earlyExitPipeline *Pipeline -} - -func TestPipelineSuite(t *testing.T) { - suite.Run(t, new(itPipelineTest)) -} - -type errorFilter struct{} - -func newErrorFilter() filter.Filter { - return &errorFilter{} -} - -func (*errorFilter) Execute(_ context.Context, _ fetch.Transaction) (bool, error) { - return false, errors.New("crash") -} - -type earlyExitFilter struct{} - -func newEarlyExitFilter() filter.Filter { - return &earlyExitFilter{} -} - -func (*earlyExitFilter) Execute(_ context.Context, _ fetch.Transaction) (bool, error) { - return false, nil -} - -func (s *itPipelineTest) SetupSuite() { - logger := logf.New( - logf.Opts{ - Level: logf.DebugLevel, - }, - ) - - fetcher := fetch.NewGraphqlFetcher(fetch.GraphqlOpts{ - GraphqlEndpoint: graphqlEndpoint, - }) - - noopFilter := filter.NewNoopFilter(filter.NoopFilterOpts{ - Logg: logger, - }) - errFilter := newErrorFilter() - earlyFilter := newEarlyExitFilter() - - s.errorPipeline = NewPipeline(PipelineOpts{ - Filters: []filter.Filter{ - noopFilter, - errFilter, - }, - BlockFetcher: fetcher, - Logg: logger, - }) - - s.normalPipeline = NewPipeline(PipelineOpts{ - Filters: []filter.Filter{ - noopFilter, - }, - BlockFetcher: fetcher, - Logg: logger, - }) - - s.earlyExitPipeline = NewPipeline(PipelineOpts{ - Filters: []filter.Filter{ - noopFilter, - earlyFilter, - errFilter, - }, - BlockFetcher: fetcher, - Logg: logger, - }) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Existing_Block_No_Err() { - err := s.normalPipeline.Run(context.Background(), 14974600) - s.NoError(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Non_Existing_Block_No_Err() { - err := s.normalPipeline.Run(context.Background(), 14974600000) - s.Error(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Existing_Block_Early() { - err := s.earlyExitPipeline.Run(context.Background(), 14974600) - s.NoError(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Existing_Block_With_Err() { - err := s.errorPipeline.Run(context.Background(), 14974600) - s.Error(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Non_Existing_Block_With_Err() { - err := s.errorPipeline.Run(context.Background(), 14974600000) - s.Error(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Non_Existing_Block_Early() { - err := s.earlyExitPipeline.Run(context.Background(), 14974600000) - s.Error(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Empty_Block_With_No_Err() { - err := s.normalPipeline.Run(context.Background(), 15370320) - s.NoError(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Empty_Block_With_Err() { - err := s.errorPipeline.Run(context.Background(), 15370320) - s.NoError(err) -} - -func (s *itPipelineTest) Test_E2E_Pipeline_Run_On_Empty_Block_Early() { - err := s.earlyExitPipeline.Run(context.Background(), 15370320) - s.NoError(err) -}