Full node synced with Sequencer cannot submit transactions

[jbowen93]

Using Ethermint as the test application

Metamask

[ethjs-query] while formatting outputs from RPC '{"value":{"code":-32603,"data":{"code":-32000,"message":"invalid nonce; got 3, expected 4: invalid sequence: invalid sequence"}}}'

Full Node

eth_getFeeHistory

9:17AM ERR RPC method eth_feeHistory crashed: runtime error: index out of range [0] with length 0
goroutine 1313 [running]:
github.com/ethereum/go-ethereum/rpc.(*callback).call.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:200 +0x89
panic({0x226cee0, 0xc00b115350})
        runtime/panic.go:838 +0x207
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).EthBlockFromTendermint(0xc001f56800, 0xc00ad79680, 0x1)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:382 +0x148d
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).GetBlockByNumber(0xc001f56800, 0xc00b1442c0?, 0xd8?)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:154 +0x59
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).FeeHistory(0xc001f56800, 0x2456ee7?, 0xc008cd8cf0?, {0xc00af436a0, 0x3, 0x4})
        github.com/tharsis/ethermint/rpc/ethereum/backend/feebackend.go:167 +0x2a6
github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth.(*PublicAPI).FeeHistory(0xc0016f6000, 0x0?, 0xc0001da6c0?, {0xc00af436a0, 0x3, 0x4})
        github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/api.go:230 +0x94
reflect.Value.call({0xc0016f2180?, 0xc0000d0670?, 0x7fdd868a1a68?}, {0x23c455d, 0x4}, {0xc00b000680, 0x4, 0x12589d2?})
        reflect/value.go:556 +0x845
reflect.Value.Call({0xc0016f2180?, 0xc0000d0670?, 0x16?}, {0xc00b000680, 0x4, 0x5})
        reflect/value.go:339 +0xbf
github.com/ethereum/go-ethereum/rpc.(*callback).call(0xc00168cba0, {0x2bc7a20?, 0xc00af4be00}, {0xc0080b0230, 0xe}, {0xc00af2f770, 0x3, 0x4e6137?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:206 +0x3f0
github.com/ethereum/go-ethereum/rpc.(*handler).runMethod(0xc00af2dde8?, {0x2bc7a20?, 0xc00af4be00?}, 0xc00b01b570, 0x3?, {0xc00af2f770?, 0x0?, 0x7fdd57dddfff?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:389 +0x45
github.com/ethereum/go-ethereum/rpc.(*handler).handleCall(0xc00af61950, 0xc00b030000, 0xc00b01b570)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:337 +0x239
github.com/ethereum/go-ethereum/rpc.(*handler).handleCallMsg(0xc00af61950, 0xc0000a0c00?, 0xc00b01b570)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:298 +0xd1
github.com/ethereum/go-ethereum/rpc.(*handler).handleMsg.func1(0xc00b030000)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:139 +0x35
github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:226 +0xc5
created by github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:222 +0x8d
 module=geth
9:17AM INF Served eth_feeHistory conn=62.96.59.114:63519 duration=5.440689 err="method handler crashed" module=geth reqid=4959826036196238

eth_sendRawTransaction

9:17AM ERR failed to broadcast tx error=": tx already in mempool" client=json-rpc
9:17AM INF Served eth_sendRawTransaction conn=62.96.59.114:63519 duration=0.81588 err=": tx already in mempool" module=geth reqid=1763619415304

eth_getBlockByNumber

9:17AM ERR RPC method eth_getBlockByNumber crashed: runtime error: index out of range [0] with length 0
goroutine 1424 [running]:
github.com/ethereum/go-ethereum/rpc.(*callback).call.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:200 +0x89
panic({0x226cee0, 0xc0000c7668})
        runtime/panic.go:838 +0x207
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).EthBlockFromTendermint(0xc001f56800, 0xc0016f63c0, 0x0)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:382 +0x148d
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).GetBlockByNumber(0xc001f56800, 0x2545209?, 0x98?)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:154 +0x59
github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth.(*PublicAPI).GetBlockByNumber(0xc0016f6000, 0xc001843dd0?, 0x0)
        github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/api.go:718 +0x153
reflect.Value.call({0xc0016f2300?, 0xc0000d06f8?, 0x7fdd868a1a68?}, {0x23c455d, 0x4}, {0xc001931c20, 0x3, 0x12589d2?})
        reflect/value.go:556 +0x845
reflect.Value.Call({0xc0016f2300?, 0xc0000d06f8?, 0x16?}, {0xc001931c20, 0x3, 0x4})
        reflect/value.go:339 +0xbf
github.com/ethereum/go-ethereum/rpc.(*callback).call(0xc00168ce40, {0x2bc7a20?, 0xc001854f80}, {0xc0000c6fa8, 0x14}, {0xc001851f20, 0x2, 0x4e6137?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:206 +0x3f0
github.com/ethereum/go-ethereum/rpc.(*handler).runMethod(0xc001a2f150?, {0x2bc7a20?, 0xc001854f80?}, 0xc00810a000, 0x2?, {0xc001851f20?, 0x7?, 0x0?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:389 +0x45
github.com/ethereum/go-ethereum/rpc.(*handler).handleCall(0xc001837440, 0xc001851ec0, 0xc00810a000)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:337 +0x239
github.com/ethereum/go-ethereum/rpc.(*handler).handleCallMsg(0xc001837440, 0xc00a324000?, 0xc00810a000)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:298 +0xd1
github.com/ethereum/go-ethereum/rpc.(*handler).handleMsg.func1(0xc001851ec0)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:139 +0x35
github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:226 +0xc5
created by github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:222 +0x8d
 module=geth
9:17AM INF Served eth_getBlockByNumber conn=62.96.59.114:63519 duration=1.36158 err="method handler crashed" module=geth reqid=4959826036196259

eth_getBlockByHash

9:17AM ERR RPC method eth_getBlockByHash crashed: runtime error: index out of range [0] with length 0
goroutine 1506 [running]:
github.com/ethereum/go-ethereum/rpc.(*callback).call.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:200 +0x89
panic({0x226cee0, 0xc0085e80a8})
        runtime/panic.go:838 +0x207
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).EthBlockFromTendermint(0xc001f56800, 0xc0016f65a0, 0x0)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:382 +0x148d
github.com/tharsis/ethermint/rpc/ethereum/backend.(*EVMBackend).GetBlockByHash(0xc001f56800, {0x1, 0x8a, 0x25, 0xf9, 0x36, 0x6c, 0x46, 0x83, 0x2c, ...}, ...)
        github.com/tharsis/ethermint/rpc/ethereum/backend/backend.go:176 +0xaf
github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth.(*PublicAPI).GetBlockByHash(0xc0016f6000, {0x1, 0x8a, 0x25, 0xf9, 0x36, 0x6c, 0x46, 0x83, 0x2c, ...}, ...)
        github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/api.go:712 +0x22b
reflect.Value.call({0xc0016f2280?, 0xc0000d06d8?, 0x7fdd868a1a68?}, {0x23c455d, 0x4}, {0xc00813b4a0, 0x3, 0x12589d2?})
        reflect/value.go:556 +0x845
reflect.Value.Call({0xc0016f2280?, 0xc0000d06d8?, 0x16?}, {0xc00813b4a0, 0x3, 0x4})
        reflect/value.go:339 +0xbf
github.com/ethereum/go-ethereum/rpc.(*callback).call(0xc00168cde0, {0x2bc7a20?, 0xc0081dbe00}, {0xc0000c7830, 0x12}, {0xc00820d950, 0x2, 0x4e6137?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/service.go:206 +0x3f0
github.com/ethereum/go-ethereum/rpc.(*handler).runMethod(0xc00819ca50?, {0x2bc7a20?, 0xc0081dbe00?}, 0xc00810ac40, 0x2?, {0xc00820d950?, 0x455a56?, 0x0?})
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:389 +0x45
github.com/ethereum/go-ethereum/rpc.(*handler).handleCall(0xc00817b560, 0xc00820d8f0, 0xc00810ac40)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:337 +0x239
github.com/ethereum/go-ethereum/rpc.(*handler).handleCallMsg(0xc00817b560, 0xc0000a0c00?, 0xc00810ac40)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:298 +0xd1
github.com/ethereum/go-ethereum/rpc.(*handler).handleMsg.func1(0xc00820d8f0)
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:139 +0x35
github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc.func1()
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:226 +0xc5
created by github.com/ethereum/go-ethereum/rpc.(*handler).startCallProc
        github.com/ethereum/go-ethereum@v1.10.16/rpc/handler.go:222 +0x8d
 module=geth
9:17AM INF Served eth_getBlockByHash conn=62.96.59.114:36191 duration=1.011229 err="method handler crashed" module=geth reqid=4575065975289