Buy crypto anonymously on the Incognito pDEX   

(Solved) Nodes Stalled, sync too fast

#1

Had one of my PCs running several nodes go offline while I was out of town due to a disk space issue. Freed some space, ran the duplicated files cleaner script and then decided to bootstrap since they had been offline for a couple weeks. Beacon would sync but the shards would stall with the following in the logs:

[ERR] Peerv2 log: [stream] rpc error: code = Unknown desc = Sync too fast, last time sync blocks of CID 2 from height 3284678 is 2023-04-29 22:17:26.412307228 +0000 UTC

Was planning on upgrading this box from SSD to NVMe anyways so I nuked it, did a fresh Debian 11 install, setup the 3 vnodes with the latest blink script, bootstrapped the beaon shard for mainnet_0, used the duplicated files cleaner to bootstrap the beacon shard for the other 2 vnodes and then bootstrapped the corresponding shards for all 3 vnodes. Started the docker containers and all three are showing as stalling with the same message above but for 3 different shards. Not sure how to troubleshoot or proceed. Running 20230429_1

#2

Hello @bjeung,

Let’s pull some more configs to get a better picture of what is going on.

On your server run the following command:

sudo docker container inspect inc_mainnet_0

Paste the output as a comment inside a blockquote (so it’s easier to read).

Be sure to redact your mining key and infura data.

#3

[
{
“Id”: “477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17”,
“Created”: “2023-04-29T20:57:53.704087291Z”,
“Path”: “/bin/bash”,
“Args”: [
“run_incognito.sh”
],
“State”: {
“Status”: “running”,
“Running”: true,
“Paused”: false,
“Restarting”: false,
“OOMKilled”: false,
“Dead”: false,
“Pid”: 4240,
“ExitCode”: 0,
“Error”: “”,
“StartedAt”: “2023-04-29T21:58:56.050447685Z”,
“FinishedAt”: “2023-04-29T21:58:38.94453976Z”
},
“Image”: “sha256:dce07da6a62c04ee18e298ddfda59aa44c98335f030e259c1588a7a3f7b1e3ab”,
“ResolvConfPath”: “/var/lib/docker/containers/477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17/resolv.conf”,
“HostnamePath”: “/var/lib/docker/containers/477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17/hostname”,
“HostsPath”: “/var/lib/docker/containers/477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17/hosts”,
“LogPath”: “/var/lib/docker/containers/477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17/477ea2b425b4c7da8b030b90ca5bfcbfda929c69e0577595acf71bdd1c9e2f17-json.log”,
“Name”: “/inc_mainnet_0”,
“RestartCount”: 0,
“Driver”: “overlay2”,
“Platform”: “linux”,
“MountLabel”: “”,
“ProcessLabel”: “”,
“AppArmorProfile”: “docker-default”,
“ExecIDs”: null,
“HostConfig”: {
“Binds”: [
“/home/incognito/node_data_0:/data”
],
“ContainerIDFile”: “”,
“LogConfig”: {
“Type”: “json-file”,
“Config”: {}
},
“NetworkMode”: “inc_net”,
“PortBindings”: {
“8334/tcp”: [
{
“HostIp”: “”,
“HostPort”: “8334”
}
],
“9433/tcp”: [
{
“HostIp”: “”,
“HostPort”: “9433”
}
]
},
“RestartPolicy”: {
“Name”: “always”,
“MaximumRetryCount”: 0
},
“AutoRemove”: false,
“VolumeDriver”: “”,
“VolumesFrom”: null,
“CapAdd”: null,
“CapDrop”: null,
“CgroupnsMode”: “private”,
“Dns”: [],
“DnsOptions”: [],
“DnsSearch”: [],
“ExtraHosts”: null,
“GroupAdd”: null,
“IpcMode”: “private”,
“Cgroup”: “”,
“Links”: null,
“OomScoreAdj”: 0,
“PidMode”: “”,
“Privileged”: false,
“PublishAllPorts”: false,
“ReadonlyRootfs”: false,
“SecurityOpt”: null,
“UTSMode”: “”,
“UsernsMode”: “”,
“ShmSize”: 67108864,
“Runtime”: “runc”,
“ConsoleSize”: [
0,
0
],
“Isolation”: “”,
“CpuShares”: 0,
“Memory”: 0,
“NanoCpus”: 0,
“CgroupParent”: “”,
“BlkioWeight”: 0,
“BlkioWeightDevice”: [],
“BlkioDeviceReadBps”: null,
“BlkioDeviceWriteBps”: null,
“BlkioDeviceReadIOps”: null,
“BlkioDeviceWriteIOps”: null,
“CpuPeriod”: 0,
“CpuQuota”: 0,
“CpuRealtimePeriod”: 0,
“CpuRealtimeRuntime”: 0,
“CpusetCpus”: “”,
“CpusetMems”: “”,
“Devices”: [],
“DeviceCgroupRules”: null,
“DeviceRequests”: null,
“KernelMemory”: 0,
“KernelMemoryTCP”: 0,
“MemoryReservation”: 0,
“MemorySwap”: 0,
“MemorySwappiness”: null,
“OomKillDisable”: null,
“PidsLimit”: null,
“Ulimits”: null,
“CpuCount”: 0,
“CpuPercent”: 0,
“IOMaximumIOps”: 0,
“IOMaximumBandwidth”: 0,
“MaskedPaths”: [
“/proc/asound”,
“/proc/acpi”,
“/proc/kcore”,
“/proc/keys”,
“/proc/latency_stats”,
“/proc/timer_list”,
“/proc/timer_stats”,
“/proc/sched_debug”,
“/proc/scsi”,
“/sys/firmware”
],
“ReadonlyPaths”: [
“/proc/bus”,
“/proc/fs”,
“/proc/irq”,
“/proc/sys”,
“/proc/sysrq-trigger”
]
},
“GraphDriver”: {
“Data”: {
“LowerDir”: “/var/lib/docker/overlay2/86e77055fb8f2f4fe03fcbeb6c442e1e883aec0228c98d173b915b6e785ed5be-init/diff:/var/lib/docker/overlay2/6382d65007d40bf5295863301d74d8278eaafea5f94101edc63bb11e2990c0ea/diff:/var/lib/docker/overlay2/0210d670c681c80ec0851cd7193bc414da7e707bd62d890a32348296f831f7c3/diff:/var/lib/docker/overlay2/30cb46daee92419ea69f9fa04d667440263296fe0420e1943e965b88e39505d1/diff:/var/lib/docker/overlay2/12a399417b5f1cd3e7983bff83c8d097d1fef8eb6c3b63b5b54aebc475d6ed78/diff:/var/lib/docker/overlay2/1063f4e997f87c470d7cf4c1d6b538aa5f248ced360bd33880f444917cfb00d1/diff:/var/lib/docker/overlay2/b0745ea9c7b910adbd40c4da2eb8096c21d72b018e6cbb4b4d649325dfa05b89/diff:/var/lib/docker/overlay2/71c4ff35c9af5960a14d12924f7628de02fccab5d9b8dc41312ce6bb57b7b287/diff:/var/lib/docker/overlay2/ef6d527b543be34bb756a85b1ed4ac784bc5f92dc9c21606c06b493f8f4e94af/diff:/var/lib/docker/overlay2/b6c909b1494c6bfcf9b828c7c3e0aa70b9635187f8b4e0933aa879384d959cc1/diff:/var/lib/docker/overlay2/a3e323c7c5e6b231a7de3e269e232deee3b184fae83259343a0d01821f7faed1/diff:/var/lib/docker/overlay2/635b3195e9ae796fd21d57dc87d9eb4c21e181082e37ae2b801db16f72003234/diff:/var/lib/docker/overlay2/7af15e33cc94f5d99f9f1a54c36497a4619b3559c3c6b7fccd1715bc3b9f4d5a/diff:/var/lib/docker/overlay2/5d338ddb1fc2d2ed43a1287a2be983bcaf21623a22a3edbd20234c9e94b7516e/diff:/var/lib/docker/overlay2/c9ad577cbda25b9b0185771541d7c6b4574c8752c2a0b12253fac5a6fe58d0b1/diff:/var/lib/docker/overlay2/32fe1718c5005d2af660c633aee96fc61528bf01883c42562f1af63787b33c87/diff:/var/lib/docker/overlay2/a6e0a4cfbef244a54f84ad35d92bc50596383d5f0610d409e210988b02da0bec/diff:/var/lib/docker/overlay2/25b8136048b04ae08f55609a180cfef0ec251924840cb310e13e8686affe807e/diff:/var/lib/docker/overlay2/4257384d9ec45d8ebef5e7564c8bbb9f9abb0daa4e054b616e9bd82c9c248e9e/diff:/var/lib/docker/overlay2/42288ec8b3abd64924ad00b58ceae966709365eb90ecd74efb7ce07c7cb36e5e/diff”,
“MergedDir”: “/var/lib/docker/overlay2/86e77055fb8f2f4fe03fcbeb6c442e1e883aec0228c98d173b915b6e785ed5be/merged”,
“UpperDir”: “/var/lib/docker/overlay2/86e77055fb8f2f4fe03fcbeb6c442e1e883aec0228c98d173b915b6e785ed5be/diff”,
“WorkDir”: “/var/lib/docker/overlay2/86e77055fb8f2f4fe03fcbeb6c442e1e883aec0228c98d173b915b6e785ed5be/work”
},
“Name”: “overlay2”
},
“Mounts”: [
{
“Type”: “bind”,
“Source”: “/home/incognito/node_data_0”,
“Destination”: “/data”,
“Mode”: “”,
“RW”: true,
“Propagation”: “rprivate”
}
],
“Config”: {
“Hostname”: “477ea2b425b4”,
“Domainname”: “”,
“User”: “”,
“AttachStdin”: false,
“AttachStdout”: false,
“AttachStderr”: false,
“ExposedPorts”: {
“8334/tcp”: {},
“9433/tcp”: {}
},
“Tty”: false,
“OpenStdin”: false,
“StdinOnce”: false,
“Env”: [
“RPC_PORT=8334”,
“GETH_PROTOCOL=”,
“FULLNODE=”,
“MININGKEY=XXX”,
“NODE_PORT=9433”,
“BOOTNODE_IP=mainnet-bootnode.incognito.org:9330”,
“GETH_NAME=https://mainnet.infura.io/v3/XXX”,
“GETH_PORT=”,
“TESTNET=false”,
“INDEXER_ACCESS_TOKEN=679518c94e928e6223cecdcf354daf67e1cb0c976ebed804066b42f7bc1979aa”,
“NUM_INDEXER_WORKERS=0”,
“PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin”,
“commit=06beebbfa”
],
“Cmd”: [
“/bin/bash”,
“run_incognito.sh”
],
“Image”: “incognitochain/incognito-mainnet:20230429_1”,
“Volumes”: null,
“WorkingDir”: “”,
“Entrypoint”: null,
“OnBuild”: null,
“Labels”: {}
},
“NetworkSettings”: {
“Bridge”: “”,
“SandboxID”: “cc4b8443e9be595f2f04472ba243325941bbd372390e5ee17ee661d22e3cec7c”,
“HairpinMode”: false,
“LinkLocalIPv6Address”: “”,
“LinkLocalIPv6PrefixLen”: 0,
“Ports”: {
“8334/tcp”: [
{
“HostIp”: “0.0.0.0”,
“HostPort”: “8334”
}
],
“9433/tcp”: [
{
“HostIp”: “0.0.0.0”,
“HostPort”: “9433”
}
]
},
“SandboxKey”: “/var/run/docker/netns/cc4b8443e9be”,
“SecondaryIPAddresses”: null,
“SecondaryIPv6Addresses”: null,
“EndpointID”: “”,
“Gateway”: “”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“IPAddress”: “”,
“IPPrefixLen”: 0,
“IPv6Gateway”: “”,
“MacAddress”: “”,
“Networks”: {
“inc_net”: {
“IPAMConfig”: null,
“Links”: null,
“Aliases”: [
“477ea2b425b4”
],
“NetworkID”: “d01de4949ae56cc104b82971923eda9f49b5079ec9cbafff0bcb245000505324”,
“EndpointID”: “fbb9ff93509cd8ebea7773ec3e0e5e31d9f6c7b67e22cc410bb5b790449bc91c”,
“Gateway”: “172.18.0.1”,
“IPAddress”: “172.18.0.2”,
“IPPrefixLen”: 16,
“IPv6Gateway”: “”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“MacAddress”: “02:42:ac:12:00:02”,
“DriverOpts”: null
}
}
}
}
]

#4

Hmm, your configuration appears correct. Are you still experiencing issues with your node?

#5

Yep. Woke up this morning, all 3 nodes still stalled with the same error message. Just restarted all 3 containers and no change

#6

So this seems to be resolved now. I ended up bootstrapping the shards for each vnode and that cleared up the issue. Since this was a brand new box I imagine there was something wrong with the original files that were bootstrapped.

1 Like