TimeoutError: request timed out when syncing between databases

[Karsten Ohme]

I get:

Uncaught TimeoutError: request timed out
maybeThrowTimeoutError with-timeout-option.js:41
(anonymous function) with-timeout-option.js:97

This is the example code:

import { createOrbitDB } from '@orbitdb/core'
import { create } from 'ipfs-core'
import { v4 as uuidv4 } from 'uuid';
import {EventEmitter, setMaxListeners} from "events";
import IPFSBlockStorage from "@orbitdb/core/src/storage/ipfs-block.js";

// The config will set up a TCP connection when dialling other node.js peers.
// You can find out more about peer connectivity at https://connectivity.libp2p.io/.
const config1 = {
  Addresses: {
    API: '/ip4/127.0.0.1/tcp/0',
    Swarm: ['/ip4/0.0.0.0/tcp/0'],
    Gateway: '/ip4/0.0.0.0/tcp/0',
    NoAnnounce: [
    ]
  },
  Swarm: {
    AddrFilters: null,
  },
  Bootstrap: [],
  Discovery: {
    MDNS: {
      Enabled: true,
      Interval: 0
    }
  },
  Routing: {
    Routers: {
      Type: "dht",
      Parameters: {
        Mode: "auto",
        PublicIPNetwork: false,
        AcceleratedDHTClient: false
      }
    }
  }
}

const config2 = {
  Addresses: {
    API: '/ip4/127.0.0.1/tcp/0',
    Swarm: ['/ip4/0.0.0.0/tcp/0'],
    Gateway: '/ip4/0.0.0.0/tcp/0',
    NoAnnounce: [
    ]
  },
  Swarm: {
    AddrFilters: null,
  },
  Bootstrap: [],
  Discovery: {
    MDNS: {
      Enabled: true,
      Interval: 0
    }
  },
  Routing: {
    Routers: {
      Type: "dht",
      Parameters: {
        Mode: "auto",
        PublicIPNetwork: false,
        AcceleratedDHTClient: false
      }
    }
  }
}

IPFSBlockStorage.defaultTimeout = 3600000
setMaxListeners(0)
EventEmitter.defaultMaxListeners = 0;

const ipfs1 = await create({ preload: { enabled: false }, offline: true, EXPERIMENTAL: { pubsub: true }, config: config1, repo: './ipfs/1' })
const ipfs2 = await create({ preload: { enabled: false }, offline: true, EXPERIMENTAL: { pubsub: true }, config: config2, repo: './ipfs/2' })

// The decentralized nature if IPFS can make it slow for peers to find one
// another. You can speed up a connection between two peers by "dialling-in"
// to one peer from another.
// const ipfs1PeerId = await ipfs1.id()
// await ipfs2.swarm.connect(ipfs1PeerId.id)

const orbitdb1 = await createOrbitDB({ ipfs: ipfs1, id: 'agw1', directory: './orbitdb/1' })
const orbitdb2 = await createOrbitDB({ ipfs: ipfs2, id: 'agw2', directory: './orbitdb/2' })

// This opens a new db. Default db type will be 'events'.
const db1 = await orbitdb1.open('test', {type: 'documents'})

// We connect to the first db using its address. This initiates a
// synchronization of the heads between db1 and db2.
const db2 = await orbitdb2.open(db1.address)


// We write some data to db1. This will not be replicated on db2 until we
// explicitly request these records using db2's iterator or all() convenience
// function.
for (let i=0; i<10; i++) {
  console.log("putting data", i)
  await db1.put({_id: uuidv4(), content: "content "+i})
}

let db2Updated = false

// Listen for the connection of ipfs1 to ipfs2.
// If we want to listen for connections from ipfs2 to ipfs1, add a "join"
// listener to db1.
db2.events.on('join', async (peerId, heads) => {
  // The peerId of the ipfs1 node.
  console.log("peer joining db 2: " + peerId, (await ipfs1.id()).id)
})

// Listen for any updates to db2. This is especially useful when listening for
// new heads that are available on db1.
// If we want to listen for new data on db2, add an "update" listener to db1.
db2.events.on('update', async (entry) => {
  // Full replication is achieved by explicitly retrieving all records from db1.
  console.log("new entry:\n", entry)
  // console.log("All current data:\n")
  // console.log(await db2.all())
  db2Updated = true
})

// wait for db2 to complete updating.
await new Promise((resolve, reject) => {
  setInterval(() => {
    if (db2Updated) {
      resolve()
    }
  }, 1000)
})

// Close db1 and its underlying ipfs peer.
await db1.close()
await orbitdb1.stop()
await ipfs1.stop()

// Close db2 and its underlying ipfs peer.
await db2.close()
await orbitdb2.stop()
await ipfs2.stop()

[Karsten Ohme]

I can add this information from the log trace:

TimeoutError: request timed out
at maybeThrowTimeoutError (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/ipfs-core-utils/src/with-timeout-option.js:41:15)
at file:///home/foobar/Projekte/orbitdbtest/src/node_modules/ipfs-core-utils/src/with-timeout-option.js:97:9
at async Object.get (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/storage/ipfs-block.js:62:19)
at async Object.get (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/storage/composed.js:52:15)
at async get (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/oplog/log.js:128:19)
at async Promise.all (index 0)
at async traverseAndVerify (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/oplog/log.js:264:23)
at async Object.joinEntry (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/oplog/log.js:289:5)
at async task (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/database.js:145:25)
at async file:///home/foobar/Projekte/orbitdbtest/src/node_modules/p-queue/dist/index.js:118:36
Emitted 'error' event at:
at task (file:///home/foobar/Projekte/orbitdbtest/src/node_modules/@orbitdb/core/src/sync.js:227:16)
at async file:///home/foobar/Projekte/orbitdbtest/src/node_modules/p-queue/dist/index.js:118:36 {
code: 'ERR_TIMEOUT'
}

I'm using Node.js v20.11.0

[Hayden Young]

Are the two IPFS nodes finding each other? At a preliminary glance it looks like getting blocks from ipfs is timing out.

[Karsten Ohme]

Yes, they do. Example output (I have added a counter compared to the code above):

putting data 0
orbit.js:97putting data 1
orbit.js:97putting data 2
orbit.js:97putting data 3
orbit.js:97putting data 4
orbit.js:97putting data 5
orbit.js:97putting data 6
orbit.js:97putting data 7
orbit.js:97putting data 8
orbit.js:97putting data 9
orbit.js:116new entry 0:
 
Object {v: 2, id: "/orbitdb/zdpuAsM7GMYLUS4cmdPi5RzQoZ9aYmUXATPxNVmAFNXEgQJ5X", key: "03fc12776196808c19ef5045ddd9d13022dbac7db2db5baf78aeee7ba1306aac23", sig: "3044022010d86740993a29d8dc0423047ce207270a5c86da57…cae4b1c4a4d8803f5d78884a43f1ec4c1cb252de4eae574fc", next: Array(1), ...}
orbit.js:116new entry 1:
 
Object {v: 2, id: "/orbitdb/zdpuAsM7GMYLUS4cmdPi5RzQoZ9aYmUXATPxNVmAFNXEgQJ5X", key: "03fc12776196808c19ef5045ddd9d13022dbac7db2db5baf78aeee7ba1306aac23", sig: "3045022100cacf844a8bf7dadb044abb95da55ab951c61be41…1381d4cf897a4f66363a6b55336635daf2a156b2c04b55ef7", next: Array(1), ...}
orbit.js:116new entry 2:
 
Object {v: 2, id: "/orbitdb/zdpuAsM7GMYLUS4cmdPi5RzQoZ9aYmUXATPxNVmAFNXEgQJ5X", key: "03fc12776196808c19ef5045ddd9d13022dbac7db2db5baf78aeee7ba1306aac23", sig: "30440220694c809d5095d460e868c760ce5eb53b53355277cd…bbf7e46e8fd8fbbf046adf4d74c428e39a3113f4b675241cc", next: Array(1), ...}
orbit.js:116new entry 3:

What I can also see is that increasing the IPFSBlockStorage.defaultTimeout does not solve this. But when increasing the time interval int the promise more nodes are updated before the error is thrown.

[Hayden Young]

Indeed, orbitdb2 is trying to fetch items from orbitdb1 after orbitdb1's ipfs connection is closed which causes orbitdb2 to still be fetching records from orbitdb1 even though there is no connection (and, hence, the underlying ipfs block get times out).

Change db2.events.on to:

db2.events.on('update', async (entry) => {
  // Full replication is achieved by explicitly retrieving all records from db1.
  // console.log("All current data:\n")
  // console.log(await db2.all())
  if ((await db2.all()).length == (await db1.all()).length) {
    db2Updated = true
  }
})

This will ensure db2 is fully in sync with db1 b(in this example) before closing the connection between the two orbitdb instances.

[Karsten Ohme]

Great! Now I see that the await db1.close() is otherwise called too early.