Get "upstream connect error or disconnect/reset before headers. reset reason: connection termination" from dispatch_http_call method

[Lugia]

I call a dispatch_http_call in the on_configure stage of my wasm plugin to retrieve the business routing configuration. However, in on_http_call_response, I sometimes receive the error "upstream connect error or disconnect/reset before headers. reset reason: connection termination". I'm sure my upstream server didn't crash, so I'm not sure what's causing this.

Code like this:

fn on_configure(&mut self, _: usize) -> bool {
      if let Some(config_bytes) = self.get_plugin_configuration() {
          if str::from_utf8(&config_bytes).is_err() {
              return false;
          }
          let t: WasmConfiguration = serde_json::from_str(str::from_utf8(&config_bytes).unwrap()).unwrap();
          
          let rsp = self.dispatch_http_call(t.cluster.as_str(), 
              vec![
                      (":method", "GET"),
                      (":path", t.path.as_str()),
                      (":authority", t.host.as_str()),
                      ("accept", "*/*"),
                      ("x-wasm-plugin-config-name", t.name.as_str()),
                  ], 
              None, 
              vec![], 
              Duration::from_secs(5));
          
          self.wasm_configuration = Rc::new(t);

          match rsp {
              Ok(_) => {
                  return true;
              }
              Err(st) => {
                  warn!("http call status: {:?}", st);
                  return false;
              }
          }
      } else {
          return false;
      }
  }
...
...
fn on_http_call_response(
        &mut self,
        _token_id: u32,
        _num_headers: usize,
        _body_size: usize,
        _num_trailers: usize,
    ) {
        let config_bytes: Vec<u8> = match self.get_http_call_response_body(0, _body_size) {
            Some(config_bytes) => config_bytes,
            None => {
                panic!("no http call response");
            }
        };
        let rsp: Result<PluginConfiguration, serde_json::Error> = serde_json::from_str(str::from_utf8(&config_bytes).unwrap());
        match rsp {
            Ok(t)=> {
                self.configuration = Rc::new(t);
            }
            Err(e) => {
                panic!("data:{} err:{}", str::from_utf8(&config_bytes).unwrap(), e)
            }
        }
        
    }

Output:

2023-05-10T06:29:12.167376Z	critical	envoy wasm	wasm log zonesvr-mesh-935388: panicked at 'data:upstream connect error or disconnect/reset before headers. reset reason: connection termination err:expected value at line 1 column 1', src/lib.rs:62:17
2023-05-10T06:29:12.167705Z	error	envoy wasm	Function: proxy_on_http_call_response failed: Uncaught RuntimeError: unreachable
Proxy-Wasm plugin in-VM backtrace:
  0:  0x29f55 - __rust_start_panic
  1:  0x29f38 - rust_panic
  2:  0x29f08 - _ZN3std9panicking20rust_panic_with_hook17hb09154fa23e06c37E
  3:  0x29521 - _ZN3std9panicking19begin_panic_handler28_$u7b$$u7b$closure$u7d$$u7d$17h6091c197f0d08bf0E
  4:  0x29460 - _ZN3std10sys_common9backtrace26__rust_end_short_backtrace17h004afb3e6a867c40E
  5:  0x29b9f - rust_begin_unwind
  6:  0x2b22a - _ZN4core9panicking9panic_fmt17h9e229748e3ae9f9dE
  7:  0xda8e - _ZN70_$LT$route..HttpHeadersRoot$u20$as$u20$proxy_wasm..traits..Context$GT$21on_http_call_response17h85e4c5782c224c75E
  8:  0x1d543 - _ZN10proxy_wasm10dispatcher10Dispatcher21on_http_call_response17h0a7ab49d79b48ea3E
  9:  0x20fcf - proxy_on_http_call_response