bug: 启动stonedb出现 [ERROR] [ha_tianmu.cpp:2244] MSG: An exception error is caught: file TIANMU_INSERT_BUFFER size 536875008 is unexpeceted!.

[kevin]

Have you read the Contributing Guidelines on issues?

• I have read the Contributing Guidelines on issues.

Please confirm if bug report does NOT exists already ?

• I confirm there is no existing issue for this

Describe the problem

[ERROR] [ha_tianmu.cpp:2244] MSG: An exception error is caught: file TIANMU_INSERT_BUFFER size 536875008 is unexpeceted!.

Expected behavior

No response

How To Reproduce

No response

Environment

No response

Are you interested in submitting a PR to solve the problem?

• Yes, I will!

[kevin]

  try {
    std::string log_file = mysql_home_ptr;
    log_setup(log_file + "/log/tianmu.log");
    tianmu_control_.addOutput(new system::FileOut(log_file + "/log/trace.log"));
    tianmu_querylog_.addOutput(new system::FileOut(log_file + "/log/query.log"));

    struct hostent *hent = nullptr;
    hent = gethostbyname(glob_hostname);
    if (hent)
      strmov_str(global_hostIP_, inet_ntoa(*(struct in_addr *)(hent->h_addr_list[0])));

    my_snprintf(global_serverinfo_, sizeof(global_serverinfo_), "\tServerIp:%s\tServerHostName:%s\tServerPort:%d",
                global_hostIP_, glob_hostname, mysqld_port);

    // startup tianmu engine.
    tianmu_hton->data = reinterpret_cast<void *>(new core::Engine());
    if (tianmu_hton->data) {
      ret = reinterpret_cast<core::Engine *>(tianmu_hton->data)->Init(total_ha);
      TIANMU_LOG(LogCtl_Level::INFO,
                 "\n"
                 "------------------------------------------------------------"
                 "----------------------------------"
                 "-------------\n"
                 "    ######  ########  #######  ##     ## ######## ######   "
                 "######   \n"
                 "   ##    ##    ##    ##     ## ####   ## ##       ##    ## "
                 "##    ## \n"
                 "   ##          ##    ##     ## ## ##  ## ##       ##    ## "
                 "##    ## \n"
                 "    ######     ##    ##     ## ##  ## ## ######   ##    ## "
                 "######## \n"
                 "         ##    ##    ##     ## ##   #### ##       ##    ## "
                 "##    ## \n"
                 "   ##    ##    ##    ##     ## ##    ### ##       ##    ## "
                 "##    ## \n"
                 "    ######     ##     #######  ##     ## ######## ######   "
                 "######   \n"
                 "------------------------------------------------------------"
                 "----------------------------------"
                 "-------------\n");
    }

  } catch (std::exception &e) {
    my_message(static_cast<int>(common::ErrorCode::UNKNOWN_ERROR), e.what(), MYF(0));
    TIANMU_LOG(LogCtl_Level::ERROR, "An exception error is caught: %s.", e.what());
  } catch (...) {
    my_message(static_cast<int>(common::ErrorCode::UNKNOWN_ERROR), "An unknown system exception error caught.", MYF(0));
    TIANMU_LOG(LogCtl_Level::ERROR, "An unknown system exception error caught.");
  }

[kevin]

  MappedCircularBuffer(const std::string &file, size_t sz_MB)
      : hdr_size_(sysconf(_SC_PAGE_SIZE)), buf_size_(round_up(sz_MB * 1_MB, hdr_size_)) {
    bool file_exists = false;
    fd_ = ::open(file.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
    if (fd_ < 0) {
      throw std::invalid_argument("failed to open file " + file + ". error " + std::to_string(errno) + ": " +
                                  std::strerror(errno));
    }

    struct stat sb;
    if (::fstat(fd_, &sb) == -1) {
      throw std::invalid_argument("failed to stat file " + file + ". error " + std::to_string(errno) + ": " +
                                  std::strerror(errno));
    }

    if (sb.st_size == 0) {
      if (::ftruncate(fd_, FileSize()) == -1) {
        throw std::invalid_argument("failed to truncate " + file + ". error " + std::to_string(errno) + ": " +
                                    std::strerror(errno));
      }
      TIANMU_LOG(LogCtl_Level::INFO, "created delayed buffer file %s with size %ldMB", file.c_str(), buf_size_ / 1_MB);
    } else {
      // file already exists
      if (sb.st_size == static_cast<decltype(sb.st_size)>(FileSize())) {
        file_exists = true;
        TIANMU_LOG(LogCtl_Level::INFO, "use delayed buffer file %s with size %ldMB", file.c_str(), buf_size_ / 1_MB);
      } else {
        // for now just drop the old file.
        if (::ftruncate(fd_, 0) == -1 || ::ftruncate(fd_, FileSize()) == -1) {
          throw std::invalid_argument("failed to truncate " + file + ". error " + std::to_string(errno) + ": " +
                                      std::strerror(errno));
        }
        TIANMU_LOG(LogCtl_Level::WARN, "old buffer file (size %ldM) purged!!! ", sb.st_size / 1_MB);
      }
    }
    MapFile(file);

    if (file_exists && hdr->version != current_version) {
      throw std::invalid_argument("file " + file + " size " + std::to_string(sb.st_size) + " is unexpeceted!");
    }
    hdr->version = current_version;
    GetStat();
  }

[kevin]

    if (file_exists && hdr->version != current_version) {
      throw std::invalid_argument("file " + file + " size " + std::to_string(sb.st_size) + " is unexpeceted!");
    }

[kevin]

  struct header {
    uint64_t version;
    volatile uint64_t write_offset;
    volatile uint64_t read_offset;
  } * hdr;

[kevin]

简单分析这些场景:

1. 在写入数据时候mysqld进程异常崩溃导致内存映射的文件被写错乱
2. 写入数据的版本更新但是数据没有fsync到磁盘上
3. 业务逻辑问题
4. 客户使用-9信号暴力杀进程导致mysqld无法捕捉信号导致被内核暴力赐死而导致数据异常