TSDB设置10K的BUFF SIZE时，写入的数据不能迭代查询 #249

GarMingLi · 2023-10-13T01:56:27Z

代码如下：

struct fal_flash_dev nor_flash = {
    .name       = NOR_FLASH_DEV_NAME,
    .addr       = 0,
    .len        = 512*1024,    
    .blk_size   = 16*1024,
    .ops        = {init, read, write, erase},
    .write_gran = 1,
};



static fdb_time_t get_db_next_seq(void)
{
    return ++storage_db_write_seq;
}


static void storage_db_init(void)
{
    fdb_tsdb_control(&storage_db, FDB_TSDB_CTRL_SET_LOCK, storage_lock);
    fdb_tsdb_control(&storage_db, FDB_TSDB_CTRL_SET_UNLOCK, storage_unlock);
    bool flag_not_format = false;
    fdb_tsdb_control(&storage_db, FDB_TSDB_CTRL_SET_NOT_FORMAT, &flag_not_format);
    if (fdb_tsdb_init(&storage_db, "storage_tsdb", "storage_tsdb", get_db_next_seq, 10 * 1024, NULL) != FDB_NO_ERR) {
        LOG_ERROR("storage_db init error!!!");
        return;
    }
    fdb_tsdb_control(&storage_db, FDB_TSDB_CTRL_GET_LAST_TIME, &storage_db_write_seq);
}



static bool storage_data_append(void *data, uint32_t len)
{
    struct fdb_blob blob;
    if (fdb_tsl_append(&storage_db, fdb_blob_make(&blob, data, len)) != FDB_NO_ERR) {
        LOG_ERROR("storage data append failed");
        return false;
    }

    return true;
}

static bool storage_query_cb(fdb_tsl_t tsl, void *arg)
{
//...
}


storage_data_append(data1, 7*1024);
storage_data_append(data2, 8*1024);
storage_data_append(data3, 9*1024);

fdb_tsl_iter_by_time(&storage_db, storage_db_read_seq, get_db_curr_seq(), storage_query_cb, &storage_db);

写入数据是成功的，但是不能迭代查询，回调一直不会触发。
如果fdb_tsdb_init函数初始化的buff size为8K，写入的数据少入8K一条，就正常。

The text was updated successfully, but these errors were encountered:

GarMingLi · 2023-10-13T09:06:28Z

写入两个数据还是正常的，当写入到第三个数据，fdb_tsl_iter_by_time就不会触发到回调函数。

do {
    read_tsl(db, &tsl);
    if (tsl.status != FDB_TSL_UNUSED) {
        if ((from <= to && tsl.time >= from && tsl.time <= to)
                || (from > to && tsl.time <= from && tsl.time >= to)) {
            /* iterator is interrupted when callback return true */
            if (cb(&tsl, cb_arg)) {
                goto __exit;
            }
        } else {
            goto __exit;
        }
    }
   
} while ((tsl.addr.index = get_tsl_addr(&sector, &tsl)) != FAILED_ADDR);

上面代码中，tsl.time为-1。

armink · 2023-10-13T09:10:38Z

写入第三个，fdb_tsl_append 有返回错误嘛？

GarMingLi · 2023-10-13T09:17:00Z

写入第三个，fdb_tsl_append 有返回错误嘛？

没有返回错误

static bool storage_data_append(void *data, uint32_t len)
{
    struct fdb_blob blob;
    if (fdb_tsl_append(&storage_db, fdb_blob_make(&blob, data, len)) != FDB_NO_ERR) {
        LOG_ERROR("storage data append failed");
        return false;
    }

    return true;
}

我没有看到有错误的log打印

armink · 2023-10-13T09:22:02Z

要不你单步调试先分析一下？

GarMingLi · 2023-10-13T09:34:42Z

要不你单步调试先分析一下？

我是用移远的opensdk移植的，没有在线debug的功能……；
刚刚测试发现：fdb_tsl_query_count、fdb_tsl_iter_by_time都是不行的，当数据大于三条的时候；但是fdb_tsl_iter这个函数就能把数据迭代出来，打印的tsl->time是正确的序号，tsl->status也都是2（FDB_TSL_WRITE）。
发现情况有点像：[https://github.com//issues/10#issuecomment-653344273]
以前我用flashdb移植到华大的MCU的，也小批量过生产过的，只是以前的业务每条的数据比较小，现在的每条数据几K的大小。

GarMingLi · 2023-10-13T09:39:35Z

我使用的版本是4e56774

GarMingLi · 2023-10-13T11:38:06Z

我把fdb_tsl_iter_by_time函数中：

  /* search the first start TSL address */
  tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  /* search all TSL */

修改成以前的版本：

  /* search the first start TSL address */
  // tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  while (start <= end) {
      tsl.addr.index = start + ((end - start) / 2 + 1) / LOG_IDX_DATA_SIZE * LOG_IDX_DATA_SIZE;
      read_tsl(db, &tsl);
      if (tsl.time < from) {
          start = tsl.addr.index + LOG_IDX_DATA_SIZE;
      } else {
          end = tsl.addr.index - LOG_IDX_DATA_SIZE;
      }
  }
  tsl.addr.index = start;
  /* search all TSL */

就可以工作了。

armink · 2023-10-13T12:35:06Z

以前的版本是具体哪个版本？

GarMingLi · 2023-10-13T13:15:56Z

以前的版本是具体哪个版本？

1.1.2 7062902

armink · 2023-10-13T13:19:10Z

你的分区大小是按照什么样配置的，我按照你的配置试试

GarMingLi · 2023-10-13T13:30:45Z

你的分区大小是按照什么样配置的，我按照你的配置试试

fal_cfg.h:

#define FACTORY_KV_DB_ADDR 0
#define FACTORY_KV_DB_SIZE (32 * 1024)
#define CONFIG_KV_DB_ADDR  (FACTORY_KV_DB_ADDR + FACTORY_KV_DB_SIZE)
#define CONFIG_KV_DB_SIZE  (64 * 1024)
#define FAULT_TS_DB_ADDR   (CONFIG_KV_DB_ADDR + CONFIG_KV_DB_SIZE)
#define FAULT_TS_DB_SIZE   (512 * 1024)

/* ===================== Flash device Configuration ========================= */
extern struct fal_flash_dev nor_flash;

/* flash device table */
#define FAL_FLASH_DEV_TABLE                                                                                            \
    {                                                                                                                  \
        &nor_flash,                                                                                                    \
    }

/* ====================== Partition Configuration ========================== */
#ifdef FAL_PART_HAS_TABLE_CFG
/* partition table */
#define FAL_PART_TABLE                                                                                                 \
    {                                                                                                                  \
        {FAL_PART_MAGIC_WORD, "factory_kvdb", "norflash", FACTORY_KV_DB_ADDR, FACTORY_KV_DB_SIZE, 0},                  \
            {FAL_PART_MAGIC_WORD, "config_kvdb", "norflash", CONFIG_KV_DB_ADDR, CONFIG_KV_DB_SIZE, 0},                 \
            {FAL_PART_MAGIC_WORD, "storage_tsdb", "norflash", FAULT_TS_DB_ADDR, FAULT_TS_DB_SIZE, 0},                   \
    }
#endif /* FAL_PART_HAS_TABLE_CFG */

#endif /* _FAL_CFG_H_ */

port.c

#define FLASH_ERASE_MIN_SIZE (16 * 1024)


static int init(void);
static int read(long offset, uint8_t *buf, size_t size);
static int write(long offset, const uint8_t *buf, size_t size);
static int erase(long offset, size_t size);




// 1.定义 flash 设备

struct fal_flash_dev nor_flash = {
    .name       = NOR_FLASH_DEV_NAME,
    .addr       = 0,
    .len        = 0,    // 初始化时赋值
    .blk_size   = FLASH_ERASE_MIN_SIZE,
    .ops        = {init, read, write, erase},
    .write_gran = 1,
};




static int init(void)
{
    ql_norflash_get_addr((unsigned char *)SYS_FLASH_DB_PARTITION_NAME, (unsigned int *)&nor_flash.addr,
                         (unsigned int *)&nor_flash.len);
    return 1;
}

static int read(long offset, uint8_t *buf, size_t size)
{
    int32_t      ret;
    unsigned int addr;

    addr = nor_flash.addr + offset;
    ret  = ql_norflash_do_read((unsigned char *)SYS_FLASH_DB_PARTITION_NAME, addr, (unsigned int)buf,
                               (unsigned int)size);
    // LOG_DEBUG("[read ret->%d] addr->%08x  offset->%08x  size->%08x", ret, nor_flash.addr, offset, size);
    return ret == 0 ? (int)size : -1;
}

static int write(long offset, const uint8_t *buf, size_t size)
{
    int32_t      ret;
    unsigned int addr;

    addr = nor_flash.addr + offset;
    ret  = ql_norflash_do_write((unsigned char *)SYS_FLASH_DB_PARTITION_NAME, addr, (unsigned int)buf,
                                (unsigned int)size);
    // LOG_DEBUG("[write ret->%d] addr->%08x  offset->%08x  size->%08x", ret, nor_flash.addr, offset, size);
    return ret == 0 ? (int)size : -1;
}

static int erase(long offset, size_t size)
{
    int32_t      ret;
    unsigned int addr;

    addr = nor_flash.addr + offset;
    ret  = ql_norflash_do_erase((unsigned char *)SYS_FLASH_DB_PARTITION_NAME, addr, size);

    return ret == 0 ? (int)size : -1;
}

armink · 2023-10-15T09:53:47Z

我尝试增加你的场景的测试用例，但是还是没有复现你的问题，帮忙确认下咱们用法是否完全一致

https://github.com/armink/FlashDBAutoTestBSP/blob/39b80bfc9c32c43cea9dee4175a28070f5b3e5e2/packages/FlashDB/tests/fdb_tsdb_tc.c#L409-L458

另外，你也可以在

使用 fdb_tsl_iter 迭代时，把 TSL1: 700KB, TSL2: 800KB, TSL3: 900KB 信息
fdb_tsl_query_count 对应的使用条件补充一下

大致如下：

TSL	长度	存储地址	扇区索引（如：0、1、2）
TSL1	700KB	xxyy	xxyy
TSL2	800KB	xxyy	xxyy
TSL3	900KB	xxyy	xxyy

fdb_tsl_query_count 的入参：

from : ?
to : ?

GarMingLi · 2023-10-16T02:22:48Z

我尝试增加你的场景的测试用例，但是还是没有复现你的问题，帮忙确认下咱们用法是否完全一致

https://github.com/armink/FlashDBAutoTestBSP/blob/39b80bfc9c32c43cea9dee4175a28070f5b3e5e2/packages/FlashDB/tests/fdb_tsdb_tc.c#L409-L458

另外，你也可以在

使用 fdb_tsl_iter 迭代时，把 TSL1: 700KB, TSL2: 800KB, TSL3: 900KB 信息

fdb_tsl_query_count 对应的使用条件补充一下

大致如下：

TSL 长度存储地址扇区索引（如：0、1、2）
TSL1 700KB xxyy xxyy
TSL2 800KB xxyy xxyy
TSL3 900KB xxyy xxyy
fdb_tsl_query_count 的入参：

from : ?

to : ?

get_time函数我是封装成 ++seq;
插入3条数据后，from是0，to是3

armink · 2023-10-16T13:33:33Z

++seq 这种方式我测试过也是没有复现

你把 TSL 的地址信息及时间信息 dump 出来再看下？

TSL	长度	存储地址	扇区索引（如：0、1、2）
TSL1	700KB	xxyy	xxyy
TSL2	800KB	xxyy	xxyy
TSL3	900KB	xxyy	xxyy

ZakiLiu · 2023-10-18T07:18:02Z

我把fdb_tsl_iter_by_time函数中：

  /* search the first start TSL address */
  tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  /* search all TSL */

修改成以前的版本：

  /* search the first start TSL address */
  // tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  while (start <= end) {
      tsl.addr.index = start + ((end - start) / 2 + 1) / LOG_IDX_DATA_SIZE * LOG_IDX_DATA_SIZE;
      read_tsl(db, &tsl);
      if (tsl.time < from) {
          start = tsl.addr.index + LOG_IDX_DATA_SIZE;
      } else {
          end = tsl.addr.index - LOG_IDX_DATA_SIZE;
      }
  }
  tsl.addr.index = start;
  /* search all TSL */

就可以工作了。

朱总，我遇到了同样的问题，换回旧版本就函数，就可以工作了，用的time_t的时间戳.

armink · 2023-10-18T07:22:47Z

我把fdb_tsl_iter_by_time函数中：

  /* search the first start TSL address */
  tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  /* search all TSL */

修改成以前的版本：

  /* search the first start TSL address */
  // tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  while (start <= end) {
      tsl.addr.index = start + ((end - start) / 2 + 1) / LOG_IDX_DATA_SIZE * LOG_IDX_DATA_SIZE;
      read_tsl(db, &tsl);
      if (tsl.time < from) {
          start = tsl.addr.index + LOG_IDX_DATA_SIZE;
      } else {
          end = tsl.addr.index - LOG_IDX_DATA_SIZE;
      }
  }
  tsl.addr.index = start;
  /* search all TSL */

就可以工作了。

朱总，我遇到了同样的问题，换回旧版本就函数，就可以工作了，用的time_t的时间戳.

@ZakiLiu 需要你们提供一下具体的测试场景呢，我这边的测试用例可能还没覆盖到你们的场景

ZakiLiu · 2023-10-22T09:25:53Z

我把fdb_tsl_iter_by_time函数中：
  /* search the first start TSL address */
  tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  /* search all TSL */
修改成以前的版本：
  /* search the first start TSL address */
  // tsl.addr.index = search_start_tsl_addr(db, start, end, from, to);
  while (start <= end) {
      tsl.addr.index = start + ((end - start) / 2 + 1) / LOG_IDX_DATA_SIZE * LOG_IDX_DATA_SIZE;
      read_tsl(db, &tsl);
      if (tsl.time < from) {
          start = tsl.addr.index + LOG_IDX_DATA_SIZE;
      } else {
          end = tsl.addr.index - LOG_IDX_DATA_SIZE;
      }
  }
  tsl.addr.index = start;
  /* search all TSL */
就可以工作了。
朱总，我遇到了同样的问题，换回旧版本就函数，就可以工作了，用的time_t的时间戳.
@ZakiLiu 需要你们提供一下具体的测试场景呢，我这边的测试用例可能还没覆盖到你们的场景

朱总，你试试每次查找，from和to都是同一个时间戳的场景。

GarMingLi closed this as completed Oct 13, 2023

GarMingLi reopened this Oct 13, 2023

armink added a commit to armink/FlashDBAutoTestBSP that referenced this issue Oct 15, 2023

【增加】armink/FlashDB#249 场景测试用例

39b80bf

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

TSDB设置10K的BUFF SIZE时，写入的数据不能迭代查询 #249

TSDB设置10K的BUFF SIZE时，写入的数据不能迭代查询 #249

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 15, 2023 •

edited

GarMingLi commented Oct 16, 2023

armink commented Oct 16, 2023

ZakiLiu commented Oct 18, 2023

armink commented Oct 18, 2023

ZakiLiu commented Oct 22, 2023

TSDB设置10K的BUFF SIZE时，写入的数据不能迭代查询 #249

TSDB设置10K的BUFF SIZE时，写入的数据不能迭代查询 #249

Comments

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 13, 2023

GarMingLi commented Oct 13, 2023

armink commented Oct 15, 2023 • edited

GarMingLi commented Oct 16, 2023

armink commented Oct 16, 2023

ZakiLiu commented Oct 18, 2023

armink commented Oct 18, 2023

ZakiLiu commented Oct 22, 2023

armink commented Oct 15, 2023 •

edited