Skip to content

Commit

Permalink
Add MultiGetEntity AttributeGroup API to stress test (#12640)
Browse files Browse the repository at this point in the history 
Summary:
Continuing from #12605, adding AttributeGroup `MultiGetEntity` API to stress tests.

Pull Request resolved: #12640

Test Plan:
**AttributeGroup Tests**

NonBatchOps
```
python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=1 --use_put_entity_one_in=1 --use_multi_get=1
```

BatchOps
```
python3 tools/db_crashtest.py blackbox  --test_batches_snapshots=1 --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=1 --use_put_entity_one_in=1 --use_multi_get=1
```

CfConsistency Test
```
python3 tools/db_crashtest.py blackbox --cf_consistency --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=1 --use_put_entity_one_in=1 --use_multi_get=1
```

**Non-AttributeGroup Tests**

NonBatchOps
```
python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=0 --use_put_entity_one_in=1 --use_multi_get=1
```

BatchOps
```
python3 tools/db_crashtest.py blackbox  --test_batches_snapshots=1 --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=0 --use_put_entity_one_in=1 --use_multi_get=1
```

CfConsistency Test
```
python3 tools/db_crashtest.py blackbox --cf_consistency --max_key=25000000 --write_buffer_size=4194304 --use_attribute_group=0 --use_put_entity_one_in=1 --use_multi_get=1
```

Reviewed By: ltamasi

Differential Revision: D57233931

Pulled By: jaykorean

fbshipit-source-id: 8cea771ac2e5749050bf5319360c6c5aa476d7d5
  • Loading branch information
@jaykorean @facebook-github-bot
jaykorean authored and facebook-github-bot committed May 14, 2024
1 parent b9fc13d commit b4c6956
Show file tree
Hide file tree
Showing 3 changed files with 535 additions and 207 deletions.
180 changes: 128 additions & 52 deletions db_stress_tool/batched_ops_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,10 @@ class BatchedOpsStressTest : public StressTest {
std::array<PinnableWideColumns, num_keys> column_results;
std::array<PinnableAttributeGroups, num_keys> attribute_group_results;

std::string error_msg_header = FLAGS_use_attribute_group
? "GetEntity (AttributeGroup) error"
: "GetEntity error";

for (size_t i = 0; i < num_keys; ++i) {
const std::string key = std::to_string(i) + key_suffix;

Expand All @@ -323,7 +327,8 @@ class BatchedOpsStressTest : public StressTest {
}

if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str());
fprintf(stderr, "%s: %s\n", error_msg_header.c_str(),
s.ToString().c_str());
thread->stats.AddErrors(1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
Expand All @@ -343,9 +348,8 @@ class BatchedOpsStressTest : public StressTest {
: column_results[i].columns();

if (!CompareColumns(columns_to_compare, columns)) {
fprintf(stderr,
"GetEntity error: inconsistent entities for key %s: %s, %s\n",
StringToHex(key_suffix).c_str(),
fprintf(stderr, "%s: inconsistent entities for key %s: %s, %s\n",
error_msg_header.c_str(), StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns_to_compare).c_str(),
WideColumnsToHex(columns).c_str());
}
Expand All @@ -360,20 +364,18 @@ class BatchedOpsStressTest : public StressTest {

if (value.empty() || value[value.size() - 1] != expected) {
fprintf(stderr,
"GetEntity error: incorrect column value for key "
"%s: incorrect column value for key "
"%s, entity %s, column value %s, expected %c\n",
StringToHex(key_suffix).c_str(),
error_msg_header.c_str(), StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str(),
value.ToString(/* hex */ true).c_str(), expected);
}
}

if (!VerifyWideColumns(columns)) {
fprintf(
stderr,
"GetEntity error: inconsistent columns for key %s, entity %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str());
fprintf(stderr, "%s: inconsistent columns for key %s, entity %s\n",
error_msg_header.c_str(), StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str());
}
}
}
Expand Down Expand Up @@ -407,66 +409,140 @@ class BatchedOpsStressTest : public StressTest {

std::array<std::string, num_prefixes> keys;
std::array<Slice, num_prefixes> key_slices;
std::array<PinnableWideColumns, num_prefixes> results;
std::array<Status, num_prefixes> statuses;

for (size_t j = 0; j < num_prefixes; ++j) {
keys[j] = std::to_string(j) + key_suffix;
key_slices[j] = keys[j];
}

db_->MultiGetEntity(read_opts_copy, cfh, num_prefixes, key_slices.data(),
results.data(), statuses.data());
if (FLAGS_use_attribute_group) {
// AttributeGroup MultiGetEntity verification

std::vector<PinnableAttributeGroups> results;
results.reserve(num_prefixes);
for (size_t j = 0; j < num_prefixes; ++j) {
PinnableAttributeGroups attribute_groups;
attribute_groups.emplace_back(cfh);
results.emplace_back(std::move(attribute_groups));
}
db_->MultiGetEntity(read_opts_copy, num_prefixes, key_slices.data(),
results.data());

const WideColumns& cmp_columns = results[0][0].columns();

for (size_t j = 0; j < num_prefixes; ++j) {
const auto& attribute_groups = results[j];
assert(attribute_groups.size() == 1);
const Status& s = attribute_groups[0].status();
if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "MultiGetEntity (AttributeGroup) error: %s\n",
s.ToString().c_str());
thread->stats.AddErrors(1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
} else {
thread->stats.AddGets(1, 1);
}

for (size_t j = 0; j < num_prefixes; ++j) {
const Status& s = statuses[j];
const WideColumns& columns = results[j][0].columns();
if (!CompareColumns(cmp_columns, columns)) {
fprintf(stderr,
"MultiGetEntity (AttributeGroup) error: inconsistent "
"entities for key %s: %s, "
"%s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(cmp_columns).c_str(),
WideColumnsToHex(columns).c_str());
}
if (!columns.empty()) {
// The last character of each column value should be 'j' as a
// decimal digit
const char expected = static_cast<char>('0' + j);

for (const auto& column : columns) {
const Slice& value = column.value();

if (value.empty() || value[value.size() - 1] != expected) {
fprintf(stderr,
"MultiGetEntity (AttributeGroup) error: incorrect "
"column value for key "
"%s, entity %s, column value %s, expected %c\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str(),
value.ToString(/* hex */ true).c_str(), expected);
}
}

if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str());
thread->stats.AddErrors(1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
} else {
thread->stats.AddGets(1, 1);
if (!VerifyWideColumns(columns)) {
fprintf(stderr,
"MultiGetEntity (AttributeGroup) error: inconsistent "
"columns for key %s, "
"entity %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str());
}
}
}
} else {
// Non-AttributeGroup MultiGetEntity verification

std::array<PinnableWideColumns, num_prefixes> results;
std::array<Status, num_prefixes> statuses;

db_->MultiGetEntity(read_opts_copy, cfh, num_prefixes,
key_slices.data(), results.data(), statuses.data());

const WideColumns& cmp_columns = results[0].columns();
const WideColumns& columns = results[j].columns();

if (!CompareColumns(cmp_columns, columns)) {
fprintf(stderr,
"MultiGetEntity error: inconsistent entities for key %s: %s, "
"%s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(cmp_columns).c_str(),
WideColumnsToHex(columns).c_str());
}
for (size_t j = 0; j < num_prefixes; ++j) {
const Status& s = statuses[j];

if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str());
thread->stats.AddErrors(1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
} else {
thread->stats.AddGets(1, 1);
}
const WideColumns& columns = results[j].columns();

if (!columns.empty()) {
// The last character of each column value should be 'j' as a decimal
// digit
const char expected = static_cast<char>('0' + j);
if (!CompareColumns(cmp_columns, columns)) {
fprintf(
stderr,
"MultiGetEntity error: inconsistent entities for key %s: %s, "
"%s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(cmp_columns).c_str(),
WideColumnsToHex(columns).c_str());
}

for (const auto& column : columns) {
const Slice& value = column.value();
if (!columns.empty()) {
// The last character of each column value should be 'j' as a
// decimal digit
const char expected = static_cast<char>('0' + j);

for (const auto& column : columns) {
const Slice& value = column.value();

if (value.empty() || value[value.size() - 1] != expected) {
fprintf(stderr,
"MultiGetEntity error: incorrect column value for key "
"%s, entity %s, column value %s, expected %c\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str(),
value.ToString(/* hex */ true).c_str(), expected);
}
}

if (value.empty() || value[value.size() - 1] != expected) {
if (!VerifyWideColumns(columns)) {
fprintf(stderr,
"MultiGetEntity error: incorrect column value for key "
"%s, entity %s, column value %s, expected %c\n",
"MultiGetEntity error: inconsistent columns for key %s, "
"entity %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str(),
value.ToString(/* hex */ true).c_str(), expected);
WideColumnsToHex(columns).c_str());
}
}

if (!VerifyWideColumns(columns)) {
fprintf(stderr,
"MultiGetEntity error: inconsistent columns for key %s, "
"entity %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str());
}
}
}
}
Expand Down

0 comments on commit b4c6956

Please sign in to comment.