Print non-ok status for multi_ops_txns_stress test (#12660)

Summary:
Currently `assert(s.ok())` does not offer much information to debug.

Pull Request resolved: #12660

Test Plan:
revert #12639 and run `python3 ./tools/db_crashtest.py blackbox --write_policy write_prepared --recycle_log_file_num=1 --threads=1 --test_multiops_txn`

before this PR:
```
Exit Before Killing
stdout:

stderr:
 db_stress: db_stress_tool/multi_ops_txns_stress.cc:1529: void rocksdb::MultiOpsTxnsStressTest::PreloadDb(rocksdb::SharedState*, int, uint32_t, uint32_t, uint32_t, uint32_t): Assertion `s.ok()' failed.
```

after this PR:
```
Exit Before Killing
stdout:

stderr:
 Verification failed: PreloadDB failed: Invalid argument: WriteOptions::disableWAL option is not supported if DBOptions::recycle_log_file_num > 0
db_stress: db_stress_tool/db_stress_test_base.cc:517: void rocksdb::StressTest::ProcessStatus(rocksdb::SharedState*, std::string, rocksdb::Status, bool) const: Assertion `false' failed.
Received signal 6 (Aborted)
```

Reviewed By: ajkr

Differential Revision: D57410819

Pulled By: cbi42

fbshipit-source-id: a03c2202c3fd666eb2f58bae24e0c9e3e6ed4265

db_stress_tool/db_stress_test_base.cc

@@ -505,11 +505,13 @@ Status StressTest::AssertSame(DB* db, ColumnFamilyHandle* cf,
 }
 
 void StressTest::ProcessStatus(SharedState* shared, std::string opname,
-                               Status s) const {
+                               const Status& s,
+                               bool ignore_injected_error) const {
   if (s.ok()) {
     return;
   }
-  if (!s.IsIOError() || !std::strstr(s.getState(), "injected")) {
+  if (!s.IsIOError() || !std::strstr(s.getState(), "injected") ||
+      !ignore_injected_error) {
     std::ostringstream oss;
     oss << opname << " failed: " << s.ToString();
     VerificationAbort(shared, oss.str());

db_stress_tool/db_stress_test_base.h

@@ -245,7 +245,8 @@ class StressTest {
     return Status::NotSupported("TestCustomOperations() must be overridden");
   }
 
-  void ProcessStatus(SharedState* shared, std::string msg, Status s) const;
+  void ProcessStatus(SharedState* shared, std::string msg, const Status& s,
+                     bool ignore_injected_error = true) const;
 
   void VerificationAbort(SharedState* shared, std::string msg) const;
 

db_stress_tool/multi_ops_txns_stress.cc

@@ -1331,14 +1331,16 @@ uint32_t MultiOpsTxnsStressTest::GenerateNextC(ThreadState* thread) {
 }
 
 void MultiOpsTxnsStressTest::ProcessRecoveredPreparedTxnsHelper(
-    Transaction* txn, SharedState*) {
+    Transaction* txn, SharedState* shared) {
   thread_local Random rand(static_cast<uint32_t>(FLAGS_seed));
   if (rand.OneIn(2)) {
     Status s = txn->Commit();
-    assert(s.ok());
+    ProcessStatus(shared, "ProcessRecoveredPreparedTxnsHelper", s,
+                  /*ignore_injected_error=*/false);
   } else {
     Status s = txn->Rollback();
-    assert(s.ok());
+    ProcessStatus(shared, "ProcessRecoveredPreparedTxnsHelper", s,
+                  /*ignore_injected_error=*/false);
   }
 }
 
@@ -1517,22 +1519,23 @@ void MultiOpsTxnsStressTest::PreloadDb(SharedState* shared, int threads,
     WriteBatch wb;
     const auto primary_index_entry = record.EncodePrimaryIndexEntry();
     Status s = wb.Put(primary_index_entry.first, primary_index_entry.second);
-    assert(s.ok());
+    ProcessStatus(shared, "PreloadDB", s, /*ignore_injected_error=*/false);
 
     const auto secondary_index_entry = record.EncodeSecondaryIndexEntry();
     s = wb.Put(secondary_index_entry.first, secondary_index_entry.second);
-    assert(s.ok());
+    ProcessStatus(shared, "PreloadDB", s, /*ignore_injected_error=*/false);
 
     s = txn_db_->Write(wopts, &wb);
     assert(s.ok());
+    ProcessStatus(shared, "PreloadDB", s, /*ignore_injected_error=*/false);
 
     // TODO (yanqin): make the following check optional, especially when data
     // size is large.
     Record tmp_rec;
     tmp_rec.SetB(record.b_value());
     s = tmp_rec.DecodeSecondaryIndexEntry(secondary_index_entry.first,
                                           secondary_index_entry.second);
-    assert(s.ok());
+    ProcessStatus(shared, "PreloadDB", s, /*ignore_injected_error=*/false);
     assert(tmp_rec == record);
 
     existing_a_uniqs[tid].insert(a);