Skip to content

Commit 661a329

Browse files
simobenzianeclaude
andauthored
fix FileLogger2: drain the transition queue in the destructor after joining the writer thread (#1198)
FileLogger2::writerLoop() checks `loop` between batches, and ~FileLogger2() sets `loop = false` and joins without draining transitions_queue. A transition pushed after the writer's last swap -- typically the root's RUNNING->IDLE from a haltTree() issued right before the logger is destroyed -- is lost, so the .btlog ends mid-flight. Reproduced with a serve loop that halts the tree on a broken pipe and then resets its logger; under load the last IDLE is missing in a fraction of runs. This extracts the batch write into writeBatch(), adds drainQueue() (swap under the mutex, then write), and calls it in the destructor after the join, before the file is closed. No behaviour change for callers. A new looping test, FileLogger2_DrainsQueueAfterHaltThenImmediateDestroy, ticks a tree to RUNNING, halts it, and destroys the logger immediately, then reads the .btlog back and asserts the root's last transition is IDLE, 300 times per run to exercise the scheduling gap. On the unpatched destructor it fails intermittently under load (3/10 and 10/10 process invocations failed across two load runs); with the fix, 3000/3000 iterations passed under the same load. Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
1 parent e6086f2 commit 661a329

3 files changed

Lines changed: 137 additions & 15 deletions

File tree

‎include/behaviortree_cpp/loggers/bt_file_logger_v2.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22
#include "behaviortree_cpp/loggers/abstract_logger.h"
33

4+
#include <deque>
45
#include <filesystem>
56
#include <memory>
67

@@ -61,6 +62,11 @@ class FileLogger2 : public StatusChangeLogger
6162
std::unique_ptr<Pimpl> _p;
6263

6364
void writerLoop();
65+
// Write one batch of transitions to the file (the loop's own 9-byte layout, one place).
66+
void writeBatch(std::deque<Transition>& transitions);
67+
// Move whatever is queued onto this thread and write it: the destructor's drain after the
68+
// writer thread has joined.
69+
void drainQueue();
6470
};
6571

6672
} // namespace BT

‎src/loggers/bt_file_logger_v2.cpp‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ FileLogger2::~FileLogger2()
100100
_p->loop = false;
101101
_p->queue_cv.notify_one();
102102
_p->writer_thread.join();
103+
// The writer thread checks `loop` between batches; a transition pushed after its last swap
104+
// (a haltTree() right before this destructor is the common case) would otherwise never reach
105+
// the file. Drain it here, on the destroying thread, after the join.
106+
drainQueue();
103107
_p->file_stream.close();
104108
}
105109

@@ -146,22 +150,35 @@ void FileLogger2::writerLoop()
146150
// simple way to pop all the transitions from _p->transitions_queue into transitions
147151
std::swap(transitions, _p->transitions_queue);
148152
}
149-
{
150-
const std::scoped_lock file_lock(_p->file_mutex);
151-
while(!transitions.empty())
152-
{
153-
const auto trans = transitions.front();
154-
std::array<char, 9> write_buffer{};
155-
std::memcpy(write_buffer.data(), &trans.timestamp_usec, 6);
156-
std::memcpy(write_buffer.data() + 6, &trans.node_uid, 2);
157-
std::memcpy(write_buffer.data() + 8, &trans.status, 1);
158-
159-
_p->file_stream.write(write_buffer.data(), 9);
160-
transitions.pop_front();
161-
}
162-
_p->file_stream.flush();
163-
}
153+
writeBatch(transitions);
154+
}
155+
}
156+
157+
void FileLogger2::writeBatch(std::deque<Transition>& transitions)
158+
{
159+
const std::scoped_lock file_lock(_p->file_mutex);
160+
while(!transitions.empty())
161+
{
162+
const auto trans = transitions.front();
163+
std::array<char, 9> write_buffer{};
164+
std::memcpy(write_buffer.data(), &trans.timestamp_usec, 6);
165+
std::memcpy(write_buffer.data() + 6, &trans.node_uid, 2);
166+
std::memcpy(write_buffer.data() + 8, &trans.status, 1);
167+
168+
_p->file_stream.write(write_buffer.data(), 9);
169+
transitions.pop_front();
170+
}
171+
_p->file_stream.flush();
172+
}
173+
174+
void FileLogger2::drainQueue()
175+
{
176+
std::deque<Transition> transitions;
177+
{
178+
std::unique_lock lock(_p->queue_mutex);
179+
std::swap(transitions, _p->transitions_queue);
164180
}
181+
writeBatch(transitions);
165182
}
166183

167184
} // namespace BT

‎tests/gtest_loggers.cpp‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
#include "behaviortree_cpp/loggers/bt_minitrace_logger.h"
1717
#include "behaviortree_cpp/loggers/bt_sqlite_logger.h"
1818

19+
#include <array>
20+
#include <cstdint>
1921
#include <cstdio>
22+
#include <cstring>
2023
#include <filesystem>
2124
#include <fstream>
2225

@@ -192,6 +195,102 @@ TEST_F(LoggerTest, FileLogger2_MultipleTicks)
192195
ASSERT_TRUE(std::filesystem::exists(filepath));
193196
}
194197

198+
// A minimal action that stays RUNNING until halted -- SimpleActionNode/SyncActionNode explicitly
199+
// forbid returning RUNNING, so the drain-race repro below needs a real (if synchronous) action
200+
// node instead.
201+
namespace
202+
{
203+
class AlwaysRunningNode : public BT::ActionNodeBase
204+
{
205+
public:
206+
AlwaysRunningNode(const std::string& name, const BT::NodeConfig& config)
207+
: BT::ActionNodeBase(name, config)
208+
{}
209+
210+
static BT::PortsList providedPorts()
211+
{
212+
return {};
213+
}
214+
215+
BT::NodeStatus tick() override
216+
{
217+
return BT::NodeStatus::RUNNING;
218+
}
219+
220+
void halt() override
221+
{}
222+
};
223+
} // namespace
224+
225+
// FileLogger2's destructor used to join the writer thread without draining
226+
// _p->transitions_queue afterward: a transition pushed after the writer's last swap (typically
227+
// the root's RUNNING->IDLE from a haltTree() issued right before the logger is destroyed) could
228+
// be lost, leaving the .btlog missing its final transition. The window is a scheduling race, not
229+
// deterministic, so this loops many iterations rather than asserting on a single run.
230+
TEST_F(LoggerTest, FileLogger2_DrainsQueueAfterHaltThenImmediateDestroy)
231+
{
232+
BT::BehaviorTreeFactory local_factory;
233+
local_factory.registerNodeType<AlwaysRunningNode>("AlwaysRunning");
234+
235+
const std::string xml_text = R"(
236+
<root BTCPP_format="4">
237+
<BehaviorTree>
238+
<Sequence>
239+
<AlwaysRunning name="ActionA"/>
240+
</Sequence>
241+
</BehaviorTree>
242+
</root>)";
243+
244+
constexpr int kIterations = 300;
245+
246+
for(int i = 0; i < kIterations; i++)
247+
{
248+
auto tree = local_factory.createTreeFromText(xml_text);
249+
const std::string filepath = test_dir + "/drain_" + std::to_string(i) + ".btlog";
250+
const uint16_t root_uid = tree.rootNode()->UID();
251+
252+
{
253+
FileLogger2 logger(tree, filepath);
254+
tree.tickOnce(); // root -> RUNNING, pushed onto the queue
255+
tree.haltTree(); // root -> IDLE, pushed right before the logger is destroyed
256+
} // ~FileLogger2() runs here: join the writer thread, then (with the fix) drain the queue
257+
258+
// Parse the .btlog by hand (format documented in bt_file_logger_v2.h): 18-byte magic
259+
// ("BTCPP4-FileLogger2"), 1-byte protocol, 4-byte XML length, the XML itself, 8-byte first
260+
// timestamp, then a sequence of 9-byte Transition records (6 bytes timestamp, 2 bytes
261+
// node_uid, 1 byte status).
262+
std::ifstream file(filepath, std::ios::binary);
263+
ASSERT_TRUE(file.is_open()) << "iteration " << i;
264+
265+
file.seekg(18 + 1);
266+
int32_t xml_len = 0;
267+
file.read(reinterpret_cast<char*>(&xml_len), sizeof(xml_len));
268+
ASSERT_TRUE(file.good()) << "iteration " << i;
269+
file.seekg(xml_len, std::ios::cur);
270+
file.seekg(8, std::ios::cur); // skip the first timestamp
271+
272+
uint8_t last_root_status = 0xFF;
273+
std::array<char, 9> record{};
274+
while(file.read(record.data(), record.size()))
275+
{
276+
uint16_t uid = 0;
277+
uint8_t status = 0xFF;
278+
std::memcpy(&uid, record.data() + 6, sizeof(uid));
279+
std::memcpy(&status, record.data() + 8, sizeof(status));
280+
if(uid == root_uid)
281+
{
282+
last_root_status = status;
283+
}
284+
}
285+
286+
ASSERT_EQ(last_root_status, static_cast<uint8_t>(NodeStatus::IDLE))
287+
<< "iteration " << i << ": the root's last recorded transition is status "
288+
<< int(last_root_status)
289+
<< ", not IDLE(0) -- the tree's halt never reached the .btlog before the file "
290+
"closed";
291+
}
292+
}
293+
195294
// ============ MinitraceLogger tests ============
196295

197296
TEST_F(LoggerTest, MinitraceLogger_Creation)

0 commit comments

Comments
 (0)