Keep the same ports across a restart

Summary:
The sandbox took a fresh ephemeral port on every restart, so anything already
pointed at it had to be repointed. `tao_sandbox_cli start` now binds a fixed
block -- `mock_tao` 18400/18401, `mock_ucache` 18402/18403 -- and refuses to
start if any of it is taken rather than quietly moving to a free port. Why that
block, and the reserved ranges it steers around, is in `SandboxPorts.h`.

Only `/tmp/tao_sandbox` claims the block. The flock is per output directory, so
a `--mock` instance or a test fixture on its own directory would otherwise
fight over it; those keep the ephemeral ports they take today. That is also how
tests isolate: a custom output directory, not a port override. `--random-ports`
is the single escape hatch, restoring the old behaviour wholesale.

`SO_REUSEADDR` is set on the fixed sockets so `TIME_WAIT` from the previous
instance does not block a rebind, while a live listener still fails. Without it
every restart inside the `TIME_WAIT` window would fail to start.

`tao_sandbox_cli start` now bails on a non-zero daemon exit with the tail of
`tao_sandbox.err`; without it a port conflict spins for the full 300s timeout.

Out of scope: `mock_ucache`'s thrift listener, `mock_prod`, and toaster mode's
MySQL/gateway/writer/ucache/wormhole instances stay ephemeral.

Reviewed By: ghostonhuang

Differential Revision: D118218228

fbshipit-source-id: cde007cd942ee45bc66f78349916a0c43839bb58

Author: Vrishal Kulkarni

mcrouter/lib/network/test/ListenSocket.cpp

@@ -27,7 +27,8 @@ namespace memcache {
 
 std::pair<int, uint16_t> createAndBind(
     uint16_t port,
-    bool zeroCopyEnable = false) {
+    bool zeroCopyEnable = false,
+    bool reuseAddr = false) {
   struct addrinfo hints;
   struct addrinfo* res;
 
@@ -53,6 +54,14 @@ std::pair<int, uint16_t> createAndBind(
         port,
         folly::errnoStr(errno));
   }
+  int reuse = 1;
+  if (reuseAddr &&
+      ::setsockopt(socketFd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse))) {
+    auto errStr = folly::errnoStr(errno);
+    ::close(socketFd);
+    throwRuntime("Failed to set SO_REUSEADDR for port {}: {}", port, errStr);
+  }
+
   if (::bind(socketFd, res->ai_addr, res->ai_addrlen) != 0) {
     auto errStr = folly::errnoStr(errno);
     ::close(socketFd);
@@ -78,18 +87,33 @@ std::pair<int, uint16_t> createAndBind(
   return std::make_pair(socketFd, ntohs(addr.sin_port));
 }
 
-ListenSocket::ListenSocket(bool zeroCopyEnabled) {
-  auto sockPort = createAndBind(0, zeroCopyEnabled);
-  socketFd_ = sockPort.first;
-  port_ = sockPort.second;
-  if (::listen(socketFd_, SOMAXCONN) != 0) {
+namespace {
+std::pair<int, uint16_t>
+bindAndListen(uint16_t port, bool zeroCopyEnabled, bool reuseAddr) {
+  auto sockPort = createAndBind(port, zeroCopyEnabled, reuseAddr);
+  if (::listen(sockPort.first, SOMAXCONN) != 0) {
+    auto errStr = folly::errnoStr(errno);
+    ::close(sockPort.first);
     throwRuntime(
         "Failed to listen on a socket for port {}: {}",
-        port_,
-        folly::errnoStr(errno));
+        sockPort.second,
+        errStr);
   }
 
-  VLOG(1) << "Listening on " << socketFd_ << ", port " << port_;
+  VLOG(1) << "Listening on " << sockPort.first << ", port " << sockPort.second;
+  return sockPort;
+}
+} // namespace
+
+ListenSocket::ListenSocket(bool zeroCopyEnabled) {
+  auto sockPort = bindAndListen(0, zeroCopyEnabled, /* reuseAddr */ false);
+  socketFd_ = sockPort.first;
+  port_ = sockPort.second;
+}
+
+ListenSocket ListenSocket::createOnPort(uint16_t port, bool zeroCopyEnabled) {
+  auto sockPort = bindAndListen(port, zeroCopyEnabled, /* reuseAddr */ true);
+  return ListenSocket(sockPort.first, sockPort.second);
 }
 
 void ListenSocket::setCloseOnExec(bool value) {

mcrouter/lib/network/test/ListenSocket.h

@@ -23,6 +23,18 @@ class ListenSocket {
   explicit ListenSocket(bool zeroCopyEnabled = false);
   ~ListenSocket();
 
+  /**
+   * Listen on `port` instead of on an OS-assigned ephemeral one.
+   *
+   * SO_REUSEADDR is set, so a port that a previous process left with
+   * connections in TIME_WAIT can be rebound immediately. That does not permit
+   * a second live listener, so a port another process is listening on still
+   * fails.
+   *
+   * @throws std::runtime_error  if the port cannot be bound
+   */
+  static ListenSocket createOnPort(uint16_t port, bool zeroCopyEnabled = false);
+
   uint16_t getPort() const {
     return port_;
   }
@@ -61,6 +73,9 @@ class ListenSocket {
   ListenSocket& operator=(const ListenSocket&) = delete;
 
  private:
+  ListenSocket(int socketFd, uint16_t port)
+      : socketFd_(socketFd), port_(port) {}
+
   int socketFd_{-1};
   uint16_t port_{0};
 };