core/amqp_leader: fix cluster cron never starting (#72)

* core: fix cluster cron never starting

amqp_cron has never started a task on any node. Two defects, the second
masked by the first.

1. amqp_leader declared no dependency on kazoo_globals, despite
   amqp_leader_listener gating readiness on kz_nodes:is_up/1. The release
   boot script therefore started amqp_leader and amqp_cron *before*
   kazoo_globals, so the kz_nodes ETS table did not exist yet and
   ets:match/2 raised badarg instead of returning []. The retry loop in
   ready_when_node_is_up/0 only handles is_up -> false, so the badarg
   escaped, burned amqp_leader_proc_sup's restart budget in ~5ms and left
   it dead: transient children are not restarted after a 'shutdown' exit.

   Declaring the dependency fixes the boot order (kazoo_globals 32 ->
   amqp_leader 33 -> amqp_cron 34). No cycle: nothing kazoo_globals
   depends on reaches amqp_leader.

2. With the gate cleared, amqp_cron reached its first AMQP message and
   died on "undefined function amqp_cron:handle_info/3".
   amqp_leader_proc:exec_callback/2 appends [ModState, Leader] to every
   callback's arguments. amqp_cron honours that for handle_cast/3,
   handle_call/4, handle_DOWN/3 and from_leader/3, but exported
   handle_info/2.

Also make kz_nodes:is_up/1 total: it now answers false when the registry
is not running rather than raising, which is what its one caller has
always wanted. The unused expires = '$2' binding is dropped -- it was
matched and discarded, implying a recency check that is not performed.

Verified on a dev node: amqp_cron:status/0 now reports an elected leader,
the pid is stable across minutes, and the boot log contains no ets:match
badarg, no reached_max_restart_intensity and no handle_info crash.

* CONTEXT: name the cluster-membership layer

The glossary had no vocabulary for the layer this bug lives in: no term
for the node registry, for leader election, or for cluster-wide scheduled
work. That gap is plausibly why amqp_leader's dependency on kazoo_globals
went undeclared for so long -- the relationship had no name.

Adds Node, Node registry, Leader and Cluster cron. The Node registry
entry records the semantics kz_nodes actually implements: membership is
liveness, pruned by the expiry sweep, with no separate recency check.

Author: noahmehl

CONTEXT.md

@@ -121,6 +121,27 @@ _Avoid_: switch, PBX
 The RabbitMQ message bus every whapp communicates over.
 _Avoid_: queue, broker
 
+**Node**:
+One Erlang VM running Kazoo, named `<whapp>@<host>`. A node and a cluster
+member are the same thing; a FreeSWITCH box is a media server, never a node.
+_Avoid_: server, instance, box
+
+**Node registry**:
+Every node's view of cluster membership, built from heartbeats over the AMQP
+bus and pruned by an expiry sweep. Membership *is* liveness: a node is up
+exactly while it holds a row, and nothing checks recency separately.
+_Avoid_: node list, cluster state, membership table
+
+**Leader**:
+The single node elected to act for a named leader group, chosen over the AMQP
+bus. Groups elect independently, so one node may lead some groups and not others.
+_Avoid_: master, primary, coordinator
+
+**Cluster cron**:
+Scheduled work owned by exactly one node per cluster — its group's leader —
+rather than run on every node.
+_Avoid_: scheduler, job runner, crontab
+
 **kapi**:
 The typed contract modules that define AMQP messages. Prefix `kapi_`.
 _Avoid_: event schema

core/amqp_cron/src/amqp_cron.erl

@@ -51,7 +51,7 @@
 -export([init/1,
          handle_cast/3,
          handle_call/4,
-         handle_info/2,
+         handle_info/3,
          handle_leader_call/4,
          handle_leader_cast/3,
          handle_DOWN/3,
@@ -301,8 +301,8 @@ handle_call(_Request, _From, State, _Election) ->
 handle_cast(_Msg, State, _Election) ->
     {'noreply', State}.
 
--spec handle_info(any(), state()) -> kz_types:handle_info_ret_state(state()).
-handle_info(_Info, State) ->
+-spec handle_info(any(), state(), any()) -> kz_types:handle_info_ret_state(state()).
+handle_info(_Info, State, _Election) ->
     {'noreply', State}.
 
 -spec terminate(any(), state()) -> 'ok'.

core/amqp_leader/src/amqp_leader.app.src

@@ -8,6 +8,9 @@
                  , stdlib
 
                  , kazoo_amqp
+                 %% amqp_leader_listener gates readiness on kz_nodes:is_up/1,
+                 %% so the node registry must be started before this app.
+                 , kazoo_globals
 
                  , lager
                  ]},

core/kazoo_globals/src/kz_nodes.erl

@@ -125,16 +125,21 @@ start_link() ->
                            ,[]
                            ).
 
+%% @doc Is `Node' a member of the node registry?
+%%
+%% Membership, not heartbeat recency: a node stays "up" until the `expire_nodes'
+%% sweep removes its row, which is the registry's single source of truth for
+%% liveness. This deliberately does not re-check `expires'.
+%%
+%% Answers `false' rather than crashing when the registry itself is not running
+%% yet. Callers poll this during startup (see `amqp_leader_listener'), and a node
+%% whose registry has not started is not up in any sense they can use.
 -spec is_up(node()) -> boolean().
 is_up(Node) ->
-    case ets:match(?MODULE, #kz_node{node = Node
-                                    ,expires = '$2'
-                                    ,_ = '_'
-                                    })
-    of
-        [] -> 'false';
-        [_] -> 'true'
-    end.
+    'undefined' =/= ets:whereis(?MODULE)
+        andalso [] =/= ets:match(?MODULE, #kz_node{node = Node
+                                                  ,_ = '_'
+                                                  }).
 
 -spec node_to_json(kz_term:text() | kz_types:kz_node()) -> kz_json:object().
 node_to_json(NodeName) when is_atom(NodeName) ->