feat(metrics): extend node_info and add node status gauges (#7049)

Author: warku123

common/src/main/java/org/tron/common/prometheus/MetricKeys.java

@@ -42,6 +42,12 @@ public static class Gauge {
     public static final String DB_SST_LEVEL = "tron:db_sst_level";
     public static final String MANAGER_QUEUE = "tron:manager_queue_size";
     public static final String TX_CACHE = "tron:tx_cache";
+    // per local consensus witness miner; value 1, label is the Base58Check witness address
+    public static final String WITNESS_ADDRESS = "tron:witness_address";
+    // numeric backup-election role of this node: 0 = init, 1 = master, 2 = slaver
+    public static final String NODE_ROLE = "tron:node_role";
+    // witness flag of this node: 0 = not witness, 1 = witness
+    public static final String IS_WITNESS = "tron:is_witness";
 
     private Gauge() {
       throw new IllegalStateException("Gauge");

common/src/main/java/org/tron/common/prometheus/MetricLabels.java

@@ -62,6 +62,9 @@ public static class Gauge {
 
     public static final String PEERS_VALID = "valid";
 
+    // ADDRESS is the label of tron:witness_address (node_role and is_witness carry no label)
+    public static final String ADDRESS = "address";
+
     private Gauge() {
       throw new IllegalStateException("Gauge");
     }

common/src/main/java/org/tron/common/prometheus/MetricsGauge.java

@@ -19,6 +19,15 @@ class MetricsGauge {
     init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron  db  size .", "type", "db", "level");
     init(MetricKeys.Gauge.DB_SST_LEVEL, "tron  db  files .", "type", "db", "level");
     init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type");
+    init(MetricKeys.Gauge.WITNESS_ADDRESS,
+        "1 for each consensus witness address this node holds a miner for.",
+        MetricLabels.Gauge.ADDRESS);
+    init(MetricKeys.Gauge.IS_WITNESS,
+        "Whether this node runs in witness mode, encoded as a number: 0 = not witness, "
+            + "1 = witness.");
+    init(MetricKeys.Gauge.NODE_ROLE,
+        "Backup-election role of this node, encoded as a number: 0 = init, "
+            + "1 = master, 2 = slaver.");
   }
 
   private MetricsGauge() {

common/src/main/java/org/tron/common/prometheus/SRMetrics.java

@@ -10,6 +10,39 @@ private SRMetrics() {
     throw new IllegalStateException("SRMetrics");
   }
 
+  // is_witness: 0 = not witness, 1 = witness
+  public static final int NOT_WITNESS = 0;
+  public static final int WITNESS = 1;
+
+  // node_role: 0 = init (pre-election / re-election), 1 = master, 2 = slaver
+  public static final int NODE_ROLE_INIT = 0;
+  public static final int NODE_ROLE_MASTER = 1;
+  public static final int NODE_ROLE_SLAVER = 2;
+
+  /** Publishes is_witness as one label-less series; 0 = not witness, 1 = witness. */
+  public static void recordIsWitness(int isWitness) {
+    if (!Metrics.enabled()) {
+      return;
+    }
+    Metrics.gaugeSet(MetricKeys.Gauge.IS_WITNESS, isWitness);
+  }
+
+  /** Publishes node_role as one label-less series; role changes rewrite it in place. */
+  public static void recordNodeRole(int role) {
+    if (!Metrics.enabled()) {
+      return;
+    }
+    Metrics.gaugeSet(MetricKeys.Gauge.NODE_ROLE, role);
+  }
+
+  /** Publishes one tron:witness_address series (value 1) for a local witness address. */
+  public static void recordWitnessAddress(String address) {
+    if (!Metrics.enabled()) {
+      return;
+    }
+    Metrics.gaugeSet(MetricKeys.Gauge.WITNESS_ADDRESS, 1, address);
+  }
+
   public static void recordSrSetChange(List<ByteString> currentWits, List<ByteString> newWits) {
     if (!Metrics.enabled()) {
       return;

docs/metrics-changelog.md

@@ -10,6 +10,9 @@ This file tracks Prometheus metric additions, changes, and removals in java-tron
 #### Core
 
 - `tron:node_info` (Info, labels `version`, `genesis_block_id`) — static node identity: the running node version string plus the full genesis block hash as the canonical chain identifier, for fleet dashboards and version/chain alert rules. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
+- `tron:is_witness` (Gauge, no labels) — whether this node runs in witness mode, encoded as a number: `0` = `not witness` (started without witness mode; witness keys are not loaded and no `tron:witness_address` series is published), `1` = `witness` (started with `-w`). It is a process identity, published once at startup and independent of the backup-election role. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
+- `tron:node_role` (Gauge, no labels) — backup-election role of this node, encoded as a number: `0` = `init` (backup election configured but not yet run, and again while a re-election is running after the master lost keep-alive), `1` = `master` (the backup primary; also the standalone default for both node types when backup is not configured), `2` = `slaver` (standing by for a master). The role is a node-level status, independent of `tron:is_witness`, so the metric is a single series: a role change rewrites its value, so no stale series remain. Refreshed at startup and on every backup-status change. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
+- `tron:witness_address` (Gauge, label `address`) — value `1` for each consensus witness miner this node holds; the label is the Base58Check-encoded witness account address. The set is assembled once at startup from the finalized consensus miners: one series per configured witness private key, or — when a single private key signs for an explicitly configured witness account address — that witness account address. With several witness keys, each address gets its own series. A plain fullnode without a witness miner publishes no `tron:witness_address` series at all. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
 - `tron:block_fetch_armed` (Counter) — incremented when `FetchBlockService` arms a fetch tracking for a block; denominator for secondary-fetch rates. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
 - `tron:block_fetch_secondary` (Counter) — incremented when a secondary fetch request is sent to an alternate peer. ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))
 - `tron:block_already_known` (Counter) — incremented when a block response matches an outstanding adv request whose exact block ID is already known before that response is processed (best-effort signal; concurrent arrivals may be missed). ([#6923](https://github.com/tronprotocol/java-tron/issues/6923))

framework/src/main/java/org/tron/common/backup/BackupManager.java

@@ -14,6 +14,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import javax.annotation.PostConstruct;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -25,6 +26,8 @@
 import org.tron.common.backup.socket.UdpEvent;
 import org.tron.common.es.ExecutorServiceManager;
 import org.tron.common.parameter.CommonParameter;
+import org.tron.common.prometheus.Metrics;
+import org.tron.common.prometheus.SRMetrics;
 import org.tron.p2p.utils.NetUtil;
 
 @Slf4j(topic = "backup")
@@ -67,6 +70,41 @@ public class BackupManager implements EventHandler {
   public void setStatus(BackupStatusEnum status) {
     logger.info("Change backup status to {}", status);
     this.status = status;
+    recordNodeRole();
+  }
+
+  /** Publishes tron:node_role once at startup, and again on every backup-status change. */
+  @PostConstruct
+  public synchronized void recordNodeRole() {
+    if (!Metrics.enabled()) {
+      return;
+    }
+    SRMetrics.recordNodeRole(currentRole());
+  }
+
+  private int currentRole() {
+    // no backup configured: default to master, for either identity
+    if (!isBackupConfigured()) {
+      return SRMetrics.NODE_ROLE_MASTER;
+    }
+    // status still defaults to MASTER before backup init; not a real election result yet
+    if (!isInit) {
+      return SRMetrics.NODE_ROLE_INIT;
+    }
+    // post-init INIT covers the pre-keep-alive window and master-loss re-election
+    switch (status) {
+      case MASTER:
+        return SRMetrics.NODE_ROLE_MASTER;
+      case SLAVER:
+        return SRMetrics.NODE_ROLE_SLAVER;
+      default:
+        return SRMetrics.NODE_ROLE_INIT;
+    }
+  }
+
+  private boolean isBackupConfigured() {
+    // same condition BackupServer.initServer() uses to start the election transport
+    return parameter.getBackupPort() > 0 && !parameter.getBackupMembers().isEmpty();
   }
 
   public void init() {

framework/src/main/java/org/tron/core/consensus/ConsensusService.java

@@ -11,6 +11,9 @@
 import org.springframework.stereotype.Component;
 import org.tron.common.crypto.SignUtils;
 import org.tron.common.parameter.CommonParameter;
+import org.tron.common.prometheus.Metrics;
+import org.tron.common.prometheus.SRMetrics;
+import org.tron.common.utils.StringUtil;
 import org.tron.consensus.Consensus;
 import org.tron.consensus.base.Param;
 import org.tron.consensus.base.Param.Miner;
@@ -77,12 +80,47 @@ public void start() {
     }
 
     param.setMiners(miners);
+    recordWitnessAddresses(miners);
+    recordIsWitness();
     param.setBlockHandle(blockHandle);
     param.setPbftInterface(pbftBaseImpl);
     consensus.start(param);
     logger.info("consensus service start success");
   }
 
+  private void recordWitnessAddresses(List<Miner> miners) {
+    if (!Metrics.enabled() || miners == null) {
+      return;
+    }
+    for (Miner miner : miners) {
+      ByteString witnessAddress = miner.getWitnessAddress();
+      try {
+        String address = StringUtil.encode58Check(witnessAddress.toByteArray());
+        SRMetrics.recordWitnessAddress(address);
+      } catch (Exception e) {
+        logger.warn("Failed to record witness address metrics.", e);
+      }
+    }
+  }
+
+  /**
+   * Publishes tron:is_witness once at consensus startup; 0 = not witness, 1 = witness. It is
+   * published independently of the witness-address publication, and even when no miner is
+   * configured, so the flag series always exists. Catches its own failure so a metrics
+   * problem can neither skip the witness publication nor abort consensus startup.
+   */
+  private void recordIsWitness() {
+    if (!Metrics.enabled()) {
+      return;
+    }
+    try {
+      SRMetrics.recordIsWitness(Args.getInstance().isWitness()
+          ? SRMetrics.WITNESS : SRMetrics.NOT_WITNESS);
+    } catch (Exception e) {
+      logger.warn("Failed to record is-witness metrics.", e);
+    }
+  }
+
   public void stop() {
     logger.info("consensus service closed start.");
     consensus.stop();

framework/src/test/java/org/tron/common/backup/BackupManagerTest.java

@@ -1,6 +1,7 @@
 package org.tron.common.backup;
 
 import io.netty.channel.Channel;
+import io.prometheus.client.CollectorRegistry;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
@@ -20,14 +21,19 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
 import org.tron.common.TestConstants;
 import org.tron.common.backup.BackupManager.BackupStatusEnum;
 import org.tron.common.backup.message.KeepAliveMessage;
 import org.tron.common.backup.socket.BackupServer;
+import org.tron.common.backup.socket.MessageHandler;
 import org.tron.common.backup.socket.UdpEvent;
 import org.tron.common.parameter.CommonParameter;
+import org.tron.common.prometheus.MetricKeys;
+import org.tron.common.prometheus.SRMetrics;
 import org.tron.common.utils.PublicMethod;
 import org.tron.common.utils.ReflectUtils;
+import org.tron.consensus.base.Param;
 import org.tron.core.config.args.Args;
 import org.tron.core.config.args.InetUtil;
 
@@ -76,6 +82,9 @@ public void tearDown() throws Exception {
     });
     InetUtil.dnsLookup = previousDnsLookup;
     Args.clearParam();
+    // the consensus miner list is a process-wide singleton: reset it so this class
+    // cannot leak state into other tests of the same JVM
+    Param.getInstance().setMiners(new ArrayList<>());
     BackupTestUtils.throwIfAnyError(errors);
   }
 
@@ -183,6 +192,64 @@ public void testBackupServerLifecycleDuringKeepAliveInterval() throws Exception
     BackupTestUtils.assertExecutorsTerminated(manager, backupServer);
   }
 
+  /**
+   * A backup group whose only member is this node's own address is a "single-master" group:
+   * no peer ever answers a keep-alive, so the node must leave the pre-election INIT window and
+   * self-promote to MASTER once the keep-alive timeout elapses.
+   *
+   * <p>Drives the real {@link BackupManager#init()} and its real scheduled keep-alive task. The
+   * timeout condition is made deterministic by ageing the last keep-alive time, so the first
+   * (fixed 1s) scheduler tick is guaranteed to observe an expired keep-alive; recovery is then
+   * awaited by polling, so there is no fixed sleep. The published {@code tron:node_role} must
+   * track the transition (0 = init, 1 = master) and the manager executor must terminate.
+   */
+  @Test(timeout = 30000)
+  public void singleMasterGroupRecoversFromInitToMaster() throws Exception {
+    CommonParameter parameter = CommonParameter.getInstance();
+    parameter.setBackupPriority(8);
+    // the group's only member is ourselves, so init() drops it and no peer can answer
+    parameter.setBackupMembers(
+        Collections.singletonList(InetAddress.getLocalHost().getHostAddress()));
+
+    boolean previousMetrics = Args.getInstance().isMetricsPrometheusEnable();
+    Args.getInstance().setMetricsPrometheusEnable(true);
+    BackupManager single = new BackupManager();
+    // absorb any keep-alive send in case the local IP lookup behaves differently
+    single.setMessageHandler(Mockito.mock(MessageHandler.class));
+    try {
+      // backup configured but not yet init'ed: the pre-election window reports init (0)
+      Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus());
+      single.recordNodeRole();
+      Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0);
+
+      single.init();
+      Assert.assertEquals(BackupManager.BackupStatusEnum.INIT, single.getStatus());
+      Assert.assertEquals(SRMetrics.NODE_ROLE_INIT, nodeRoleGauge(), 0.0);
+
+      // no keep-alive has ever arrived, so the next keep-alive tick must self-promote.
+      // setStatus() flips the volatile status before it republishes the gauge, so wait on
+      // the published series (the externally visible result), not on the status field alone.
+      ReflectUtils.setFieldValue(single, "lastKeepAliveTime", 0L);
+      BackupTestUtils.awaitCondition("lone master to recover from INIT",
+          () -> single.getStatus() == BackupManager.BackupStatusEnum.MASTER
+              && Double.valueOf(SRMetrics.NODE_ROLE_MASTER).equals(nodeRoleGauge()));
+      Assert.assertEquals(BackupManager.BackupStatusEnum.MASTER, single.getStatus());
+      Assert.assertEquals(SRMetrics.NODE_ROLE_MASTER, nodeRoleGauge(), 0.0);
+    } finally {
+      single.stop();
+      Args.getInstance().setMetricsPrometheusEnable(previousMetrics);
+    }
+
+    ScheduledExecutorService executorService =
+        (ScheduledExecutorService) ReflectUtils.getFieldObject(single, "executorService");
+    Assert.assertTrue("single-master executor must terminate", executorService.isTerminated());
+  }
+
+  private Double nodeRoleGauge() {
+    return CollectorRegistry.defaultRegistry.getSampleValue(
+        MetricKeys.Gauge.NODE_ROLE, new String[] {}, new String[] {});
+  }
+
   // ===== domain-handling tests for init() =====
 
   @Test(timeout = 5000)