Skip to content

Commit d505f6f

Browse files
authored
refactor: improve activation and local diagnosis tool (#2945)
Arthas could previously be activated through unprotected HTTP endpoints, with diagnostic ports bound to 0.0.0.0. Server now requires configured authentication and an authenticated administrator for PUT /arthas; Store's unauthenticated /v1/arthasstart returns HTTP 403.
1 parent db85924 commit d505f6f

34 files changed

Lines changed: 1471 additions & 69 deletions

File tree

‎.github/workflows/pd-store-ci.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,12 @@ jobs:
374374
run: |
375375
mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end
376376
377+
- name: Verify Store Arthas local CLI attachment
378+
run: |
379+
VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
380+
STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION
381+
bash "$TRAVIS_DIR/test-arthas-cli.sh" "$STORE_DIR"
382+
377383
- name: Verify Surefire single-class routing
378384
run: |
379385
SUREFIRE_DIR=hugegraph-store/hg-store-test/target/surefire-reports
@@ -446,12 +452,12 @@ jobs:
446452
echo "PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION" >> "$GITHUB_ENV"
447453
echo "STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION" >> "$GITHUB_ENV"
448454
449-
- name: Run historical Store ingress tests
455+
- name: Run Store ingress and Arthas security tests
450456
run: |
451457
set -o pipefail
452458
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
453459
mvn test -pl hugegraph-store/hg-store-node -am \
454-
-Dtest=StoredRowIngressTest -Dsurefire.failIfNoSpecifiedTests=false \
460+
-Dtest=StoredRowIngressTest,ArthasSecurityTest -Dsurefire.failIfNoSpecifiedTests=false \
455461
-Djacoco.sessionId=store-node-test \
456462
-Djacoco.destFile="$PWD/hugegraph-store/hg-store-test/target/store-node-test.exec" 2>&1 | tee "$RUNTIME_LOG"
457463
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"
@@ -526,6 +532,8 @@ jobs:
526532
"$TEST_REPORT_DIR/TEST-org.apache.hugegraph.store.service.ServerSuiteTest.xml" \
527533
--require-test-report \
528534
"hugegraph-store/hg-store-node/target/surefire-reports/TEST-org.apache.hugegraph.store.business.StoredRowIngressTest.xml" \
535+
--require-test-report \
536+
"hugegraph-store/hg-store-node/target/surefire-reports/TEST-org.apache.hugegraph.store.node.controller.ArthasSecurityTest.xml" \
529537
--require-covered-group hg-store-common \
530538
--require-covered-group hg-store-client \
531539
--require-covered-group hg-store-rocksdb \

‎.github/workflows/server-tests.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,13 @@ jobs:
104104
$TRAVIS_DIR/test-java-security-properties.sh \
105105
"$SERVER_DIR" "$GITHUB_WORKSPACE"
106106
107+
- name: Check bundled Arthas local CLI
108+
if: ${{ env.BACKEND == 'rocksdb' }}
109+
run: |
110+
VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
111+
SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION
112+
bash "$TRAVIS_DIR/test-arthas-cli.sh" "$SERVER_DIR"
113+
107114
- name: Verify Surefire single-class routing
108115
if: ${{ env.BACKEND == 'rocksdb' }}
109116
run: |

‎docs/server-runtime.md‎

Lines changed: 132 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,138 @@ existing precedence; see [container heap sizing](../docker/README.md#server-heap
7979

8080
### Arthas diagnostics
8181

82-
Server bundles Arthas 4.1.5. Activate the agent with `PUT /arthas` using the Server's
83-
configured authentication. The existing `arthas.ip`, `arthas.httpPort`,
84-
`arthas.telnetPort` and `arthas.disabledCommands` options still configure the agent.
85-
HugeGraph disables the bundled agent's optional MCP endpoint and keeps the existing
86-
activation response and diagnostic HTTP/telnet interfaces.
82+
Server and Store bundle Arthas 4.1.5, managed by the root POM. PD does not embed
83+
an Arthas agent. Diagnostic ports bind to `127.0.0.1` by default. Administrators
84+
can explicitly configure another valid address, including wildcard `0.0.0.0` or
85+
`::`; `localhost` is normalized to `127.0.0.1`. Wildcard listeners can accept both
86+
IPv4 and IPv6 depending on the OS/JVM configuration; restrict both in firewall
87+
rules. Server's `PUT /arthas`
88+
requires authentication to be enabled and the caller to be an authenticated administrator.
89+
Without authentication, the Server endpoint returns HTTP 403, including requests
90+
from localhost or through a reverse proxy. Store has no equivalent administrator
91+
authentication, so `GET /v1/arthasstart` always returns HTTP 403.
92+
93+
For local diagnosis of either running service, use its distribution's bundled
94+
helper as the same OS user that owns the JVM:
95+
96+
```bash
97+
bin/arthas.sh
98+
```
99+
100+
The distribution prepares the agent in `lib/arthas` at build time. The helper
101+
requires JDK 17 and uses `bin/pid`, without downloads, runtime extraction, a user
102+
cache, service restarts or configuration edits. A new local agent listens on
103+
`127.0.0.1:3658`, with HTTP, MCP and reverse tunnels disabled. Use `--pid` or
104+
`--telnet-port` for another running instance, and `--command version` for a one-time
105+
command. Interactive mode requires a terminal; for pipes, scripts or `docker exec`
106+
without `-it`, use `--command` instead. Run `bin/arthas.sh --help` for usage.
107+
When the helper exits, including after `--command`, `quit`, or INT/TERM/HUP, it
108+
stops the agent and all its diagnostic sessions; the application JVM keeps running.
109+
SIGKILL or a host failure cannot run this cleanup: stop any surviving agent manually.
110+
For remote diagnosis, log in to the service host over SSH and run the same command,
111+
or explicitly configure a reachable listener with appropriate authentication and
112+
network access controls as described below.
113+
Official Server (including HStore) and Store images include JDK 17, so the same
114+
helper works inside the container, for example
115+
`docker exec -it <store-container> /hugegraph-store/bin/arthas.sh`.
116+
A JRE-only installation needs JDK 17 for this helper.
117+
If the Server API has already activated Arthas, connect to its existing telnet
118+
port instead: `bin/arthas.sh --telnet-port 8562` with the default configuration.
119+
Use `--target-ip` to select another bind address, for example
120+
`bin/arthas.sh --target-ip 0.0.0.0` or
121+
`bin/arthas.sh --target-ip ::1 --telnet-port 8562 --command version`.
122+
The default is `127.0.0.1`; `localhost` maps to `127.0.0.1`. Wildcard addresses
123+
`0.0.0.0` and `::` may accept both IPv4 and IPv6; the helper connects and cleans
124+
up through `127.0.0.1` for either wildcard. Remote clients connect
125+
to the service host's actual reachable address, not the wildcard address.
126+
An already running agent retains its listener and authentication settings;
127+
helper options do not reconfigure it. To switch to the helper's
128+
HTTP-disabled defaults, run `bin/arthas.sh --telnet-port 8562 --command stop`, then
129+
`bin/arthas.sh`; this stops only Arthas, and leaves the service JVM running.
130+
Arthas can execute code inside the JVM. OS attach permissions control initial
131+
activation; HugeGraph administrator authentication protects only the activation
132+
API, not subsequent diagnostic connections. We recommend enabling Arthas native
133+
authentication, especially when binding beyond loopback. However, native defaults
134+
allow connections from `127.0.0.1` to skip authentication even with credentials;
135+
other OS users or containers in the same Pod can still connect this way. To disable
136+
this bypass, set `-Darthas.localConnectionNonAuth=false` on the service JVM and
137+
restart it. Restrict host/Pod and port-forward access to trusted operators, and
138+
limit firewall rules, port mappings and SSH forwarding accordingly. Authentication
139+
does not encrypt telnet
140+
or ordinary HTTP traffic; use a protected network or encrypted transport for
141+
remote access. Avoid concurrent helper sessions because exiting one stops the
142+
shared agent. A disabled-command list is optional and is not access control.
143+
144+
Pass native Arthas credentials when starting or reusing an agent:
145+
146+
```bash
147+
bin/arthas.sh --target-ip 0.0.0.0 \
148+
--username diagnostics --password '<arthas-password>'
149+
```
150+
151+
Use single-line credentials; printable ASCII avoids the bundled Arthas 4.1.5
152+
telnet client's Unicode limitation. Password-only use defaults to username `arthas`.
153+
Native JVM credentials also apply to helper-started agents: if the target JVM has
154+
`-Darthas.username` or `-Darthas.password`, supply the effective credentials with
155+
`--username` / `--password` for authenticated commands and cleanup, especially
156+
when `-Darthas.localConnectionNonAuth=false` is set. The helper does not clear
157+
native credentials when the corresponding option is omitted.
158+
These credentials are separate from HugeGraph accounts. Batch commands and exit
159+
cleanup use the supplied credentials automatically. For interactive diagnosis,
160+
if prompted, enter `auth --username diagnostics <arthas-password>` in the Arthas
161+
console. Existing agents require their existing credentials; passing new values
162+
does not change them. Password arguments may appear in shell history and process
163+
listings, so handle them on a trusted host. With `--username` alone, native Arthas
164+
generates a password only when none is already configured in the target JVM's
165+
native settings; otherwise it uses that inherited password. Supply `--password` for
166+
authenticated batch commands and automatic cleanup. If cleanup fails, the helper
167+
returns nonzero and the agent may remain running; use the configured or generated credentials with
168+
native `auth`, then `stop`, to close it without stopping the service JVM.
169+
170+
[Arthas native authentication](https://arthas.aliyun.com/en/doc/auth.html) remains
171+
optional and follows upstream defaults: `arthas.localConnectionNonAuth=true`
172+
allows connections from `127.0.0.1` to skip authentication. The helper uses a private
173+
configuration, so editing `lib/arthas/arthas.properties` does not configure it.
174+
If required, the target JVM property `-Darthas.localConnectionNonAuth=false` is
175+
honored after restarting the service JVM. Arthas 4.1.5 generates a password when
176+
binding `0.0.0.0` or `::` without an effective native password and prints it in the target
177+
JVM output and Arthas log; explicitly supplying credentials is more convenient
178+
for remote diagnosis and authenticated cleanup.
179+
180+
Authenticated Server administrators can continue using `PUT /arthas` without an
181+
additional enable switch or restart. Its HTTP/telnet interfaces use the configured
182+
bind address; HugeGraph disables the optional MCP endpoint. The activation response
183+
still contains the same four Arthas configuration keys. For API activation with
184+
native credentials, add `-Darthas.username=diagnostics` and
185+
`-Darthas.password=<arthas-password>` to the Server JVM startup options and restart
186+
Server before activation. These are native JVM properties, not new REST
187+
configuration keys, and also apply to agents started by the local helper as described
188+
above. They may expose secrets in process listings. Without native
189+
credential settings, a specific non-loopback bind address does not require login; wildcard
190+
binds use the generated password described above. API activation alone
191+
leaves the agent running until an operator sends `stop`; connecting through the
192+
helper applies the helper's exit cleanup to that existing agent too.
193+
194+
Before reusing an API-started agent, leave `session` and `stop` out of
195+
`arthas.disabled_commands` (or its legacy alias). `session` verifies the target
196+
PID; `stop` closes Arthas without stopping the service JVM. Existing configurations
197+
remain accepted, but failed PID verification or listener cleanup makes the helper
198+
print an error and exit nonzero; the agent may remain running. To recover from an
199+
agent with `stop` disabled, remove that restriction and restart the service before
200+
activating Arthas again. Newly attached local agents keep both commands available.
201+
202+
No commands are disabled by default. Administrators can optionally set
203+
`arthas.disabled_commands=jad,mc` to restrict selected commands for API activation;
204+
new local agents do not inherit this list.
205+
206+
The existing `arthas.ip`, `arthas.httpPort`, `arthas.telnetPort` and
207+
`arthas.disabledCommands` options configure the Server API-activated agent.
208+
The distribution uses `arthas.telnet_port`, `arthas.http_port` and
209+
`arthas.disabled_commands`. These snake_case keys take precedence over the legacy
210+
camelCase aliases when both are present, including explicit default values.
211+
Conflicting values produce a warning naming both keys and the effective key when
212+
the agent is activated. Remove the unused alias to avoid ambiguity. Store's legacy
213+
configuration fields are retained; the local helper does not read them.
87214

88215
For a synchronous one-time command, POST `{"action":"exec","command":"version"}`
89216
to the agent's `/api` endpoint without a `sessionId`. Check `state=SUCCEEDED`, the

‎hugegraph-server/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh
3535

3636
# 2nd stage: runtime env
3737
# Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13
38-
FROM eclipse-temurin:17-jre-jammy
38+
FROM eclipse-temurin:17-jdk-jammy
39+
# Local Arthas attach requires the JDK attach and compiler modules.
3940

4041
LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
4142

‎hugegraph-server/Dockerfile-hstore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh
3535

3636
# 2nd stage: runtime env
3737
# Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13
38-
FROM eclipse-temurin:17-jre-jammy
38+
FROM eclipse-temurin:17-jdk-jammy
39+
# Local Arthas attach requires the JDK attach and compiler modules.
3940

4041
LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
4142

‎hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/arthas/ArthasAPI.java‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,48 +17,109 @@
1717

1818
package org.apache.hugegraph.api.arthas;
1919

20+
import java.net.InetAddress;
21+
import java.net.UnknownHostException;
2022
import java.util.HashMap;
2123

2224
import org.apache.hugegraph.api.API;
25+
import org.apache.hugegraph.auth.HugeAuthenticator;
26+
import org.apache.hugegraph.config.ConfigOption;
2327
import org.apache.hugegraph.config.HugeConfig;
2428
import org.apache.hugegraph.config.ServerOptions;
29+
import org.apache.hugegraph.core.GraphManager;
2530
import org.apache.hugegraph.util.JsonUtil;
31+
import org.apache.hugegraph.util.Log;
32+
import org.slf4j.Logger;
2633

2734
import com.codahale.metrics.annotation.Timed;
2835
import com.taobao.arthas.agent.attach.ArthasAgent;
2936

3037
import io.swagger.v3.oas.annotations.Operation;
3138
import io.swagger.v3.oas.annotations.tags.Tag;
39+
import jakarta.annotation.security.RolesAllowed;
3240
import jakarta.inject.Singleton;
41+
import jakarta.ws.rs.ForbiddenException;
3342
import jakarta.ws.rs.PUT;
3443
import jakarta.ws.rs.Path;
3544
import jakarta.ws.rs.Produces;
3645
import jakarta.ws.rs.core.Context;
46+
import jakarta.ws.rs.core.SecurityContext;
3747

3848
@Path("arthas")
3949
@Singleton
4050
@Tag(name = "ArthasAPI")
4151
public class ArthasAPI extends API {
4252

53+
private static final Logger LOG = Log.logger(ArthasAPI.class);
54+
4355
@Context
4456
private jakarta.inject.Provider<HugeConfig> configProvider;
4557

4658
@PUT
4759
@Timed
4860
@Produces(APPLICATION_JSON_WITH_CHARSET)
61+
@RolesAllowed("admin")
4962
@Operation(summary = "start arthas agent")
50-
public Object startArthas() {
63+
public Object startArthas(@Context GraphManager manager,
64+
@Context SecurityContext context) {
65+
checkAccess(manager, context);
5166
HugeConfig config = this.configProvider.get();
52-
HashMap<String, String> configMap = new HashMap<>(4);
53-
configMap.put("arthas.telnetPort", config.get(ServerOptions.ARTHAS_TELNET_PORT));
54-
configMap.put("arthas.httpPort", config.get(ServerOptions.ARTHAS_HTTP_PORT));
55-
configMap.put("arthas.ip", config.get(ServerOptions.ARTHAS_IP));
56-
configMap.put("arthas.disabledCommands",
57-
config.get(ServerOptions.ARTHAS_DISABLED_COMMANDS));
67+
HashMap<String, String> configMap = arthasConfig(config);
5868
String response = JsonUtil.toJson(configMap);
5969
// Keep the optional MCP endpoint disabled without extending the activation response.
6070
configMap.put("arthas.mcpEndpoint", "");
71+
// Do not let external Arthas properties override the configured listener.
72+
configMap.put("arthas.config.overrideAll", "false");
73+
configMap.put("arthas.tunnelServer", "");
6174
ArthasAgent.attach(configMap);
6275
return response;
6376
}
77+
78+
static void checkAccess(GraphManager manager, SecurityContext context) {
79+
// No-auth requests receive an anonymous admin role, so a role check alone is insufficient.
80+
if (!manager.requireAuthentication()) {
81+
throw new ForbiddenException("HTTP Arthas activation requires authentication; " +
82+
"use bin/arthas.sh on the server host instead");
83+
}
84+
if (context == null || context.getUserPrincipal() == null ||
85+
HugeAuthenticator.USER_ANONY.equals(context.getUserPrincipal().getName()) ||
86+
!context.isUserInRole("admin")) {
87+
throw new ForbiddenException("Arthas activation requires an authenticated administrator");
88+
}
89+
}
90+
91+
static HashMap<String, String> arthasConfig(HugeConfig config) {
92+
HashMap<String, String> configMap = new HashMap<>(4);
93+
configMap.put("arthas.telnetPort",
94+
option(config, ServerOptions.ARTHAS_TELNET_PORT,
95+
ServerOptions.ARTHAS_TELNET_PORT_LEGACY));
96+
configMap.put("arthas.httpPort",
97+
option(config, ServerOptions.ARTHAS_HTTP_PORT,
98+
ServerOptions.ARTHAS_HTTP_PORT_LEGACY));
99+
String ip = config.get(ServerOptions.ARTHAS_IP);
100+
ip = "localhost".equalsIgnoreCase(ip) ? "127.0.0.1" : ip;
101+
try {
102+
InetAddress.getByName(ip);
103+
} catch (UnknownHostException e) {
104+
throw new IllegalArgumentException("Invalid or unresolvable Arthas bind address '" + ip + "'", e);
105+
}
106+
configMap.put("arthas.ip", ip);
107+
configMap.put("arthas.disabledCommands",
108+
option(config, ServerOptions.ARTHAS_DISABLED_COMMANDS,
109+
ServerOptions.ARTHAS_DISABLED_COMMANDS_LEGACY));
110+
return configMap;
111+
}
112+
113+
private static String option(HugeConfig config, ConfigOption<String> canonical,
114+
ConfigOption<String> legacy) {
115+
if (!config.containsKey(canonical.name())) {
116+
return config.get(legacy);
117+
}
118+
String value = config.get(canonical);
119+
if (config.containsKey(legacy.name()) && !value.equals(config.get(legacy))) {
120+
LOG.warn("Conflicting Arthas options '{}' and '{}': '{}' takes precedence",
121+
canonical.name(), legacy.name(), canonical.name());
122+
}
123+
return value;
124+
}
64125
}

0 commit comments

Comments
 (0)