Skip to content

Commit 5958f54

Browse files
authored
Merge pull request DeusData#2273 from DeusData/distill/1245-service-pattern-boundaries
fix(service-patterns): match library ids on identifier boundaries
2 parents 5c0b241 + c314dea commit 5958f54

2 files changed

Lines changed: 295 additions & 12 deletions

File tree

‎internal/cbm/service_patterns.c‎

Lines changed: 114 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,20 @@ static const lib_pattern_t http_libraries[] = {
140140
{"socket.http", CBM_SVC_HTTP, NULL},
141141
{"resty.http", CBM_SVC_HTTP, NULL},
142142

143+
/* Glued-name libraries. match_qn needs an identifier boundary around an
144+
* id, so a library whose own name glues a lowercase prefix/suffix onto
145+
* another id ("grequests", "curlpp") is listed explicitly: no boundary
146+
* rule can tell "grequests" from "myrequests". */
147+
{"grequests", CBM_SVC_HTTP, NULL}, /* Python: gevent + requests */
148+
{"txrequests", CBM_SVC_HTTP, NULL}, /* Python: Twisted + requests */
149+
{"redaxios", CBM_SVC_HTTP, NULL}, /* JS: axios-compatible fetch client */
150+
{"gaxios", CBM_SVC_HTTP, NULL}, /* JS: Google's axios-style client */
151+
{"libcurl", CBM_SVC_HTTP, NULL}, /* C, node-libcurl */
152+
{"curlpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */
153+
{"curlcpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */
154+
{"hyperlocal", CBM_SVC_HTTP, NULL}, /* Rust: hyper over unix sockets */
155+
{"guzzlehttp", CBM_SVC_HTTP, NULL}, /* PHP: Guzzle's composer vendor */
156+
143157
{NULL, CBM_SVC_NONE, NULL},
144158
};
145159

@@ -263,6 +277,33 @@ static const lib_pattern_t async_libraries[] = {
263277
{"dapr.clients.grpc", CBM_SVC_ASYNC, "dapr"},
264278
{"DaprClient", CBM_SVC_ASYNC, "dapr"},
265279

280+
/* Glued-name libraries (see the note in http_libraries). */
281+
{"aiokafka", CBM_SVC_ASYNC, "kafka"},
282+
{"pykafka", CBM_SVC_ASYNC, "kafka"},
283+
{"librdkafka", CBM_SVC_ASYNC, "kafka"},
284+
{"rdkafkacpp", CBM_SVC_ASYNC, "kafka"},
285+
{"rskafka", CBM_SVC_ASYNC, "kafka"},
286+
{"aioamqp", CBM_SVC_ASYNC, "rabbitmq"},
287+
{"amqpstorm", CBM_SVC_ASYNC, "rabbitmq"},
288+
{"pamqp", CBM_SVC_ASYNC, "rabbitmq"},
289+
{"pyamqp", CBM_SVC_ASYNC, "rabbitmq"},
290+
{"amqprs", CBM_SVC_ASYNC, "rabbitmq"},
291+
{"amqpcpp", CBM_SVC_ASYNC, "rabbitmq"},
292+
{"pynats", CBM_SVC_ASYNC, "nats"},
293+
{"jnats", CBM_SVC_ASYNC, "nats"},
294+
{"aiomqtt", CBM_SVC_ASYNC, "mqtt"},
295+
{"amqtt", CBM_SVC_ASYNC, "mqtt"},
296+
{"hbmqtt", CBM_SVC_ASYNC, "mqtt"},
297+
{"umqtt", CBM_SVC_ASYNC, "mqtt"},
298+
{"mqttools", CBM_SVC_ASYNC, "mqtt"},
299+
{"emqtt", CBM_SVC_ASYNC, "mqtt"},
300+
{"rumqtt", CBM_SVC_ASYNC, "mqtt"},
301+
{"gomqtt", CBM_SVC_ASYNC, "mqtt"},
302+
{"libmosquitto", CBM_SVC_ASYNC, "mqtt"},
303+
{"mosquittopp", CBM_SVC_ASYNC, "mqtt"},
304+
{"pubsublite", CBM_SVC_ASYNC, "pubsub"},
305+
{"gocelery", CBM_SVC_ASYNC, "celery"},
306+
266307
{NULL, CBM_SVC_NONE, NULL},
267308
};
268309

@@ -308,6 +349,13 @@ static const lib_pattern_t config_libraries[] = {
308349
{"Application.get_env", CBM_SVC_CONFIG, NULL},
309350
{"Application.fetch_env", CBM_SVC_CONFIG, NULL},
310351

352+
/* Glued-name accessors (see the note in http_libraries). */
353+
{"wgetenv", CBM_SVC_CONFIG, NULL}, /* Windows CRT: _wgetenv, _wgetenv_s */
354+
{"getenvb", CBM_SVC_CONFIG, NULL}, /* Python: os.getenvb */
355+
{"qgetenv", CBM_SVC_CONFIG, NULL}, /* Qt */
356+
{"dotenvx", CBM_SVC_CONFIG, NULL}, /* JS: @dotenvx/dotenvx */
357+
{"phpdotenv", CBM_SVC_CONFIG, NULL}, /* PHP: vlucas/phpdotenv */
358+
311359
{NULL, CBM_SVC_NONE, NULL},
312360
};
313361

@@ -377,6 +425,11 @@ static const lib_pattern_t route_reg_libraries[] = {
377425
{"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL},
378426
{"play.api.routing", CBM_SVC_ROUTE_REG, NULL},
379427

428+
/* Glued-name frameworks (see the note in http_libraries). */
429+
{"apiflask", CBM_SVC_ROUTE_REG, NULL}, /* Python: Flask-based */
430+
{"honox", CBM_SVC_ROUTE_REG, NULL}, /* JS: Hono meta-framework */
431+
{"fasthttprouter", CBM_SVC_ROUTE_REG, NULL}, /* Go: httprouter for fasthttp */
432+
380433
{NULL, CBM_SVC_NONE, NULL},
381434
};
382435

@@ -448,6 +501,11 @@ static const lib_pattern_t graphql_libraries[] = {
448501
{"async-graphql", CBM_SVC_GRAPHQL, NULL},
449502
{"juniper", CBM_SVC_GRAPHQL, NULL},
450503

504+
/* Glued-name libraries (see the note in http_libraries). */
505+
{"gqlparser", CBM_SVC_GRAPHQL, NULL}, /* Go: vektah/gqlparser */
506+
{"gqlgenc", CBM_SVC_GRAPHQL, NULL}, /* Go: gqlgen client generator */
507+
{"aiogqlc", CBM_SVC_GRAPHQL, NULL}, /* Python: asyncio GraphQL client */
508+
451509
{NULL, CBM_SVC_NONE, NULL},
452510
};
453511

@@ -534,17 +592,67 @@ static const method_suffix_t method_suffixes[] = {
534592

535593
/* ── Matching implementation ───────────────────────────────────── */
536594

537-
/* Check if any library identifier appears as a substring in the QN.
538-
* Case-sensitive: "requests" matches "project.venv.requests.api.get"
539-
* but not "Requests". Library names are specific enough to avoid
540-
* false positives even with substring matching. */
595+
static bool qn_is_lower(char ch) {
596+
return ch >= 'a' && ch <= 'z';
597+
}
598+
599+
static bool is_digit_char(char ch) {
600+
return ch >= '0' && ch <= '9';
601+
}
602+
603+
static bool qn_is_alnum(char ch) {
604+
return qn_is_lower(ch) || is_digit_char(ch) || (ch >= 'A' && ch <= 'Z');
605+
}
606+
607+
/* True when the occurrence of `id` at `hit` sits on identifier boundaries.
608+
*
609+
* A raw substring match let short ids fire inside unrelated words: "gin."
610+
* in "plugin.", "dio" in "studio", "surf" in "surface", "express" in
611+
* "expression". Every character that is not an ASCII letter or digit is a
612+
* separator ('.', '/', '\\', ':', '_', '-', '$', '@', ...).
613+
*
614+
* BEFORE: start of string, a separator, an id that itself starts with a
615+
* separator ("@trpc/server"), or an id that starts with an
616+
* uppercase letter — a capital opens a new CamelCase word whatever
617+
* precedes it ("AsyncHttpClient", "IHttpClientFactory",
618+
* "NSURLSession"). Rejected: a lowercase/digit-initial id glued to
619+
* a letter or digit ("plugin." / "studio" / "myrequests").
620+
* AFTER: end of string, a separator, an id that itself ends in a
621+
* separator ("gin."), an uppercase letter (CamelCase glue:
622+
* "GuzzleHttp", "FeignClient", "KafkaProducer", "kafkaProducer")
623+
* or a digit (version suffix: "urllib2", "amqp091-go").
624+
* Rejected: a lowercase letter continuing the word ("surface",
625+
* "curly", "expression"). */
626+
static bool qn_hit_on_boundary(const char *qn, const char *hit, const char *id, size_t id_len) {
627+
char first = id[0];
628+
char last = id[id_len - 1];
629+
if (hit > qn && qn_is_alnum(hit[-1]) && (qn_is_lower(first) || is_digit_char(first))) {
630+
return false;
631+
}
632+
if (qn_is_alnum(last) && qn_is_lower(hit[id_len])) {
633+
return false;
634+
}
635+
return true;
636+
}
637+
638+
/* Check if any library identifier appears in the QN on identifier
639+
* boundaries (see qn_hit_on_boundary). Case-sensitive: "requests" matches
640+
* "project.venv.requests.api.get" and "svc.requests_get" but neither
641+
* "Requests" nor "myrequests". */
541642
static const lib_pattern_t *match_qn(const char *qn, const lib_pattern_t *patterns) {
542643
if (!qn || !qn[0]) {
543644
return NULL;
544645
}
545646
for (int i = 0; patterns[i].library_id != NULL; i++) {
546-
if (strstr(qn, patterns[i].library_id) != NULL) {
547-
return &patterns[i];
647+
const char *id = patterns[i].library_id;
648+
size_t id_len = strlen(id);
649+
if (id_len == 0) {
650+
continue;
651+
}
652+
for (const char *hit = strstr(qn, id); hit != NULL; hit = strstr(hit + 1, id)) {
653+
if (qn_hit_on_boundary(qn, hit, id, id_len)) {
654+
return &patterns[i];
655+
}
548656
}
549657
}
550658
return NULL;
@@ -574,10 +682,6 @@ static bool contains_segment(const char *path, const char *segment) {
574682
return false;
575683
}
576684

577-
static bool is_digit_char(char ch) {
578-
return ch >= '0' && ch <= '9';
579-
}
580-
581685
static bool has_http_route_marker(const char *path) {
582686
if (starts_with_segment(path, "api") || starts_with_segment(path, "apis") ||
583687
starts_with_segment(path, "graphql") || starts_with_segment(path, "health") ||

‎tests/test_infrascan.c‎

Lines changed: 181 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
#include "test_framework.h"
22
#include "graph_buffer/graph_buffer.h"
33
#include "pipeline/pipeline_internal.h"
4+
#include "service_patterns.h"
45

56
#include <stdbool.h>
67
#include <stdint.h>
7-
8-
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
8+
#include <stdio.h>
99

1010
static int has_data_flow(cbm_gbuf_t *gb, int64_t source_id, int64_t target_id) {
1111
const cbm_gbuf_edge_t **edges = NULL;
@@ -48,6 +48,183 @@ TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths) {
4848
PASS();
4949
}
5050

51+
/* ── Service-pattern QN boundaries (distilled from PR #1245) ─────────
52+
* A library id must sit on an identifier boundary inside the QN. A raw
53+
* substring match classified "proj.plugin.loader" as gin ("gin.") and
54+
* "proj.studio.x" as Dio ("dio"), which minted bogus Route / HTTP_CALLS
55+
* edges. The rule must stay open for CamelCase glue ("GuzzleHttp",
56+
* "FeignClient", "KafkaProducer", "IHttpClientFactory") and for version
57+
* suffixes ("urllib2", "amqp091-go"). */
58+
typedef struct {
59+
const char *qn;
60+
cbm_svc_kind_t want;
61+
} svc_case_t;
62+
63+
static int svc_case_mismatches(const svc_case_t *cases) {
64+
int bad = 0;
65+
for (int i = 0; cases[i].qn != NULL; i++) {
66+
cbm_svc_kind_t got = cbm_service_pattern_match(cases[i].qn);
67+
if (got != cases[i].want) {
68+
printf(" service-pattern mismatch: \"%s\" -> %d, expected %d\n", cases[i].qn, (int)got,
69+
(int)cases[i].want);
70+
bad++;
71+
}
72+
}
73+
return bad;
74+
}
75+
76+
TEST(infrascan_service_pattern_match_rejects_ids_inside_words) {
77+
static const svc_case_t cases[] = {
78+
/* "gin." inside plugin. */
79+
{"proj.plugins.autorun.tests.test_plugin._dispatch", CBM_SVC_NONE},
80+
{"proj.plugin.loader", CBM_SVC_NONE},
81+
/* "chi." / "echo." / "fiber." */
82+
{"proj.hibachi.grill", CBM_SVC_NONE},
83+
{"sound.reecho.apply", CBM_SVC_NONE},
84+
{"textile.microfiber.weave", CBM_SVC_NONE},
85+
/* "dio" inside studio / radio */
86+
{"proj.studio.x", CBM_SVC_NONE},
87+
{"radio.tune", CBM_SVC_NONE},
88+
/* "surf" / "resty" / "curl" / "phin" / "hyper" / "sling" / "needle" / "treq" */
89+
{"app.surface.draw", CBM_SVC_NONE},
90+
{"ui.restyle.apply", CBM_SVC_NONE},
91+
/* control: "forestry" spells "restr", never contained "resty" */
92+
{"pkg.forestry.trees", CBM_SVC_NONE},
93+
{"util.curly.brace", CBM_SVC_NONE},
94+
{"sea.dolphin.swim", CBM_SVC_NONE},
95+
{"math.hyperbolic.tanh", CBM_SVC_NONE},
96+
{"farm.gosling.walk", CBM_SVC_NONE},
97+
{"sewing.needles.sort", CBM_SVC_NONE},
98+
{"util.streq", CBM_SVC_NONE},
99+
/* "requests" glued to a lowercase prefix (upstream's case) */
100+
{"proj.myrequests.client.get", CBM_SVC_NONE},
101+
/* "express" / "rocket" / "tonic" / "nconf" */
102+
{"ast.expression.eval", CBM_SVC_NONE},
103+
{"mq.rocketmq.send", CBM_SVC_NONE},
104+
{"geo.tectonic.plates", CBM_SVC_NONE},
105+
{"app.unconfigured.warn", CBM_SVC_NONE},
106+
{"cli.runconfig.load", CBM_SVC_NONE},
107+
{"aws.s3express.session", CBM_SVC_NONE},
108+
/* not clients of the id they contain: ASGI server, URL library,
109+
* nginx platform, Memgraph OGM, Celery settings module, a word */
110+
{"hypercorn.run.serve", CBM_SVC_NONE},
111+
{"hyperlink.URL.from_text", CBM_SVC_NONE},
112+
{"openresty.core.request", CBM_SVC_NONE},
113+
{"gqlalchemy.Memgraph.execute", CBM_SVC_NONE},
114+
{"celeryconfig.broker_url", CBM_SVC_NONE},
115+
{"kafkaesque.story.tell", CBM_SVC_NONE},
116+
{NULL, CBM_SVC_NONE},
117+
};
118+
ASSERT_EQ(svc_case_mismatches(cases), 0);
119+
PASS();
120+
}
121+
122+
TEST(infrascan_service_pattern_match_keeps_real_library_qns) {
123+
static const svc_case_t cases[] = {
124+
/* Route registration: ids that end in a separator, own segment */
125+
{"github.com/gin-gonic/gin.Default", CBM_SVC_ROUTE_REG},
126+
{"proj.gin.router.GET", CBM_SVC_ROUTE_REG},
127+
{"gin.GET", CBM_SVC_ROUTE_REG},
128+
{"chi.NewRouter", CBM_SVC_ROUTE_REG},
129+
{"github.com/labstack/echo.New", CBM_SVC_ROUTE_REG},
130+
{"fiber.New", CBM_SVC_ROUTE_REG},
131+
{"proj.express.router.get", CBM_SVC_ROUTE_REG},
132+
{"@hapi/hapi.server.route", CBM_SVC_ROUTE_REG},
133+
/* HTTP: own segment / separator-joined wrapper */
134+
{"proj.venv.requests.api.get", CBM_SVC_HTTP},
135+
{"proj.service.requests_get", CBM_SVC_HTTP},
136+
{"package:dio/dio.dart", CBM_SVC_HTTP},
137+
{"dio.Dio.get", CBM_SVC_HTTP},
138+
{"pkg.net.curl.get", CBM_SVC_HTTP},
139+
{"curl_exec", CBM_SVC_HTTP},
140+
{"surf.get", CBM_SVC_HTTP},
141+
{"hyper.Client.request", CBM_SVC_HTTP},
142+
{"phin.promisified", CBM_SVC_HTTP},
143+
{"needle.get", CBM_SVC_HTTP},
144+
{"github.com/go-resty/resty.New", CBM_SVC_HTTP},
145+
{"github.com/dghubble/sling.New", CBM_SVC_HTTP},
146+
/* HTTP: CamelCase glue after the id */
147+
{"GuzzleHttp\\Client", CBM_SVC_HTTP},
148+
{"proj.GuzzleHttp.Client.get", CBM_SVC_HTTP},
149+
{"org.springframework.cloud.openfeign.FeignClientBuilder.build", CBM_SVC_HTTP},
150+
{"com.acme.net.HttpClientFactory.create", CBM_SVC_HTTP},
151+
/* HTTP: CamelCase glue before the id */
152+
{"System.Net.Http.IHttpClientFactory.CreateClient", CBM_SVC_HTTP},
153+
{"org.asynchttpclient.AsyncHttpClient.prepareGet", CBM_SVC_HTTP},
154+
{"Foundation.NSURLSession.dataTask", CBM_SVC_HTTP},
155+
/* HTTP: version suffix after the id */
156+
{"urllib2.urlopen", CBM_SVC_HTTP},
157+
{"Mint.HTTP2.request", CBM_SVC_HTTP},
158+
/* Async */
159+
{"com.acme.events.KafkaProducer.send", CBM_SVC_ASYNC},
160+
{"this.kafkaProducer.send", CBM_SVC_ASYNC},
161+
{"org.apache.kafka.clients.producer.KafkaProducer.send", CBM_SVC_ASYNC},
162+
{"github.com/rabbitmq/amqp091-go.Channel.Publish", CBM_SVC_ASYNC},
163+
{"github.com/nats-io/nats.go.Conn.Publish", CBM_SVC_ASYNC},
164+
/* Config */
165+
{"os.getenv", CBM_SVC_CONFIG},
166+
{"os.Getenv", CBM_SVC_CONFIG},
167+
/* Glued-name libraries: the library's own name glues a lowercase
168+
* prefix/suffix onto another id, so each has an explicit table entry.
169+
* Every QN below contains NO other id — only the entry can match. */
170+
{"grequests.map", CBM_SVC_HTTP},
171+
{"txrequests.Session.send", CBM_SVC_HTTP},
172+
{"redaxios.create", CBM_SVC_HTTP},
173+
{"gaxios.request", CBM_SVC_HTTP},
174+
{"libcurl.version", CBM_SVC_HTTP},
175+
{"node-libcurl.Easy.perform", CBM_SVC_HTTP},
176+
{"curlpp.Easy.perform", CBM_SVC_HTTP},
177+
{"curlcpp.easy.perform", CBM_SVC_HTTP},
178+
{"hyperlocal.UnixConnector.call", CBM_SVC_HTTP},
179+
{"guzzlehttp/psr7.Utils.streamFor", CBM_SVC_HTTP},
180+
{"aiokafka.helpers.create_ssl_context", CBM_SVC_ASYNC},
181+
{"pykafka.topic.Topic.get_producer", CBM_SVC_ASYNC},
182+
{"librdkafka.rd_version", CBM_SVC_ASYNC},
183+
{"rdkafkacpp.version", CBM_SVC_ASYNC},
184+
{"rskafka.client.partition.write", CBM_SVC_ASYNC},
185+
{"aioamqp.connect", CBM_SVC_ASYNC},
186+
{"amqpstorm.Connection.channel", CBM_SVC_ASYNC},
187+
{"pamqp.frame.marshal", CBM_SVC_ASYNC},
188+
{"kombu.transport.pyamqp.Transport.establish_connection", CBM_SVC_ASYNC},
189+
{"amqprs.channel.basic_publish", CBM_SVC_ASYNC},
190+
{"amqpcpp.TcpChannel.publish", CBM_SVC_ASYNC},
191+
{"pynats.client.send", CBM_SVC_ASYNC},
192+
{"jnats.options.build", CBM_SVC_ASYNC},
193+
{"aiomqtt.client.publish", CBM_SVC_ASYNC},
194+
{"amqtt.session.deliver", CBM_SVC_ASYNC},
195+
{"hbmqtt.session.deliver", CBM_SVC_ASYNC},
196+
{"umqtt.simple.publish", CBM_SVC_ASYNC},
197+
{"mqttools.client.publish", CBM_SVC_ASYNC},
198+
{"emqtt.publish", CBM_SVC_ASYNC},
199+
{"rumqtt.client.publish", CBM_SVC_ASYNC},
200+
{"github.com/256dpi/gomqtt/client.Publish", CBM_SVC_ASYNC},
201+
{"libmosquitto.publish", CBM_SVC_ASYNC},
202+
{"mosquittopp.publish", CBM_SVC_ASYNC},
203+
{"google.cloud.pubsublite.admin.create_topic", CBM_SVC_ASYNC},
204+
{"gocelery.client.delay", CBM_SVC_ASYNC},
205+
{"_wgetenv", CBM_SVC_CONFIG},
206+
{"_wgetenv_s", CBM_SVC_CONFIG},
207+
{"os.getenvb", CBM_SVC_CONFIG},
208+
{"qgetenv", CBM_SVC_CONFIG},
209+
{"@dotenvx/dotenvx.config", CBM_SVC_CONFIG},
210+
{"vlucas/phpdotenv.load", CBM_SVC_CONFIG},
211+
{"apiflask.scaffold.get", CBM_SVC_ROUTE_REG},
212+
{"honox.factory.createRoute", CBM_SVC_ROUTE_REG},
213+
{"fasthttprouter.Router.GET", CBM_SVC_ROUTE_REG},
214+
{"github.com/vektah/gqlparser/v2.LoadSchema", CBM_SVC_GRAPHQL},
215+
{"github.com/Yamashou/gqlgenc.client.Post", CBM_SVC_GRAPHQL},
216+
{"aiogqlc.client.execute", CBM_SVC_GRAPHQL},
217+
{NULL, CBM_SVC_NONE},
218+
};
219+
ASSERT_EQ(svc_case_mismatches(cases), 0);
220+
ASSERT_STR_EQ(cbm_service_pattern_broker("aiokafka.helpers.create_ssl_context"), "kafka");
221+
ASSERT_STR_EQ(cbm_service_pattern_broker("umqtt.simple.publish"), "mqtt");
222+
ASSERT_STR_EQ(cbm_service_pattern_broker("com.acme.events.KafkaProducer.send"), "kafka");
223+
ASSERT_STR_EQ(cbm_service_pattern_broker("github.com/rabbitmq/amqp091-go.Channel.Publish"),
224+
"rabbitmq");
225+
PASS();
226+
}
227+
51228
TEST(infrascan_route_nodes_skip_bad_http_url_paths) {
52229
cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/cbm_infrascan_route_guard");
53230
ASSERT_NOT_NULL(gb);
@@ -128,6 +305,8 @@ TEST(infrascan_http_calls_join_matching_handler_route) {
128305

129306
SUITE(infrascan) {
130307
RUN_TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths);
308+
RUN_TEST(infrascan_service_pattern_match_rejects_ids_inside_words);
309+
RUN_TEST(infrascan_service_pattern_match_keeps_real_library_qns);
131310
RUN_TEST(infrascan_route_nodes_skip_bad_http_url_paths);
132311
RUN_TEST(infrascan_http_calls_join_matching_handler_route);
133312
}

0 commit comments

Comments
 (0)