optimise out string copies in recursive descent

Author: braindigitalis

include/string.h

@@ -229,6 +229,29 @@ const char* gc_strdup(basic_ctx* ctx, const char* string);
  */
 const char* gc_try_concat(struct basic_ctx *ctx, const char* s1, const char* s2);
 
+/**
+ * @brief Stream a string literal directly from the tokenizer into the GC arena.
+ *
+ * This function optimizes the parsing of string literals by bypassing the
+ * intermediate stack buffer. It reads characters directly from the source
+ * pointer, performing a single-pass copy into the BASIC string arena.
+ *
+ * Like gc_try_concat(), this operation is atomic: if the literal length
+ * exceeds MAX_STRINGLEN or available arena space, the allocator pointer
+ * is not advanced, and NULL is returned.
+ *
+ * @note
+ * - This function handles the advancing of the internal tokenizer pointer
+ * as it consumes characters.
+ * - The resulting string is NUL-terminated and committed to the arena
+ * only upon successful completion of the parse.
+ *
+ * @param ctx BASIC interpreter context.
+ * @return const char* Pointer to the arena-allocated string literal,
+ * or NULL on overflow/error.
+ */
+const char* gc_from_tokenizer_string(struct basic_ctx *ctx);
+
 /**
  * @brief Reset the BASIC temporary string arena.
  *

src/basic/unified_expression.c

@@ -103,18 +103,19 @@ static up_value up_factor(struct basic_ctx *ctx) {
 			return up_make_int(n);
 		}
 		case STRING: {
-			char string[MAX_STRINGLEN];
-			if (!tokenizer_string(string, sizeof(string), ctx)) {
-				tokenizer_error_print(ctx, "Bad string literal");
-				return up_make_str("");
-			}
-			/* IMPORTANT: do NOT consume a ')' here; that belongs to the caller (e.g., func args). */
-			accept(STRING, ctx);
-			const char* sv = gc_strdup(ctx, string);
+			const char* sv = gc_from_tokenizer_string(ctx);
+
 			if (!sv) {
-				tokenizer_error_print(ctx, "up_factor: Out of memory!");
+				tokenizer_error_print(ctx, "String literal too long");
 				return up_make_str("");
 			}
+
+			/* * We don't use 'accept(STRING)' here because gc_from_tokenizer_string
+			 * has already advanced the internal pointer past the literal.
+			 * We just need to update the cached token.
+			 */
+			tokenizer_next(ctx);
+
 			return up_make_str(sv);
 		}
 		case VARIABLE: {
@@ -418,7 +419,11 @@ static up_value up_relation_expr(struct basic_ctx *ctx) {
 				tokenizer_error_print(ctx, "Cannot compare string with number");
 				result = 0;
 			} else {
-				int cmp = strcmp(lhs.v.s ? lhs.v.s : "", rhs.v.s ? rhs.v.s : "");
+				/* Because empties are interned, identical ptrs match without strcmp */
+				int cmp = 0;
+				if (lhs.v.s != rhs.v.s) {
+					cmp = strcmp(lhs.v.s, rhs.v.s);
+				}
 				if (op == LESSTHAN) {
 					result = (mode == 1) ? (cmp <= 0) : (cmp < 0);
 				} else if (op == GREATERTHAN) {

src/basic/variable.c

@@ -357,23 +357,39 @@ void basic_set_double_variable(const char* var, double value, struct basic_ctx*
 	}
 }
 
+static inline char get_sigil(const char *v) {
+	if (!v || !*v) {
+		return 0;
+	}
+	// Walk to the null terminator
+	while (*v) {
+		v++;
+	}
+	// Look at the character immediately preceding the null
+	return *(v - 1);
+}
+
 /**
  * @brief Returns true if 'varname' starts with FN
  * (is a function call)
- * 
+ *
  * @param varname variable name to check
  * @return char 1 if variable name is a function call, 0 if it is not
  */
 char varname_is_function(const char* varname) {
-	return varname && (*varname == 'F' && *(varname + 1) == 'N' && !strchr(varname, '#') && !strchr(varname, '$'));
+	if (!varname || varname[0] != 'F' || varname[1] != 'N') {
+		return 0;
+	}
+	char s = get_sigil(varname);
+	return (s != '$' && s != '#');
 }
 
 char varname_is_string_function(const char* varname) {
-	return varname && (*varname == 'F' && *(varname + 1) == 'N' && strchr(varname, '$') && !strchr(varname, '#'));
+	return varname && varname[0] == 'F' && varname[1] == 'N' && get_sigil(varname) == '$';
 }
 
 char varname_is_double_function(const char* varname) {
-	return varname && (*varname == 'F' && *(varname + 1) == 'N' && strchr(varname, '#') && !strchr(varname, '$'));
+	return varname && varname[0] == 'F' && varname[1] == 'N' && get_sigil(varname) == '#';
 }
 
 const char* basic_get_string_variable(const char* var, struct basic_ctx* ctx) {

src/string.c

@@ -231,8 +231,32 @@ const char* gc_strdup(basic_ctx* ctx, const char* string)
 	return dest;
 }
 
-int gc(basic_ctx* ctx)
-{
+const char* gc_from_tokenizer_string(struct basic_ctx *ctx) {
+	char* dest = ctx->string_gc_storage_next;
+	char* end = ctx->string_gc_storage + STRING_GC_AREA_SIZE;
+	char* out = dest;
+	size_t cur = 0;
+
+	if (*ctx->ptr == '"') {
+		ctx->ptr++;
+	}
+
+	while (*ctx->ptr && *ctx->ptr != '"') {
+		if (out >= end - 1 || ++cur >= MAX_STRINGLEN) {
+			return NULL;
+		}
+		*out++ = *ctx->ptr++;
+	}
+
+	if (*ctx->ptr == '"') {
+		ctx->ptr++; // Skip closing quote
+	}
+	*out = '\0';
+	ctx->string_gc_storage_next = out + 1;
+	return dest;
+}
+
+int gc(basic_ctx* ctx) {
 	if (ctx->string_gc_storage && ctx->string_gc_storage_next) {
 		/* Strings start at +1 as the base ptr is for empties */
 		ctx->string_gc_storage_next = ctx->string_gc_storage + 1;