diff --git a/examples/business-id-runtime/pom.xml b/examples/business-id-runtime/pom.xml index 4cd2051d7..322678f78 100644 --- a/examples/business-id-runtime/pom.xml +++ b/examples/business-id-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../../pom.xml teaql-business-id-example diff --git a/examples/conformance/lib/pom.xml b/examples/conformance/lib/pom.xml index eae44f2c7..0eabc1b7e 100644 --- a/examples/conformance/lib/pom.xml +++ b/examples/conformance/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.557-RELEASE + 6.3.0 UTF-8 true diff --git a/examples/conformance/pom.xml b/examples/conformance/pom.xml index 9c5443e6d..f32c5cea9 100644 --- a/examples/conformance/pom.xml +++ b/examples/conformance/pom.xml @@ -16,7 +16,7 @@ 1.0.0 UTF-8 true - 1.557-RELEASE + 6.3.0 diff --git a/examples/http-resilience/pom.xml b/examples/http-resilience/pom.xml index 37d50b531..268d843a3 100644 --- a/examples/http-resilience/pom.xml +++ b/examples/http-resilience/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - io.teaqlteaql-java-parent1.557-RELEASE../../pom.xml + io.teaqlteaql-java-parent6.3.0../../pom.xml teaql-http-resilience-example io.teaqlteaql-runtime diff --git a/examples/inheritance-save/pom.xml b/examples/inheritance-save/pom.xml index 238c7ee43..88c7c5c1d 100644 --- a/examples/inheritance-save/pom.xml +++ b/examples/inheritance-save/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - io.teaqlteaql-java-parent1.557-RELEASE../../pom.xml + io.teaqlteaql-java-parent6.3.0../../pom.xml io.teaql.examplesinherited-save-example com.exampleinheritance-matching-service-core0.0.1-SNAPSHOT diff --git a/examples/object-storage/pom.xml b/examples/object-storage/pom.xml index 7c082fe5f..57c1768af 100644 --- a/examples/object-storage/pom.xml +++ b/examples/object-storage/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - io.teaqlteaql-java-parent1.557-RELEASE../../pom.xml + io.teaqlteaql-java-parent6.3.0../../pom.xml teaql-object-storage-example io.teaqlteaql-runtime diff --git a/examples/order-management/java-app-console/pom.xml b/examples/order-management/java-app-console/pom.xml index fcea2c6f4..2c4e154e8 100644 --- a/examples/order-management/java-app-console/pom.xml +++ b/examples/order-management/java-app-console/pom.xml @@ -11,7 +11,7 @@ 21 UTF-8 - 1.557-RELEASE + 6.3.0 diff --git a/examples/order-management/java-lib-core/lib/pom.xml b/examples/order-management/java-lib-core/lib/pom.xml index 2fcf76bd7..01159a1ff 100644 --- a/examples/order-management/java-lib-core/lib/pom.xml +++ b/examples/order-management/java-lib-core/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.557-RELEASE + 6.3.0 UTF-8 true diff --git a/examples/round-trip-reference-runtime/pom.xml b/examples/round-trip-reference-runtime/pom.xml index 265b611be..69dde59ef 100644 --- a/examples/round-trip-reference-runtime/pom.xml +++ b/examples/round-trip-reference-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../../pom.xml teaql-round-trip-reference-example diff --git a/examples/school-management/lib/pom.xml b/examples/school-management/lib/pom.xml index 06d3ffb40..982a09839 100644 --- a/examples/school-management/lib/pom.xml +++ b/examples/school-management/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.557-RELEASE + 6.3.0 UTF-8 true diff --git a/examples/school-management/pom.xml b/examples/school-management/pom.xml index e7c6d0ca2..45e61b182 100644 --- a/examples/school-management/pom.xml +++ b/examples/school-management/pom.xml @@ -16,7 +16,7 @@ 0.0.1-SNAPSHOT UTF-8 true - 1.557-RELEASE + 6.3.0 diff --git a/examples/security-foundations/pom.xml b/examples/security-foundations/pom.xml index 72a89e3eb..043dfb5c2 100644 --- a/examples/security-foundations/pom.xml +++ b/examples/security-foundations/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../../pom.xml teaql-security-foundations-example diff --git a/examples/shared-load-state/pom.xml b/examples/shared-load-state/pom.xml index c9cf6df0a..89287c85a 100644 --- a/examples/shared-load-state/pom.xml +++ b/examples/shared-load-state/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - io.teaqlteaql-java-parent1.557-RELEASE + io.teaqlteaql-java-parent6.3.0 ../../pom.xml shared-load-state-example diff --git a/examples/standard-order/pom.xml b/examples/standard-order/pom.xml index e68edf37b..ef492e2df 100644 --- a/examples/standard-order/pom.xml +++ b/examples/standard-order/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 io.teaqlteaql-java-parent - 1.557-RELEASE../../pom.xml + 6.3.0../../pom.xml teaql-standard-order-example 21 diff --git a/examples/trace-chain/lib/pom.xml b/examples/trace-chain/lib/pom.xml index 20467e3f6..d92318d76 100644 --- a/examples/trace-chain/lib/pom.xml +++ b/examples/trace-chain/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.557-RELEASE + 6.3.0 UTF-8 true @@ -67,7 +67,7 @@ io.teaql teaql-sqlite - 1.557-RELEASE + 6.3.0 io.teaql diff --git a/examples/trace-chain/pom.xml b/examples/trace-chain/pom.xml index 8141f0923..3df008b9c 100644 --- a/examples/trace-chain/pom.xml +++ b/examples/trace-chain/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../../pom.xml teaql-trace-chain-example diff --git a/pom.xml b/pom.xml index 98359d085..065bfd24b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 pom teaql-java-parent diff --git a/releases/6.3.0.md b/releases/6.3.0.md new file mode 100644 index 000000000..120d58b2b --- /dev/null +++ b/releases/6.3.0.md @@ -0,0 +1,22 @@ +# 6.3.0: JSON and human runtime logging + +Both `json` and `human` logging presentations are supported. JSON remains the +default. Select the presentation when composing the runtime logger; an explicit +application option takes precedence over the server `TEAQL_LOG_FORMAT` setting. +The exact accepted values are `json` and `human`. + +Both presentations use the same privacy-safe event projection. Human SQL is +readable and indented across multiple lines. File output preserves severity, +UTF-8 whole-event rotation, UTC-day rollover, retention and concurrent writers. +A non-empty file using the other presentation is rejected before archive changes. +Plaintext SQL acknowledgement and existing privacy controls retain their meaning. + +Choose a fresh destination when changing an existing file's presentation. Existing +generated APIs do not require regeneration for this logging feature. + + +Tracking: https://github.com/teaql/teaql-java/issues/286; coordinated release https://github.com/teaql/teaql-conformance/issues/248. + +Release status and the source, internal-artifact and public-delivery gates are +recorded separately in teaql/teaql-conformance, +`evidence/2026-11-18-logging-release`. These notes do not establish publication. diff --git a/teaql-android/pom.xml b/teaql-android/pom.xml index 878162428..6f5fd8064 100644 --- a/teaql-android/pom.xml +++ b/teaql-android/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-android diff --git a/teaql-business-id-jdbc/pom.xml b/teaql-business-id-jdbc/pom.xml index 0a050b215..1049b6722 100644 --- a/teaql-business-id-jdbc/pom.xml +++ b/teaql-business-id-jdbc/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-cloud/pom.xml b/teaql-cloud/pom.xml index 3f6e2909f..c88013601 100644 --- a/teaql-cloud/pom.xml +++ b/teaql-cloud/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-cloud teaql-cloud diff --git a/teaql-context-runtime-tools/pom.xml b/teaql-context-runtime-tools/pom.xml index 38ef0c7be..e8ead8b17 100644 --- a/teaql-context-runtime-tools/pom.xml +++ b/teaql-context-runtime-tools/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-context-runtime-tools diff --git a/teaql-core/pom.xml b/teaql-core/pom.xml index 102865067..29e0308dd 100644 --- a/teaql-core/pom.xml +++ b/teaql-core/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-data-service-sql/pom.xml b/teaql-data-service-sql/pom.xml index 2098e1432..5f4179b02 100644 --- a/teaql-data-service-sql/pom.xml +++ b/teaql-data-service-sql/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-db2/pom.xml b/teaql-db2/pom.xml index 29e94a204..f4d07fd61 100644 --- a/teaql-db2/pom.xml +++ b/teaql-db2/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-db2 teaql-db2 diff --git a/teaql-dm8/pom.xml b/teaql-dm8/pom.xml index 29795df7f..95abd08b4 100644 --- a/teaql-dm8/pom.xml +++ b/teaql-dm8/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-dm8 teaql-dm8 diff --git a/teaql-duckdb/pom.xml b/teaql-duckdb/pom.xml index 4cececa5a..b4d51a903 100644 --- a/teaql-duckdb/pom.xml +++ b/teaql-duckdb/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-duckdb teaql-duckdb diff --git a/teaql-dynamic-fields-api/pom.xml b/teaql-dynamic-fields-api/pom.xml index fae9e7f30..e13a83f0c 100644 --- a/teaql-dynamic-fields-api/pom.xml +++ b/teaql-dynamic-fields-api/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-dynamic-fields-jdbc/pom.xml b/teaql-dynamic-fields-jdbc/pom.xml index 17ffdd834..a50e1e70d 100644 --- a/teaql-dynamic-fields-jdbc/pom.xml +++ b/teaql-dynamic-fields-jdbc/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-hana/pom.xml b/teaql-hana/pom.xml index d21269bff..4fdc8a78d 100644 --- a/teaql-hana/pom.xml +++ b/teaql-hana/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-hana teaql-hana diff --git a/teaql-jackson/pom.xml b/teaql-jackson/pom.xml index affedf4ae..8c1f255fa 100644 --- a/teaql-jackson/pom.xml +++ b/teaql-jackson/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-jackson diff --git a/teaql-mssql/pom.xml b/teaql-mssql/pom.xml index 678bf9c48..14ec88182 100644 --- a/teaql-mssql/pom.xml +++ b/teaql-mssql/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-mssql teaql-mssql diff --git a/teaql-mysql/pom.xml b/teaql-mysql/pom.xml index ebbc0b5ff..3d5cdbdd4 100644 --- a/teaql-mysql/pom.xml +++ b/teaql-mysql/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-mysql diff --git a/teaql-object-storage-s3/pom.xml b/teaql-object-storage-s3/pom.xml index a49b5e369..dd6acd62d 100644 --- a/teaql-object-storage-s3/pom.xml +++ b/teaql-object-storage-s3/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - io.teaqlteaql-java-parent1.557-RELEASE + io.teaqlteaql-java-parent6.3.0 teaql-object-storage-s3 S3-compatible provider for context-owned TeaQL binary object storage 2.55.16 diff --git a/teaql-opentelemetry/pom.xml b/teaql-opentelemetry/pom.xml index ed0f665be..27c59cd47 100644 --- a/teaql-opentelemetry/pom.xml +++ b/teaql-opentelemetry/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-opentelemetry diff --git a/teaql-oracle/pom.xml b/teaql-oracle/pom.xml index 873b231e0..97a7a0563 100644 --- a/teaql-oracle/pom.xml +++ b/teaql-oracle/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-oracle teaql-oracle diff --git a/teaql-postgres/pom.xml b/teaql-postgres/pom.xml index 58836c097..4180cb3af 100644 --- a/teaql-postgres/pom.xml +++ b/teaql-postgres/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-postgres teaql-postgres diff --git a/teaql-provider-jdbc/pom.xml b/teaql-provider-jdbc/pom.xml index 5e8f3d1c0..998f0a792 100644 --- a/teaql-provider-jdbc/pom.xml +++ b/teaql-provider-jdbc/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-provider-spring-jdbc/pom.xml b/teaql-provider-spring-jdbc/pom.xml index bdb64c528..18efad683 100644 --- a/teaql-provider-spring-jdbc/pom.xml +++ b/teaql-provider-spring-jdbc/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-query-json/pom.xml b/teaql-query-json/pom.xml index a6979684d..2874d62a7 100644 --- a/teaql-query-json/pom.xml +++ b/teaql-query-json/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-query-json diff --git a/teaql-redis/pom.xml b/teaql-redis/pom.xml index 182345f2c..f345b29ee 100644 --- a/teaql-redis/pom.xml +++ b/teaql-redis/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-runtime-log/pom.xml b/teaql-runtime-log/pom.xml index 4fbcf8fb0..dd6e304fa 100644 --- a/teaql-runtime-log/pom.xml +++ b/teaql-runtime-log/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-runtime-log diff --git a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/LogManager.java b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/LogManager.java index 173e6ea69..37f4cdce8 100644 --- a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/LogManager.java +++ b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/LogManager.java @@ -1,342 +1,75 @@ package io.teaql.runtime.log; -import io.teaql.runtime.config.TeaQLEnv; -import io.teaql.runtime.RuntimeLogSink; -import io.teaql.runtime.RawAuditEvent; +import io.teaql.core.ExecutionMetadata; import io.teaql.core.TraceNode; - -import java.io.File; -import java.io.FileOutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.Comparator; +import io.teaql.core.UserContext; +import io.teaql.runtime.LogPrivacy; +import io.teaql.runtime.RawAuditEvent; +import io.teaql.runtime.RuntimeLogSink; +import io.teaql.runtime.RuntimeLogger; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicLong; -import java.util.zip.GZIPOutputStream; -import java.io.FileInputStream; +import java.util.Map; +/** Optional formatting/custom-sink adapter using the runtime-owned JSONL destination. */ public class LogManager implements RuntimeLogSink { - private static final LogManager INSTANCE = new LogManager(); - private static final String EXTREME_TEST_FLAG = "__i_agree_to_disable_runtime_trace_only_for_extreme_performance_testing"; - - private final String endpoint; - private final long maxSize; - private final int maxFiles; - - private final BlockingQueue queue; - private final Thread workerThread; - - private FileChannel currentChannel; - private final AtomicLong currentSize = new AtomicLong(0); - private final Object fileLock = new Object(); - private long nextMidnightMillis; - private volatile boolean headerWritten = false; - - private String determineEndpoint() { - String mode = TeaQLEnv.get("TEAQL_TRACE_MODE"); - if ("off".equals(mode)) { - String ack = TeaQLEnv.get("TEAQL_TRACE_OFF_ACK"); - if (EXTREME_TEST_FLAG.equals(ack)) { - return "off"; - } - } - String val = TeaQLEnv.get("TEAQL_LOG_ENDPOINT"); - if (val != null && !val.trim().isEmpty()) { - return val; - } - String domain = TeaQLEnv.get("TEAQL_DOMAIN"); - if (domain != null && !domain.trim().isEmpty()) { - return domain.trim() + ".log"; - } - String command = System.getProperty("sun.java.command"); - String exeName = "teaql"; - if (command != null && !command.trim().isEmpty()) { - exeName = command.split(" ")[0]; - int lastDot = exeName.lastIndexOf('.'); - if (lastDot >= 0 && lastDot < exeName.length() - 1) { - exeName = exeName.substring(lastDot + 1); - } - } - return exeName + ".log"; - } - - private LogManager() { - this.endpoint = determineEndpoint(); - this.maxSize = TeaQLEnv.getSizeInBytes("TEAQL_LOG_MAX_SIZE", 50 * 1024 * 1024L); // default 50MB - this.maxFiles = TeaQLEnv.getInt("TEAQL_LOG_MAX_FILES", 7); - - this.queue = new ArrayBlockingQueue<>(10000); + private LogManager() { } + public static LogManager getInstance() { return INSTANCE; } - if (!"off".equals(this.endpoint) && !"stdout".equals(this.endpoint)) { - initFileChannel(); - } - - this.workerThread = new Thread(this::processQueue, "TeaQL-LogWriter-Thread"); - this.workerThread.setDaemon(true); - this.workerThread.start(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - // Drain remaining log entries before JVM exits - Runnable task; - while ((task = queue.poll()) != null) { - try { task.run(); } catch (Exception ignored) {} - } - synchronized (fileLock) { - if (currentChannel != null) { - try { currentChannel.force(true); currentChannel.close(); } catch (Exception ignored) {} - } - } - }, "TeaQL-LogFlush-Shutdown")); - } - - public static LogManager getInstance() { - return INSTANCE; - } - - @Override - public boolean requiresSensitiveSqlData() { + @Override public boolean requiresSensitiveSqlData() { return LogConfig.getInstance().includesSensitiveSqlData(); } - private void calculateNextMidnight() { - LocalDateTime tomorrowMidnight = LocalDateTime.now().plusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0); - this.nextMidnightMillis = tomorrowMidnight.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - } - - private void initFileChannel() { - synchronized (fileLock) { - try { - Path path = Paths.get(this.endpoint); - if (path.getParent() != null) { - Files.createDirectories(path.getParent()); - } - FileOutputStream fos = new FileOutputStream(path.toFile(), true); - this.currentChannel = fos.getChannel(); - this.currentSize.set(this.currentChannel.size()); - calculateNextMidnight(); - } catch (Exception e) { - System.err.println("TeaQL LogManager Failed to initialize file channel: " + e.getClass().getSimpleName()); - } - } - } - - private void processQueue() { - while (!Thread.currentThread().isInterrupted()) { - try { - Runnable task = queue.take(); - task.run(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } catch (Exception e) { - System.err.println("TeaQL LogManager Exception in worker thread: " + e.getClass().getSimpleName()); - } - } - } - - private void writeHeaderIfNeeded() { - if (headerWritten || "off".equals(endpoint)) return; - synchronized (fileLock) { - if (headerWritten) return; - try { - java.net.URL url = getClass().getResource("/log_header.txt"); - String header = "================================================================================\n" + - "๐Ÿš€ TEAQL Holographic Trace Log\n" + - "================================================================================"; - if (url != null) { - try (java.io.InputStream is = url.openStream(); - java.util.Scanner s = new java.util.Scanner(is, StandardCharsets.UTF_8.name()).useDelimiter("\\A")) { - header = readHeaderOrDefault(s); - } - } - byte[] bytes = (header + "\n").getBytes(StandardCharsets.UTF_8); - if ("stdout".equals(endpoint)) { - System.out.print(new String(bytes, StandardCharsets.UTF_8)); - } else if (currentChannel != null) { - currentChannel.write(ByteBuffer.wrap(bytes)); - currentSize.addAndGet(bytes.length); - } - } catch (Exception e) { - System.err.println("TeaQL LogManager Failed to write header: " + e.getClass().getSimpleName()); - } finally { - headerWritten = true; - } - } - } - - private void asyncWrite(String content, CustomLogSink customSink) { - if (!queue.offer(() -> syncWrite(content, customSink))) { - // Queue is full, drop or print to standard error to prevent blocking main business logic - System.err.println("TeaQL LogManager queue full, dropped log."); - } - } - - private void syncWrite(String content, CustomLogSink customSink) { - if (content == null || content.isEmpty()) return; - if (customSink != null) { - customSink.onLog(content); - } - - if ("off".equals(endpoint)) return; - writeHeaderIfNeeded(); - - byte[] bytes = (content + "\n").getBytes(StandardCharsets.UTF_8); - - if ("stdout".equals(endpoint)) { - System.out.print(new String(bytes, StandardCharsets.UTF_8)); - return; - } - - synchronized (fileLock) { - if (currentChannel == null) return; - - try { - boolean timeToRotate = System.currentTimeMillis() >= nextMidnightMillis; - boolean sizeToRotate = currentSize.get() + bytes.length > maxSize; - - if (timeToRotate || sizeToRotate) { - rotateLogFile(); - if (timeToRotate) { - calculateNextMidnight(); - } - } - - currentChannel.write(ByteBuffer.wrap(bytes)); - currentSize.addAndGet(bytes.length); - } catch (Exception e) { - System.err.println("TeaQL LogManager Failed to write to log file: " + e.getClass().getSimpleName()); - } - } - } - - private void rotateLogFile() { + private void custom(UserContext context, String content, RuntimeLogger logger) { try { - if (currentChannel != null) { - currentChannel.close(); - } - - File currentFile = new File(endpoint); - File backupFile = null; - if (currentFile.exists()) { - String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); - backupFile = new File(endpoint + "." + timestamp); - Files.move(currentFile.toPath(), backupFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } - - initFileChannel(); - cleanupOldFiles(); - - if (backupFile != null && backupFile.exists()) { - compressAsync(backupFile); - } - - } catch (Exception e) { - System.err.println("TeaQL LogManager Failed to rotate log file: " + e.getClass().getSimpleName()); + CustomLogSink sink = resolveCustomSink(context); + if (sink != null) sink.onLog(content); + } catch (RuntimeException unavailable) { + RuntimeLogger.failure(logger.logName(), "custom_log_sink_failed"); } } - private void compressAsync(File source) { - CompletableFuture.runAsync(() -> { - File target = new File(source.getAbsolutePath() + ".gz"); - try (FileInputStream fis = new FileInputStream(source); - FileOutputStream fos = new FileOutputStream(target); - GZIPOutputStream gos = new GZIPOutputStream(fos)) { - - byte[] buffer = new byte[8192]; - int len; - while ((len = fis.read(buffer)) > 0) { - gos.write(buffer, 0, len); - } - gos.finish(); - source.delete(); - } catch (Exception e) { - System.err.println("TeaQL LogManager Failed to compress log file: " + e.getClass().getSimpleName()); - } - }); - } - - private void cleanupOldFiles() { - File currentFile = new File(endpoint); - File parentDir = currentFile.getParentFile(); - if (parentDir == null) { - parentDir = new File("."); - } - - final String baseName = currentFile.getName(); - File[] files = parentDir.listFiles((dir, name) -> name.startsWith(baseName + ".")); - - if (files != null && files.length > maxFiles) { - Arrays.sort(files, Comparator.comparingLong(File::lastModified)); - int filesToDelete = files.length - maxFiles; - for (int i = 0; i < filesToDelete; i++) { - if (!files[i].delete()) { - System.err.println("TeaQL LogManager Failed to delete old log file: " + files[i].getName()); - } - } - } + @Override public void writeExecutionLog(UserContext context, ExecutionMetadata metadata) { + if (!LogConfig.getInstance().shouldLogSql(metadata.getParameterizedQuery())) return; + var logger = RuntimeLogger.forContext(context); + var safe = LogPrivacy.sql(metadata, requiresSensitiveSqlData() && LogPrivacy.plaintextEnabled()); + try { custom(context, LogFormatterFactory.getFormatter().formatExecutionLog(safe), logger); } + catch (RuntimeException unavailable) { RuntimeLogger.failure(logger.logName(), "log_projection_failed"); } + logger.writeSql(safe, requiresSensitiveSqlData()); } - @Override - public void writeExecutionLog(io.teaql.core.UserContext context, io.teaql.core.ExecutionMetadata metadata) { - if (!LogConfig.getInstance().shouldLogSql(metadata.getParameterizedQuery())) { - return; - } - metadata = io.teaql.runtime.LogPrivacy.sql(metadata, io.teaql.runtime.LogPrivacy.plaintextEnabled()); - String content = LogFormatterFactory.getFormatter().formatExecutionLog(metadata); - CustomLogSink customSink = resolveCustomSink(context); - asyncWrite(content, customSink); + @Override public void writeAuditEvent(UserContext context, RawAuditEvent event) { + List changes = event.changes().stream() + .map(change -> new FieldChange(change.field(), change.oldValue(), change.newValue())).toList(); + writeAuditLog(context, event.traceChain(), new AuditEvent(event.entityType(), event.entityId(), event.kind().name(), changes)); } - @Override - public void writeAuditEvent(io.teaql.core.UserContext context, RawAuditEvent event) { - java.util.List changes = event.changes().stream() - .map(change -> new FieldChange( - change.field(), change.oldValue(), change.newValue())) - .collect(java.util.stream.Collectors.toList()); - writeAuditLog(context, event.traceChain(), new AuditEvent( - event.entityType(), event.entityId(), event.kind().name(), changes)); - } - - public void writeAuditLog(io.teaql.core.UserContext context, List traceChain, AuditEvent event) { - if (!LogConfig.getInstance().shouldLogAudit(event.getEntityType())) { - return; - } - boolean allow = io.teaql.runtime.LogPrivacy.plaintextEnabled(); - java.util.List secrets = new java.util.ArrayList<>(); - java.util.List safeChanges = event.getChanges().stream().map(change -> { - if (allow && !io.teaql.runtime.LogPrivacy.credential(change.getField()) - && !io.teaql.runtime.LogPrivacy.hasCredentials(change.getOldValue()) - && !io.teaql.runtime.LogPrivacy.hasCredentials(change.getNewValue())) return change; + public void writeAuditLog(UserContext context, List traceChain, AuditEvent event) { + if (!LogConfig.getInstance().shouldLogAudit(event.getEntityType())) return; + boolean allow = LogPrivacy.plaintextEnabled(); + List secrets = new ArrayList<>(); + List safeChanges = event.getChanges().stream().map(change -> { + if (allow && !LogPrivacy.credential(change.getField()) + && !LogPrivacy.hasCredentials(change.getOldValue()) && !LogPrivacy.hasCredentials(change.getNewValue())) return change; secrets.add(change.getOldValue()); secrets.add(change.getNewValue()); - return new FieldChange(change.getField(), - change.getOldValue() == null ? null : io.teaql.runtime.LogPrivacy.REDACTED, - change.getNewValue() == null ? null : io.teaql.runtime.LogPrivacy.REDACTED); + return new FieldChange(change.getField(), change.getOldValue() == null ? null : LogPrivacy.REDACTED, + change.getNewValue() == null ? null : LogPrivacy.REDACTED); }).toList(); - event = new AuditEvent(event.getEntityType(), event.getEntityId(), event.getMutationKind(), safeChanges); - traceChain = io.teaql.runtime.LogPrivacy.trace(traceChain, secrets); - String content = LogFormatterFactory.getFormatter().formatAuditLog(traceChain, event); - CustomLogSink customSink = resolveCustomSink(context); - asyncWrite(content, customSink); - } - - protected String readHeaderOrDefault(java.util.Scanner s) { - return s.hasNext() ? s.next() : ""; - } - - protected CustomLogSink resolveCustomSink(io.teaql.core.UserContext context) { - return context != null ? context.capability(CustomLogSink.class) : null; + var safe = new AuditEvent(event.getEntityType(), event.getEntityId(), event.getMutationKind(), safeChanges); + var safeTrace = LogPrivacy.trace(traceChain, secrets); + var logger = RuntimeLogger.forContext(context); + try { custom(context, LogFormatterFactory.getFormatter().formatAuditLog(safeTrace, safe), logger); } + catch (RuntimeException unavailable) { RuntimeLogger.failure(logger.logName(), "log_projection_failed"); } + Map fields = new LinkedHashMap<>(); + fields.put("category", "AUDIT"); fields.put("outcome", "success"); fields.put("entity", safe.getEntityType()); + fields.put("entity_id", safe.getEntityId()); fields.put("fields", safeChanges); fields.put("trace_chain", safeTrace); + logger.write("INFO", "audit.event", "TeaQL audit event", fields); + } + + protected String readHeaderOrDefault(java.util.Scanner scanner) { return scanner.hasNext() ? scanner.next() : ""; } + protected CustomLogSink resolveCustomSink(UserContext context) { + return context == null ? null : context.capability(CustomLogSink.class); } } diff --git a/teaql-runtime-log/src/test/java/io/teaql/runtime/log/StructuredLogManagerTest.java b/teaql-runtime-log/src/test/java/io/teaql/runtime/log/StructuredLogManagerTest.java new file mode 100644 index 000000000..2494fb522 --- /dev/null +++ b/teaql-runtime-log/src/test/java/io/teaql/runtime/log/StructuredLogManagerTest.java @@ -0,0 +1,43 @@ +package io.teaql.runtime.log; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.TraceNode; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.RuntimeLogger; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import static org.junit.Assert.*; + +public class StructuredLogManagerTest { + @Rule public TemporaryFolder temporary = new TemporaryFolder(); + @Test public void optionalModuleUsesTheSameJsonLinesDestinationAndSafeCustomProjection() throws Exception { + var logger = new RuntimeLogger(new RuntimeLogger.Options("application", List.of(), Map.of(), temporary.getRoot().toPath())); + List captured = new ArrayList<>(); + var context = new DefaultUserContext(null) { + @Override public T capability(Class type) { + if (type == RuntimeLogger.class) return type.cast(logger); + if (type == CustomLogSink.class) return type.cast((CustomLogSink) captured::add); + return null; + } + }; + var metadata = new ExecutionMetadata(); + metadata.setParameterizedQuery("SELECT id FROM customer WHERE password = ?"); + metadata.setParameters(List.of("private-password-value")); + metadata.setDebugQuery("private-password-value"); metadata.setExecutionOutcome("retrying"); + LogManager.getInstance().writeExecutionLog(context, metadata); + LogManager.getInstance().writeAuditLog(context, List.of(new TraceNode("changed private-password-value")), + new AuditEvent("Customer", 42L, "UPDATE", List.of(new FieldChange("password", "private-password-value", "new-password-value")))); + String output = Files.readString(logger.path()); + assertFalse(output.contains("private-password-value")); assertFalse(output.contains("new-password-value")); + assertFalse(String.join("", captured).contains("private-password-value")); + var records = output.lines().toList(); + assertEquals(3, records.size()); + for (var record : records) System.out.println("TEAQL_LOG_MANAGER_RECORD " + record); + assertEquals(2, captured.size()); + } +} diff --git a/teaql-runtime/pom.xml b/teaql-runtime/pom.xml index 2bc329b8c..7a14f643e 100644 --- a/teaql-runtime/pom.xml +++ b/teaql-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-runtime diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java index 659331075..cac5ea5cd 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java @@ -25,11 +25,13 @@ public class DefaultUserContext implements UserContext, OptNullBasicTypeFromObjectGetter { private final TeaQLRuntime runtime; + private final RuntimeLogger runtimeLogger; private final Map storage = new ConcurrentHashMap<>(); private final List traceChain = new ArrayList<>(); public DefaultUserContext(TeaQLRuntime runtime) { this.runtime = runtime; + this.runtimeLogger = runtime == null ? RuntimeLogger.defaultLogger() : runtime.getLogger(); // Prepare immutable classpath resources during composition, not during error rendering. io.teaql.core.i18n.CheckerMessageCatalog.builtin(); io.teaql.core.checker.CheckerPresentation.Model.empty(); @@ -156,6 +158,7 @@ public T capability(Class capabilityType) { return null; } Object value = getObj(capabilityType.getName()); + if (value == null && capabilityType == RuntimeLogger.class) value = runtimeLogger; if (value == null && capabilityType == SchemaExecutor.class) { value = runtime.getSchemaExecutor(); if (value == null) { @@ -190,7 +193,7 @@ public T capability(Class capabilityType) { value = runtime.getRoundTripReferenceCodec(); } if (value == null && capabilityType == io.teaql.core.storage.ObjectStorageObserver.class) { - value = new RuntimeObjectStorageObserver(runtime == null ? RuntimeTelemetry.NOOP : runtime.getTelemetry()); + value = new RuntimeObjectStorageObserver(operationTelemetry()); } if (value == null && capabilityType == io.teaql.core.http.HttpPropagation.class) { RuntimeTelemetry telemetry = runtime == null ? RuntimeTelemetry.NOOP : runtime.getTelemetry(); @@ -198,8 +201,7 @@ public T capability(Class capabilityType) { carrier -> RuntimeTelemetry.injectSafely(telemetry, carrier); } if (value == null && capabilityType == io.teaql.core.http.HttpObservation.class) { - value = new RuntimeHttpObservation(runtime == null - ? RuntimeTelemetry.NOOP : runtime.getTelemetry()); + value = new RuntimeHttpObservation(operationTelemetry()); } if (value == null) { return null; @@ -256,9 +258,13 @@ public void removeFromRemoteCache(String key) { }, ignored -> "removed"); } + private RuntimeTelemetry operationTelemetry() { + return runtime == null ? new RuntimeLoggingTelemetry(RuntimeTelemetry.NOOP, runtimeLogger) : runtime.getOperationTelemetry(); + } + private T observeCache(String name, String operation, Supplier work, java.util.function.Function result) { - RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(runtime.getTelemetry(), + RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(operationTelemetry(), new RuntimeTelemetry.Operation("cache", name, Map.of("teaql.cache.operation", operation))); try { diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeDiagnosticLogSink.java b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeDiagnosticLogSink.java new file mode 100644 index 000000000..f0d86def0 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeDiagnosticLogSink.java @@ -0,0 +1,17 @@ +package io.teaql.runtime; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.UserContext; +import java.util.Map; + +/** The default server sink; existing text sinks remain explicit application choices. */ +public final class RuntimeDiagnosticLogSink extends DefaultTextRuntimeLogSink { + private final RuntimeLogger logger; + public RuntimeDiagnosticLogSink(RuntimeLogger logger) { this.logger = logger; } + @Override public void writeExecutionLog(UserContext context, ExecutionMetadata metadata) { logger.writeSql(metadata); } + @Override public void writeAuditEvent(UserContext context, RawAuditEvent event) { logger.writeAudit(event); } + @Override public void writeMutationGovernanceEvent(UserContext context, MutationGovernanceEvent event) { + if (event.firstOccurrence()) logger.write("WARN", "mutation.policy.warning", "TeaQL mutation policy warning", + Map.of("category", "AUDIT", "outcome", "rejected", "error_code", event.warningCode())); + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLogger.java b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLogger.java new file mode 100644 index 000000000..881f91b58 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLogger.java @@ -0,0 +1,336 @@ +package io.teaql.runtime; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.DeserializationFeature; +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.UserContext; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +/** Startup-owned structured destination, independent of optional OTel SDKs. */ +public final class RuntimeLogger { + public record Options(String logName, List modelNames, + Map environment, Path workingDirectory, String logFormat) { + public Options { + modelNames = modelNames == null ? List.of() : List.copyOf(modelNames); + environment = environment == null ? null : Map.copyOf(environment); + } + public Options(String logName, List modelNames, Map environment, Path workingDirectory) { + this(logName, modelNames, environment, workingDirectory, null); + } + public Options() { this(null, List.of(), null, null, null); } + } + + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Map GATES = new ConcurrentHashMap<>(); + private static final Map DEFAULTS = new HashMap<>(); + private static final List SETTINGS = List.of("TEAQL_DOMAIN", "TEAQL_LOG_ENDPOINT", "TEAQL_LOG_MAX_SIZE", "TEAQL_LOG_MAX_FILES", "TEAQL_LOG_FORMAT"); + private static final Pattern DEVICE = Pattern.compile("(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])"); + private static final Pattern SIZE = Pattern.compile("([0-9]+)\\s*(K|KB|M|MB|G|GB)?"); + private static final Pattern ARCHIVE = Pattern.compile("[1-9][0-9]*"); + private static final Pattern HUMAN_HEADER = Pattern.compile("^\\d{4}-\\d\\d-\\d\\dT\\S+(?:Z|\\+00:00) \\[TeaQL\\]\\[.*\\]\\[(?:INFO|WARN|ERROR)\\] .+ runtime=\\S+$"); + private static final Set EVENT_FIELDS = Set.of("timestamp", "level", "log_name", "runtime", "event", "message"); + private final String logName; + private final String logFormat; + private final Path path; + private final long maxSize; + private final int maxFiles; + private final String stream; + private final Object gate; + + public RuntimeLogger() { this(new Options()); } + + public RuntimeLogger(Options options) { + String validatedName = "teaql"; + try { + Map env = options.environment() == null ? loggingEnvironment() : options.environment(); + String domain = env.get("TEAQL_DOMAIN"); + String name = options.logName() != null ? options.logName() + : domain != null && !domain.isEmpty() ? domain + : options.modelNames().size() == 1 ? options.modelNames().get(0) : "teaql"; + validateName(name); + validatedName = name; + this.logName = name; + this.logFormat = options.logFormat() != null ? options.logFormat() : env.getOrDefault("TEAQL_LOG_FORMAT", "json"); + if (!Set.of("json", "human").contains(logFormat)) throw new IllegalArgumentException(); + this.maxSize = parseSize(env.getOrDefault("TEAQL_LOG_MAX_SIZE", "50M")); + String retention = env.getOrDefault("TEAQL_LOG_MAX_FILES", "7"); + if (!retention.matches("[0-9]+")) throw new IllegalArgumentException(); + this.maxFiles = Integer.parseInt(retention); + String endpoint = env.get("TEAQL_LOG_ENDPOINT"); + if (endpoint == null || endpoint.isEmpty()) endpoint = "./logs/" + name + ".log"; + if (Set.of("stdout", "stderr", "-").contains(endpoint)) { + this.stream = endpoint; + this.path = null; + this.gate = new Object(); + } else { + if (endpoint.startsWith("file:")) { + URI uri = URI.create(endpoint); + if (uri.getPath() == null || uri.getPath().isEmpty() || uri.getQuery() != null || uri.getFragment() != null + || uri.getHost() != null && !uri.getHost().equals("localhost")) throw new IllegalArgumentException(); + endpoint = uri.getPath(); + } else if (endpoint.contains("://")) throw new IllegalArgumentException(); + Path base = options.workingDirectory() == null ? Path.of("").toAbsolutePath() : options.workingDirectory().toAbsolutePath(); + Path destination = base.resolve(endpoint).normalize(); + Files.createDirectories(destination.getParent()); + destination = destination.getParent().toRealPath().resolve(destination.getFileName()); + try { destination = destination.toRealPath(); } + catch (NoSuchFileException absentOrRotated) { /* Reopened under ownership below. */ } + this.path = destination; + this.stream = null; + this.gate = GATES.computeIfAbsent(destination, ignored -> new Object()); + } + emit(record("INFO", "runtime.start", "TeaQL runtime started", Map.of())); + } catch (IOException | RuntimeException unavailable) { + failure(validatedName, "log_destination_unavailable"); + throw new IllegalStateException("Runtime logging initialization failed"); + } + } + + private static Map loggingEnvironment() { + Map result = new TreeMap<>(); + for (String setting : SETTINGS) { + String value = System.getenv(setting); + if (value != null) result.put(setting, value); + } + return result; + } + + public static synchronized RuntimeLogger defaultLogger() { + Map env = loggingEnvironment(); + Path cwd = Path.of("").toAbsolutePath().normalize(); + String key = cwd + "\n" + env; + return DEFAULTS.computeIfAbsent(key, ignored -> new RuntimeLogger(new Options(null, List.of(), env, cwd))); + } + + public static RuntimeLogger forContext(UserContext context) { + if (context != null) { + RuntimeLogger logger = context.capability(RuntimeLogger.class); + if (logger != null) return logger; + } + return defaultLogger(); + } + + public String logName() { return logName; } + public String logFormat() { return logFormat; } + public Path path() { return path; } + public long maxSize() { return maxSize; } + public int maxFiles() { return maxFiles; } + + private static void validateName(String name) throws IOException { + if (name == null || name.isEmpty() || name.equals(".") || name.equals("..") || name.endsWith(" ") || name.endsWith(".") + || DEVICE.matcher(name.split("\\.", -1)[0]).matches() + || name.chars().anyMatch(c -> Character.isISOControl(c) || "<>:\"/\\|?*".indexOf(c) >= 0)) throw new IllegalArgumentException(); + StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPORT).encode(CharBuffer.wrap(name)); + } + + public static long parseSize(String value) { + var match = SIZE.matcher(value.trim().toUpperCase(Locale.ROOT)); + if (!match.matches()) throw new IllegalArgumentException("Invalid log size"); + long amount = Long.parseLong(match.group(1)); + String suffix = match.group(2); + long multiplier = suffix == null ? 1 : switch (suffix) { + case "K", "KB" -> 1024; + case "M", "MB" -> 1024 * 1024; + case "G", "GB" -> 1024 * 1024 * 1024; + default -> throw new IllegalArgumentException("Invalid log size"); + }; + if (amount <= 0) throw new IllegalArgumentException("Invalid log size"); + return Math.multiplyExact(amount, multiplier); + } + + private Map record(String level, String event, String message, Map fields) { + Map record = new LinkedHashMap<>(fields); + record.put("timestamp", Instant.now().toString()); record.put("level", level); + record.put("log_name", logName); record.put("runtime", "java"); record.put("event", event); record.put("message", message); + return record; + } + + /** Accepts already-projected fields. Destination failures never escape this method. */ + public boolean write(String level, String event, String message, Map safeFields) { + try { + if (!Set.of("INFO", "WARN", "ERROR").contains(level)) throw new IllegalArgumentException(); + emit(record(level, event, message, safeFields)); + return true; + } catch (Exception unavailable) { failure(logName, "log_write_failed"); return false; } + } + + public static void failure(String name, String code) { + try { + System.err.println(JSON.writeValueAsString(Map.of("timestamp", Instant.now().toString(), "level", "ERROR", "log_name", name, + "runtime", "java", "event", "logging.failure", "message", "Runtime logging failed", "error_code", code))); + } catch (com.fasterxml.jackson.core.JsonProcessingException unavailable) { + // Reporting a logging failure must remain safe even if JSON + // serialization itself fails. Never disclose the original error. + System.err.println("{\"timestamp\":\"" + Instant.now() + "\",\"level\":\"ERROR\",\"log_name\":\"teaql\"," + + "\"runtime\":\"java\",\"event\":\"logging.failure\",\"message\":\"Runtime logging failed\"," + + "\"error_code\":\"log_reporting_failed\"}"); + } + } + + private SortedMap archives() throws IOException { + SortedMap entries = new TreeMap<>(Comparator.reverseOrder()); + String prefix = path.getFileName() + "."; + try (var files = Files.list(path.getParent())) { + for (Path file : files.toList()) { + String name = file.getFileName().toString(); + if (!name.startsWith(prefix)) continue; + String suffix = name.substring(prefix.length()); + if (!ARCHIVE.matcher(suffix).matches()) continue; + if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) throw new IOException(); + entries.put(Integer.parseInt(suffix), file); + } + } + return entries; + } + + private void emit(Map record) throws IOException { + String encoded = JSON.writeValueAsString(record); + byte[] data = (logFormat.equals("human") ? humanRecord(JSON.readTree(encoded)) : encoded + "\n").getBytes(StandardCharsets.UTF_8); + synchronized (gate) { + if (stream != null) { + var output = stream.equals("stderr") ? System.err : System.out; + output.write(data); output.flush(); + if (output.checkError()) throw new IOException(); + return; + } + Path lockPath = Path.of(path + ".lock"); + FileChannel ownership; + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(30); + while (true) { + try { ownership = FileChannel.open(lockPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); break; } + catch (FileAlreadyExistsException busy) { + if (System.nanoTime() >= deadline) throw new IOException(); + try { Thread.sleep(5); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); throw new IOException(); } + } + } + try (FileChannel acquired = ownership) { + checkPresentation(); // Before pruning or rotating any existing file. + var archives = archives(); + for (var archive : archives.entrySet()) if (archive.getKey() > maxFiles) Files.delete(archive.getValue()); + BasicFileAttributes active = null; + try { active = Files.readAttributes(path, BasicFileAttributes.class); } + catch (NoSuchFileException absent) { } + if (active != null) { + if (!active.isRegularFile()) throw new IOException(); + LocalDate day = active.lastModifiedTime().toInstant().atZone(ZoneOffset.UTC).toLocalDate(); + if (active.size() > 0 && (data.length > maxSize - active.size() || !day.equals(LocalDate.now(ZoneOffset.UTC)))) { + for (var archive : archives.entrySet()) { + int number = archive.getKey(); + if (number > maxFiles) continue; + if (number == maxFiles) Files.delete(archive.getValue()); + else Files.move(archive.getValue(), Path.of(path + "." + (number + 1))); + } + if (maxFiles == 0) Files.delete(path); else Files.move(path, Path.of(path + ".1")); + } + } + // Always reopen the active pathname after another writer rotates. + try (var output = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) { + ByteBuffer bytes = ByteBuffer.wrap(data); + while (bytes.hasRemaining()) if (output.write(bytes) == 0) throw new IOException(); + } + } finally { Files.delete(lockPath); } + } + } + + private static String humanText(String value, boolean escapeBackslash) { + StringBuilder output = new StringBuilder(); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (escapeBackslash && c == '\\') output.append("\\\\"); + else if (Character.isISOControl(c) || c == '\u2028' || c == '\u2029') + output.append(String.format(Locale.ROOT, "\\u%04x", (int) c)); + else output.append(c); + } + return output.toString(); + } + + private static String humanRecord(JsonNode record) { + StringBuilder output = new StringBuilder(); + output.append(humanText(record.get("timestamp").asText(), true)).append(" [TeaQL][") + .append(humanText(record.get("log_name").asText(), true)).append("][") + .append(record.get("level").asText()).append("] ") + .append(humanText(record.get("event").asText(), true)).append(" runtime=") + .append(humanText(record.get("runtime").asText(), true)).append("\n ") + .append(humanText(record.get("message").asText(), true)).append('\n'); + SortedSet keys = new TreeSet<>(); + record.fieldNames().forEachRemaining(keys::add); + keys.removeAll(EVENT_FIELDS); + for (String key : keys) { + JsonNode value = record.get(key); + if (key.equals("sql") && value.isTextual()) { + output.append(" sql:\n"); + for (String line : value.asText().split("\n", -1)) output.append(" ").append(humanText(line, true)).append('\n'); + } else output.append(" ").append(humanText(key, true)).append('=') + .append(humanText(value.toString(), false)).append('\n'); + } + return output.toString(); + } + + private void checkPresentation() throws IOException { + BasicFileAttributes active; + try { active = Files.readAttributes(path, BasicFileAttributes.class); } + catch (NoSuchFileException absent) { return; } + if (!active.isRegularFile()) throw new IOException(); + if (active.size() == 0) return; + String first; + try (var input = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { first = input.readLine(); } + if (first != null && logFormat.equals("human") && HUMAN_HEADER.matcher(first).matches()) return; + if (first != null && logFormat.equals("json")) { + try { + JsonNode record = JSON.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).readTree(first); + boolean valid = record != null && record.isObject(); + for (String key : EVENT_FIELDS) valid &= record != null && record.has(key) && record.get(key).isTextual(); + if (valid && Set.of("INFO", "WARN", "ERROR").contains(record.get("level").asText())) return; + } catch (IOException invalidRecord) { /* Reject before touching any archive. */ } + } + throw new IOException("incompatible_log_format"); + } + + public void writeSql(ExecutionMetadata metadata) { + writeSql(metadata, false); + } + + public void writeSql(ExecutionMetadata metadata, boolean sensitive) { + try { + var safe = LogPrivacy.sql(metadata, sensitive && LogPrivacy.plaintextEnabled()); + String outcome = Objects.requireNonNullElse(safe.getExecutionOutcome(), "unknown"); + String level = switch (outcome) { case "failure", "error", "cancelled" -> "ERROR"; case "retrying" -> "WARN"; default -> "INFO"; }; + Map fields = new LinkedHashMap<>(); + fields.put("category", "SQL"); fields.put("outcome", outcome); fields.put("batch_outcome", safe.getBatchOutcome()); + fields.put("sql", safe.getDebugQuery()); fields.put("comment", safe.getComment()); fields.put("purpose", safe.getPurpose()); + fields.put("audit_reason", safe.getAuditReason()); fields.put("trace_chain", safe.getRequestTraceChain()); + fields.put("trace_path", safe.getTracePath()); fields.put("mutation_lineage", safe.getMutationLineage()); + fields.put("request_tree", safe.getRequestTree()); fields.put("elapsed_us", safe.getElapsedUs()); + fields.put("result_count", safe.getResultCount()); fields.put("affected_rows", safe.getAffectedRows()); + if (safe.getSqlExecutionCorrelation() != null) fields.put("trace_id", safe.getSqlExecutionCorrelation().traceId()); + write(level, "sql." + outcome, "TeaQL SQL execution", fields); + } catch (Exception unavailable) { failure(logName, "log_projection_failed"); } + } + + public void writeAudit(RawAuditEvent event) { + try { + var safe = LogPrivacy.audit(event, false); + Map fields = new LinkedHashMap<>(); + fields.put("category", "AUDIT"); fields.put("outcome", "success"); fields.put("entity", safe.entityType()); + fields.put("entity_id", safe.entityId()); fields.put("fields", safe.changes()); fields.put("trace_chain", safe.traceChain()); + fields.put("audit_reason", safe.reason()); + write("INFO", "audit.event", "TeaQL audit event", fields); + } catch (Exception unavailable) { failure(logName, "log_projection_failed"); } + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLoggingTelemetry.java b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLoggingTelemetry.java new file mode 100644 index 000000000..c9da753e9 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/RuntimeLoggingTelemetry.java @@ -0,0 +1,53 @@ +package io.teaql.runtime; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** Local logging preserves the independently installed adapter and its capabilities. */ +final class RuntimeLoggingTelemetry implements RuntimeTelemetry { + // Framework-created policy errors retain their exact existing class and + // message. A weak identity hint supplies local severity without retaining + // payloads or changing the error seen by application/SDK consumers. + private static final Map POLICY_REJECTIONS = + java.util.Collections.synchronizedMap(new java.util.WeakHashMap<>()); + static io.teaql.core.TeaQLRuntimeException policyRejection(String message) { + var error = new io.teaql.core.TeaQLRuntimeException(message); + POLICY_REJECTIONS.put(error, Boolean.TRUE); + return error; + } + private final RuntimeTelemetry delegate; + private final RuntimeLogger logger; + RuntimeLoggingTelemetry(RuntimeTelemetry delegate, RuntimeLogger logger) { this.delegate = delegate; this.logger = logger; } + @Override public Scope start(Operation operation) { + var original = RuntimeTelemetry.startSafely(delegate, operation); + return new Scope() { + private boolean ended; + private boolean log(Throwable error) { + if (ended) return false; ended = true; + String category = error == null ? null : RuntimeTelemetry.errorCategory(error); + if (error instanceof io.teaql.core.checker.CheckException || error instanceof io.teaql.core.RequestIntentException) category = "validation"; + if (error != null && error.getClass() == io.teaql.core.TeaQLRuntimeException.class && POLICY_REJECTIONS.containsKey(error)) category = "authorization"; + String level = error == null ? "INFO" : java.util.Set.of("validation", "authorization", "conflict").contains(category) ? "WARN" : "ERROR"; + String outcome = error == null ? "success" : error instanceof java.util.concurrent.CancellationException ? "cancelled" : "failure"; + Map fields = new LinkedHashMap<>(); + fields.put("category", operation.family().toUpperCase(java.util.Locale.ROOT)); fields.put("outcome", outcome); + if (category != null) fields.put("error_code", category); + var correlation = original.correlation(); if (correlation != null) fields.put("trace_id", correlation.traceId()); + logger.write(level, "operation." + outcome, "TeaQL operation " + outcome, fields); + return true; + } + @Override public synchronized void success(Map attributes) { if (log(null)) original.success(attributes); } + @Override public synchronized void failure(Throwable error) { if (log(error)) original.failure(error); } + @Override public synchronized void failureWithAttributes(Throwable error, Map attributes) { if (log(error)) original.failureWithAttributes(error, attributes); } + @Override public io.teaql.core.SqlExecutionCorrelation correlation() { return original.correlation(); } + @Override public void detach() { original.detach(); } + @Override public PropagationScope activate() { return original.activate(); } + }; + } + @Override public boolean recordsSqlDiagnostics() { return delegate.recordsSqlDiagnostics(); } + @Override public void recordDiagnostic(Diagnostic event) { delegate.recordDiagnostic(event); } + @Override public void inject(Map carrier) { delegate.inject(carrier); } + @Override public PropagationScope extractAndActivate(Map carrier) { return delegate.extractAndActivate(carrier); } + @Override public void flush() { delegate.flush(); } + @Override public void shutdown() { delegate.shutdown(); } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 1cdf5459a..896457609 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -27,6 +27,8 @@ public class TeaQLRuntime implements AutoCloseable { private final MutationPolicyApprovalProvider mutationPolicyApprovalProvider; private final InternalIdGenerationService idGenerationService; private final RuntimeLogSink logSink; + private final RuntimeLogger logger; + private final RuntimeTelemetry operationTelemetry; private final boolean queryExecutionLoggingEnabled; private final boolean mutationExecutionLoggingEnabled; private final RuntimeTelemetry telemetry; @@ -53,12 +55,14 @@ private TeaQLRuntime(Builder builder) { this.mutationPolicyRegistry = builder.mutationPolicyRegistry; this.mutationPolicyApprovalProvider = builder.mutationPolicyApprovalProvider; this.idGenerationService = builder.idGenerationService; - this.logSink = builder.logSink; + this.logger = builder.logger != null ? builder.logger : new RuntimeLogger(builder.loggingOptions); + this.logSink = builder.customLogSink ? builder.logSink : new RuntimeDiagnosticLogSink(logger); this.queryExecutionLoggingEnabled = builder.queryExecutionLoggingEnabled; this.mutationExecutionLoggingEnabled = builder.mutationExecutionLoggingEnabled; this.ownsTelemetry = builder.telemetry == null; this.telemetry = builder.telemetry != null ? builder.telemetry : TelemetryStartup.initialize(System.getenv(), TeaQLRuntime.class.getClassLoader()); + this.operationTelemetry = new RuntimeLoggingTelemetry(telemetry, logger); this.schemaExecutor = builder.schemaExecutor; this.businessIdService = builder.businessIdService; this.businessIdSchemaContributor = builder.businessIdSchemaContributor; @@ -117,6 +121,10 @@ public RuntimeTelemetry getTelemetry() { return telemetry; } + public RuntimeLogger getLogger() { return logger; } + + public RuntimeTelemetry getOperationTelemetry() { return operationTelemetry; } + @Override public synchronized void close() { if (!ownsTelemetry || telemetryClosed) return; @@ -191,7 +199,7 @@ io.teaql.core.SqlStatementObservation startSqlStatement(String backend, DataServ try { if (!telemetry.recordsSqlDiagnostics()) return io.teaql.core.SqlStatementObservation.NOOP; } catch (Throwable ignored) { return io.teaql.core.SqlStatementObservation.NOOP; } String family = operation == null ? "unknown" : operation.name().toLowerCase(java.util.Locale.ROOT); - var scope = RuntimeTelemetry.startSafely(telemetry, new RuntimeTelemetry.Operation("provider", backend + ".statement", + var scope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("provider", backend + ".statement", Map.of("teaql.provider.kind", backend, "teaql.provider.operation", family, "teaql.provider.execution", "statement"))); return new io.teaql.core.SqlStatementObservation() { @@ -206,7 +214,7 @@ io.teaql.core.SqlStatementObservation startSqlStatement(String backend, DataServ @SuppressWarnings("unchecked") public SmartList executeForList(UserContext context, SearchRequest request) { - RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("query", request.getTypeName() + ".list", Map.of("teaql.entity.type", request.getTypeName()))); try { @@ -269,7 +277,7 @@ private static QueryOperation requestedListCount(SearchRequest request) { /** Executes a business-facing streaming query after the same policy gate as list queries. */ public Stream executeForStream( UserContext context, SearchRequest request) { - RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("query", request.getTypeName() + ".stream", Map.of("teaql.entity.type", request.getTypeName()))); try { @@ -334,7 +342,7 @@ public SmartList internalExecuteForList( "teaql.internal.top_n.selected_plan", "teaql.relation.selected_plan"); copyRelationPlanAttribute(request, relationAttributes, "teaql.internal.top_n.probe_count", "teaql.relation.probe_count"); - RuntimeTelemetry.Scope relationScope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope relationScope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("relation_load", request.getTypeName() + ".relation", relationAttributes)); try { @@ -403,7 +411,7 @@ private SmartList executeForListResolved( throw new TeaQLRuntimeException("No QueryExecutor registered for route: " + route); } QueryRequest queryRequest = new DefaultQueryRequest(request, intent); - RuntimeTelemetry.Scope providerScope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope providerScope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("provider", route + ".query", Map.of( "teaql.provider.kind", route, "teaql.provider.operation", "query"))); @@ -483,7 +491,7 @@ public void saveGraph(UserContext context, Object items) { } public void saveGraph(UserContext context, Entity entity) { - RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("mutation", entity.typeName() + ".save", Map.of( "teaql.entity.type", entity.typeName(), "teaql.mutation.kind", "save"))); @@ -992,7 +1000,7 @@ private MutationGovernanceSnapshot reviewMutationPlan( MutationDecision decision = Objects.requireNonNull( policy.review(context, plan), "MutationPolicy.review()"); if (!decision.allowed()) { - throw new TeaQLRuntimeException( + throw RuntimeLoggingTelemetry.policyRejection( "[MUTATION POLICY DENIED] " + decision.code() + ": " + (decision.message() == null ? "mutation rejected" : decision.message())); } @@ -1431,7 +1439,7 @@ private MutationResult mutateWithTelemetry( String entityType, String operation) { String provider = executor.getClass().getSimpleName(); - RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("provider", provider + ".mutation", Map.of( "teaql.provider.kind", provider, "teaql.provider.operation", operation, @@ -1458,7 +1466,7 @@ private List mutateBatchWithTelemetry( } var request = new io.teaql.core.MutationBatchRequest(intent, requests); String provider = executor.getClass().getSimpleName(); - RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("provider", provider + ".mutation.batch", Map.of( "teaql.provider.kind", provider, "teaql.provider.operation", operation, @@ -1533,7 +1541,7 @@ private Runnable prepareAuditEvent( } private void deliverAuditEvent(UserContext context, RawAuditEvent rawEvent, SqlIntentRedactions redactions) { - RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(telemetry, + RuntimeTelemetry.Scope telemetryScope = RuntimeTelemetry.startSafely(operationTelemetry, new RuntimeTelemetry.Operation("audit", rawEvent.entityType() + ".audit", Map.of( "teaql.entity.type", rawEvent.entityType(), "teaql.mutation.kind", rawEvent.kind().name().toLowerCase(Locale.ROOT), @@ -1621,6 +1629,9 @@ public static class Builder { MutationPolicyApprovalProvider.none(); private InternalIdGenerationService idGenerationService; private RuntimeLogSink logSink = new DefaultTextRuntimeLogSink(); + private boolean customLogSink; + private RuntimeLogger logger; + private RuntimeLogger.Options loggingOptions = new RuntimeLogger.Options(); // Operation logging is default-on, while the default sink deliberately // excludes bind values and rendered Debug SQL. private boolean queryExecutionLoggingEnabled = true; @@ -1715,6 +1726,35 @@ public Builder roundTripReferenceProvider(RoundTripReferenceProvider provider) { public Builder logSink(RuntimeLogSink logSink) { this.logSink = logSink; + this.customLogSink = true; + return this; + } + + public Builder runtimeLogger(RuntimeLogger logger) { + this.logger = Objects.requireNonNull(logger, "logger"); + return this; + } + + public Builder loggingOptions(RuntimeLogger.Options options) { + this.loggingOptions = Objects.requireNonNull(options, "options"); + return this; + } + + public Builder logName(String name) { + this.loggingOptions = new RuntimeLogger.Options(Objects.requireNonNull(name, "name"), + loggingOptions.modelNames(), loggingOptions.environment(), loggingOptions.workingDirectory(), loggingOptions.logFormat()); + return this; + } + + public Builder logFormat(String format) { + this.loggingOptions = new RuntimeLogger.Options(loggingOptions.logName(), loggingOptions.modelNames(), + loggingOptions.environment(), loggingOptions.workingDirectory(), Objects.requireNonNull(format, "format")); + return this; + } + + public Builder knownModelNames(List names) { + this.loggingOptions = new RuntimeLogger.Options(loggingOptions.logName(), names, + loggingOptions.environment(), loggingOptions.workingDirectory(), loggingOptions.logFormat()); return this; } @@ -1749,6 +1789,7 @@ public Builder diagnosticSqlLogging(boolean enabled) { this.logSink = enabled ? new SensitiveDiagnosticTextRuntimeLogSink() : new DefaultTextRuntimeLogSink(); + this.customLogSink = enabled; return this; } diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingPresentationsTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingPresentationsTest.java new file mode 100644 index 000000000..ae558e1de --- /dev/null +++ b/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingPresentationsTest.java @@ -0,0 +1,206 @@ +package io.teaql.runtime; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.teaql.core.DataServiceOperation; +import io.teaql.core.ExecutionMetadata; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.*; +import java.util.regex.Pattern; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import static org.junit.Assert.*; + +@RunWith(Parameterized.class) +public class RuntimeLoggingPresentationsTest { + @Parameterized.Parameters(name = "{0}") public static Object[] formats() { return new Object[]{"json", "human"}; } + @Rule public TemporaryFolder temporary = new TemporaryFolder(); + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Pattern HEADER = Pattern.compile("^(\\S+) \\[TeaQL\\]\\[(.*)\\]\\[(INFO|WARN|ERROR)\\] (.*) runtime=(\\S+)$"); + private final String format; + public RuntimeLoggingPresentationsTest(String format) { this.format = format; } + private RuntimeLogger logger(Path base, Map environment) { + return new RuntimeLogger(new RuntimeLogger.Options(null, List.of(), environment, base, format)); + } + private static String scalar(String value) throws Exception { return JSON.readValue("\"" + value.replace("\"", "\\\"") + "\"", String.class); } + private static List parse(String text, String format) throws Exception { + assertTrue("complete rendered events", text.endsWith("\n")); + List rows = new ArrayList<>(); List sql = null; + for (String line : text.substring(0, text.length() - 1).split("\n", -1)) { + if (format.equals("json")) { rows.add(JSON.readTree(line)); continue; } + assertFalse(line.chars().anyMatch(c -> Character.isISOControl(c) || c == 0x2028 || c == 0x2029)); + var header = HEADER.matcher(line); + if (header.matches()) { + assertTrue(header.group(1).endsWith("Z")); Instant.parse(header.group(1)); + ObjectNode row = JSON.createObjectNode().put("timestamp", header.group(1)).put("log_name", scalar(header.group(2))) + .put("level", header.group(3)).put("event", scalar(header.group(4))).put("runtime", header.group(5)); + rows.add(row); sql = null; + } else { + assertTrue("unindented/forged event continuation", !rows.isEmpty() && line.startsWith(" ")); + ObjectNode row = (ObjectNode) rows.get(rows.size() - 1); + if (!row.has("message")) row.put("message", scalar(line.substring(2))); + else if (line.equals(" sql:")) { sql = new ArrayList<>(); row.put("sql", ""); } + else if (line.startsWith(" ") && sql != null) { sql.add(scalar(line.substring(4))); row.put("sql", String.join("\n", sql)); } + else { String[] detail = line.substring(2).split("=", 2); assertEquals(2, detail.length); + row.set(scalar(detail[0]), JSON.readTree(detail[1])); sql = null; } + } + } + for (JsonNode row : rows) for (String key : List.of("timestamp", "level", "log_name", "runtime", "event", "message")) assertTrue(row.get(key).isTextual()); + return rows; + } + private List read(Path... paths) throws Exception { + List rows = new ArrayList<>(); for (Path path : paths) rows.addAll(parse(Files.readString(path), format)); return rows; + } + private Path base() { return temporary.getRoot().toPath(); } + + @Test public void selectionDefaultNamesOverrideTwoStartsAndFrozenContexts() throws Exception { + var defaultLog = new RuntimeLogger(new RuntimeLogger.Options(null, List.of(), Map.of(), temporary.newFolder("default").toPath())); + assertEquals("json", defaultLog.logFormat()); + int index = 0; + for (Object[] entry : List.of(new Object[]{"application","domain",List.of("model"),"application"},new Object[]{null,"domain",List.of("model"),"domain"}, + new Object[]{null,"",List.of("model"),"model"},new Object[]{null,"",List.of(),"teaql"},new Object[]{null,"",List.of("one","two"),"teaql"})) { + Path directory = temporary.newFolder("app" + index++).toPath(); + Map env = new HashMap<>(Map.of("TEAQL_DOMAIN",(String)entry[1],"TEAQL_LOG_FORMAT",format)); + @SuppressWarnings("unchecked") var models = (List)entry[2]; + var log = new RuntimeLogger(new RuntimeLogger.Options((String)entry[0],models,env,directory)); + assertEquals(format,log.logFormat());assertEquals(entry[3],log.logName());assertEquals(directory.resolve("logs/"+entry[3]+".log"),log.path()); + env.put("TEAQL_LOG_FORMAT","invalid-unused");env.put("TEAQL_DOMAIN","changed"); + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + var context = new DefaultUserContext(runtime);assertSame(log,RuntimeLogger.forContext(context)); + RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(),new RuntimeTelemetry.Operation("query","probe",Map.of())).success(); + new RuntimeLogger(new RuntimeLogger.Options(log.logName(),List.of(),Map.of("TEAQL_LOG_FORMAT","invalid-unused"),directory,format)); + assertEquals(3,read(log.path()).size()); + var override = logger(directory,Map.of("TEAQL_LOG_ENDPOINT","override/custom.log","TEAQL_LOG_FORMAT","invalid-unused")); + assertEquals(directory.resolve("override/custom.log"),override.path());assertEquals(format,override.logFormat()); + } + var built = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()) + .loggingOptions(new RuntimeLogger.Options(null,List.of(),Map.of(),temporary.newFolder("builder").toPath())) + .logFormat(format).logName("built").knownModelNames(List.of("model")).build(); + assertEquals(format,built.getLogger().logFormat()); + } + + @Test public void invalidValuesFailWithoutEcho() throws Exception { + var saved=System.err;var bytes=new ByteArrayOutputStream(); + try (var output=new PrintStream(bytes,true,StandardCharsets.UTF_8)) { + System.setErr(output); + for(String value:List.of("","debug","JSON","human\nFORMAT-CANARY")) for(boolean app:List.of(false,true)) { + try { new RuntimeLogger(new RuntimeLogger.Options(null,List.of(),app?Map.of():Map.of("TEAQL_LOG_FORMAT",value),base(),app?value:null));fail("invalid format accepted"); } + catch(IllegalStateException expected) {assertNull(expected.getCause());} + } + } finally {System.setErr(saved);} + String text=bytes.toString(StandardCharsets.UTF_8);assertFalse(text.contains("CANARY")); + for(JsonNode row:parse(text,"json")) assertEquals("ERROR",row.get("level").asText()); + assertEquals(0,Objects.requireNonNull(temporary.getRoot().list()).length); + } + + @Test public void streamsKeepTypedEventUnknownKindsAndEscapedSql() throws Exception { + for(String endpoint:List.of("stdout","stderr")) { + var saved=endpoint.equals("stdout")?System.out:System.err;var bytes=new ByteArrayOutputStream(); + var fields=new LinkedHashMap();fields.put("category","SQL");fields.put("trace_id","0123456789abcdef0123456789abcdef"); + fields.put("outcome","retrying");fields.put("error_code","connection_timeout");fields.put("entity_id",9007199254740993L); + fields.put("enabled",true);fields.put("result_count",0);fields.put("omission_reason",null);fields.put("changes",List.of(Map.of("field","password","value","***"))); + fields.put("sql","SELECT id\nFROM customer\n\033\r\u0085\u2028");String message="Readable \\u001b\nFORGED\033\r\u0085\u2029"; + try(var output=new PrintStream(bytes,true,StandardCharsets.UTF_8)) {if(endpoint.equals("stdout"))System.setOut(output);else System.setErr(output); + var log=logger(base(),Map.of("TEAQL_LOG_ENDPOINT",endpoint));assertTrue(log.write("WARN","unknown.kind",message,fields)); + } finally {if(endpoint.equals("stdout"))System.setOut(saved);else System.setErr(saved);} + var rows=parse(bytes.toString(StandardCharsets.UTF_8),format);assertEquals(2,rows.size());var got=(ObjectNode)rows.get(1);got.remove("timestamp"); + fields.putAll(Map.of("level","WARN","runtime","java","log_name","teaql","event","unknown.kind","message",message)); + assertEquals(JSON.valueToTree(fields),got); + } + } + + private static void privacyEvents(RuntimeLogger log) { + var metadata=new ExecutionMetadata();metadata.setOperation(DataServiceOperation.QUERY);metadata.setExecutionOutcome("retrying"); + metadata.setParameterizedQuery("SELECT id FROM customer WHERE password = ?");metadata.setParameters(List.of("PASSWORD-CANARY")); + metadata.setDebugQuery("PASSWORD-CANARY");metadata.setComment("read PASSWORD-CANARY"); + metadata.setSqlExecutionCorrelation(new io.teaql.core.SqlExecutionCorrelation("0123456789abcdef0123456789abcdef","0123456789abcdef")); + log.writeSql(metadata);log.writeSql(metadata,true);assertEquals(List.of("PASSWORD-CANARY"),metadata.getParameters()); + var event=new RawAuditEvent(MutationAuditKind.UPDATED,"Customer",9007199254740993L,List.of(new AuditFieldChange("password","OLD-CANARY","PASSWORD-CANARY")),List.of(),null,null,null,null,null); + log.writeAudit(event);assertEquals("PASSWORD-CANARY",event.changes().get(0).newValue()); + } + @Test public void nativeLevelsProjectionIdentityAndExactAck() throws Exception { + var log=logger(base(),Map.of());var runtime=TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + for(Throwable error:Arrays.asList(null,new io.teaql.core.checker.CheckException("CHECK-CANARY"),new java.io.IOException("DRIVER-CANARY"))) { + var scope=RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(),new RuntimeTelemetry.Operation("query","probe",Map.of()));if(error==null)scope.success();else scope.failure(error); + } + assertEquals(List.of("INFO","INFO","WARN","ERROR"),read(log.path()).stream().map(r->r.get("level").asText()).toList()); + assertFalse(Files.readString(log.path()).contains("CANARY")); + for(String ack:List.of("","true",LogPrivacy.ACKNOWLEDGEMENT)) { + Path directory=temporary.newFolder().toPath();var command=writer(directory,"privacy",format,"privacy");command.environment().put(LogPrivacy.ENVIRONMENT,ack); + Process child=command.start();try {assertTrue(child.waitFor(60,TimeUnit.SECONDS));assertEquals(0,child.exitValue());}finally {if(child.isAlive())child.destroyForcibly();child.waitFor();} + var rows=read(directory.resolve("logs/teaql.log"));assertEquals(4,rows.size());assertEquals("WARN",rows.get(1).get("level").asText()); + assertEquals("0123456789abcdef0123456789abcdef",rows.get(1).get("trace_id").asText()); + assertTrue(rows.get(3).get("entity_id").isIntegralNumber());assertEquals(9007199254740993L,rows.get(3).get("entity_id").asLong()); + assertFalse(Files.readString(directory.resolve("logs/teaql.log")).contains("CANARY")); + } + } + + @Test public void positiveRetentionKeepsNumericNewestOrderAndWholeBlocks() throws Exception { + var log=logger(base(),Map.of("TEAQL_LOG_MAX_SIZE","256","TEAQL_LOG_MAX_FILES","2")); + for(int i=0;i<5;i++)assertTrue(log.write("INFO","retention","็•Œ".repeat(200),Map.of("sequence",i,"sql","SELECT\n"+i))); + for(int i=0;i<3;i++){var rows=read(Path.of(log.path()+(i==0?"":"."+i)));assertEquals(1,rows.size());assertEquals(4-i,rows.get(0).get("sequence").asInt());assertEquals("SELECT\n"+(4-i),rows.get(0).get("sql").asText());} + try(var paths=Files.list(log.path().getParent())){assertEquals(3,paths.count());} + assertFalse(Files.exists(Path.of(log.path()+".3"))); + } + + @Test public void utf8WholeBlocksSizeDayRestartOversizeRetentionAndOverride() throws Exception { + for(boolean override:List.of(false,true)) { + Path directory=temporary.newFolder().toPath();var env=new HashMap<>(Map.of("TEAQL_LOG_MAX_SIZE","1K","TEAQL_LOG_MAX_FILES","100"));if(override)env.put("TEAQL_LOG_ENDPOINT","override/custom.log"); + var log=logger(directory,env);for(int index=0;index<40;index++)assertTrue(log.write("INFO","regular","็•Œ".repeat(25),Map.of("index",index,"sql","SELECT id\nFROM customer"))); + Path[] files;try(var paths=Files.list(log.path().getParent())){files=paths.toArray(Path[]::new);}for(Path file:files)assertTrue(Files.size(file)>0 && Files.size(file)<=1024); + var rows=read(files);assertEquals(41,rows.size());assertEquals(40,rows.stream().filter(r->r.has("index")).map(r->r.get("index").asInt()).distinct().count()); + for(var row:rows)if(row.has("index"))assertEquals("SELECT id\nFROM customer",row.get("sql").asText()); + assertTrue(log.write("WARN","oversize","whole",Map.of("sql","็•Œ".repeat(2000)+"\nLAST")));byte[] before=Files.readAllBytes(log.path());assertTrue(before.length>1024);assertEquals(1,read(log.path()).size()); + assertTrue(log.write("INFO","after","next",Map.of()));assertArrayEquals(before,Files.readAllBytes(Path.of(log.path()+".1"))); + for(boolean restart:List.of(false,true)){Files.setLastModifiedTime(log.path(),FileTime.from(Instant.now().minusSeconds(172800)));before=Files.readAllBytes(log.path()); + if(restart)logger(directory,env);else assertTrue(log.write("INFO","day","UTC rotation",Map.of()));assertArrayEquals(before,Files.readAllBytes(Path.of(log.path()+".1")));assertEquals(1,read(log.path()).size());} + env.put("TEAQL_LOG_MAX_FILES","0");var zero=logger(directory,env);assertTrue(zero.write("INFO","last","็•Œ".repeat(2000),Map.of())); + try(var paths=Files.list(log.path().getParent())){assertEquals(1,paths.count());}assertEquals(1,read(log.path()).size()); + } + } + + private void assertBlocks(Path directory,int count,int starts) throws Exception { + List rows;try(var paths=Files.list(directory.resolve("logs"))){rows=read(paths.toArray(Path[]::new));}Set ids=new HashSet<>(); + for(var row:rows)if(row.has("id")){String id=row.get("id").asText();assertTrue(ids.add(id));assertEquals("SELECT\n"+id,row.get("sql").asText());}assertEquals(count,ids.size());assertEquals(count+starts,rows.size()); + } + @Test public void threadsAppendWholeMultilineEvents() throws Exception { + var pool=Executors.newFixedThreadPool(4);List> children=new ArrayList<>(); + try {for(int worker=0;worker<4;worker++){int id=worker;children.add(pool.submit(()->{var log=logger(base(),Map.of("TEAQL_LOG_MAX_SIZE","2K","TEAQL_LOG_MAX_FILES","100")); + for(int i=0;i<30;i++){String key=id+"/"+i;assertTrue(log.write("INFO","parallel","ๅคš็บฟ็จ‹",Map.of("id",key,"sql","SELECT\n"+key)));}}));}for(var child:children)child.get(60,TimeUnit.SECONDS); + }finally{pool.shutdownNow();assertTrue(pool.awaitTermination(5,TimeUnit.SECONDS));}assertBlocks(base(),120,4); + } + public static class Writer { + public static void main(String[] args) {var log=new RuntimeLogger(new RuntimeLogger.Options(null,List.of(),Map.of("TEAQL_LOG_MAX_SIZE","2K","TEAQL_LOG_MAX_FILES","100"),Path.of(args[0]),args[2])); + if(args[3].equals("privacy")){privacyEvents(log);return;}for(int i=0;i<30;i++){String id=args[1]+"/"+i;if(!log.write("INFO","parallel","ๅŽŸ็”Ÿๅนถๅ‘",Map.of("id",id,"sql","SELECT\n"+id)))System.exit(1);}} + } + private static ProcessBuilder writer(Path directory,String worker,String format,String kind) { + String classPath=System.getProperty("java.class.path")+java.io.File.pathSeparator+System.getProperty("jdk.module.path",""); + var command=new ProcessBuilder(Path.of(System.getProperty("java.home"),"bin","java").toString(),"-cp",classPath,Writer.class.getName(),directory.toString(),worker,format,kind).inheritIO();command.environment().clear();return command; + } + @Test public void independentProcessesAndFollowingStartKeepWholeBlocks() throws Exception { + List children=new ArrayList<>();try{for(int worker=0;worker<4;worker++)children.add(writer(base(),Integer.toString(worker),format,"write").start()); + for(Process child:children){assertTrue(child.waitFor(60,TimeUnit.SECONDS));assertEquals(0,child.exitValue());} + var following=writer(base(),"following",format,"write").start();children.add(following);assertTrue(following.waitFor(60,TimeUnit.SECONDS));assertEquals(0,following.exitValue()); + }finally{for(Process child:children){if(child.isAlive())child.destroyForcibly();child.waitFor();}}assertBlocks(base(),150,5); + } + @Test public void mixedFormatRejectedBeforeMutationAndIoFailuresPreserveBusiness() throws Exception { + var log=logger(base(),Map.of());String other=format.equals("json")?"human":"json";Path archive=Path.of(log.path()+".10");Files.writeString(archive,"preserve archive bytes");byte[] before=Files.readAllBytes(log.path()); + try{new RuntimeLogger(new RuntimeLogger.Options(null,List.of(),Map.of("TEAQL_LOG_MAX_SIZE","1","TEAQL_LOG_MAX_FILES","0"),base(),other));fail("mixed startup accepted");}catch(IllegalStateException expected){} + assertArrayEquals(before,Files.readAllBytes(log.path()));assertEquals("preserve archive bytes",Files.readString(archive)); + var replacement=new RuntimeLogger(new RuntimeLogger.Options(null,List.of(),Map.of(),temporary.newFolder("replacement").toPath(),other));before=Files.readAllBytes(replacement.path());Files.write(log.path(),before); + var runtime=TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build();var context=new DefaultUserContext(runtime);context.putToLocalCache("probe",42,60);assertEquals(Integer.valueOf(42),context.getFromLocalCache("probe",Integer.class)); + var original=new java.io.IOException("SECRET-ERROR");RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(),new RuntimeTelemetry.Operation("query","probe",Map.of())).failure(original);assertEquals("SECRET-ERROR",original.getMessage()); + assertFalse(log.write("INFO","later","business completion",Map.of()));assertArrayEquals(before,Files.readAllBytes(log.path()));assertEquals("preserve archive bytes",Files.readString(archive)); + Files.delete(log.path());Files.createDirectory(log.path());assertFalse(log.write("ERROR","later","I/O failure",Map.of()));try{logger(base(),Map.of());fail("startup I/O failure accepted");}catch(IllegalStateException expected){} + } +} diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingTest.java new file mode 100644 index 000000000..555aefebf --- /dev/null +++ b/teaql-runtime/src/test/java/io/teaql/runtime/RuntimeLoggingTest.java @@ -0,0 +1,252 @@ +package io.teaql.runtime; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.DataServiceOperation; +import io.teaql.core.ExecutionMetadata; +import java.nio.file.*; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.*; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import static org.junit.Assert.*; + +public class RuntimeLoggingTest { + @Rule public TemporaryFolder temporary = new TemporaryFolder(); + private final ObjectMapper json = new ObjectMapper(); + private RuntimeLogger logger(Map env) { + return new RuntimeLogger(new RuntimeLogger.Options(null, List.of(), env, temporary.getRoot().toPath())); + } + private List records(Path... files) throws Exception { + List result = new ArrayList<>(); + for (Path file : files) { + String text = Files.readString(file); + assertTrue("Whole JSON lines", text.endsWith("\n")); + for (String line : text.stripTrailing().split("\n")) { + JsonNode record = json.readTree(line); + for (String key : List.of("timestamp", "level", "log_name", "runtime", "event", "message")) assertFalse(record.get(key).asText().isEmpty()); + assertEquals("java", record.get("runtime").asText()); + Instant.parse(record.get("timestamp").asText()); assertTrue(record.get("timestamp").asText().endsWith("Z")); + result.add(record); + } + } + return result; + } + + @Test public void defaultFileBeforeFirstQueryAndAppendAcrossTwoStarts() throws Exception { + var first = logger(Map.of()); + assertEquals(temporary.getRoot().toPath().resolve("logs/teaql.log"), first.path()); + assertEquals(50L * 1024 * 1024, first.maxSize()); assertEquals(7, first.maxFiles()); + assertEquals("runtime.start", records(first.path()).get(0).get("event").asText()); + var second = logger(Map.of()); assertTrue(second.write("INFO", "probe", "Completed", Map.of())); + assertEquals(3, records(first.path()).size()); + } + + @Test public void namePrecedenceUnicodeOverrideAndMultipleModels() throws Exception { + List cases = List.of(new Object[]{"application", "domain", List.of("model"), "application"}, + new Object[]{null, "domain", List.of("model"), "domain"}, new Object[]{null, "", List.of("model"), "model"}, + new Object[]{null, "", List.of("one", "two"), "teaql"}, new Object[]{"่ฎขๅ•", "", List.of(), "่ฎขๅ•"}); + int index = 0; + for (Object[] test : cases) { + Path base = temporary.newFolder("application" + index++).toPath(); + @SuppressWarnings("unchecked") List models = (List) test[2]; + var log = new RuntimeLogger(new RuntimeLogger.Options((String) test[0], models, + Map.of("TEAQL_DOMAIN", (String) test[1], "TEAQL_LOG_ENDPOINT", "other/output.jsonl"), base)); + assertEquals(test[3], log.logName()); assertEquals(base.resolve("other/output.jsonl"), log.path()); + assertEquals(test[3], records(log.path()).get(0).get("log_name").asText()); + } + } + + @Test public void invalidNamesAndConfigurationFailStartupWithoutEchoingInput() throws Exception { + for (String name : List.of("", ".", "..", "../secret", "a/b", "a\\b", "bad\nname", "NUL", "COM1.txt", "trailing.", "trailing ")) { + try { new RuntimeLogger(new RuntimeLogger.Options(name, List.of(), Map.of(), temporary.getRoot().toPath())); fail("Accepted invalid name"); } + catch (IllegalStateException expected) { assertEquals("Runtime logging initialization failed", expected.getMessage()); assertNull(expected.getCause()); } + } + for (String size : List.of("0", "-1", "1.2M", "1TB", "999999999999999999999G")) { + try { logger(Map.of("TEAQL_LOG_MAX_SIZE", size)); fail("Accepted invalid size"); } catch (IllegalStateException expected) { } + } + for (String files : List.of("-1", "invalid", "1.5", "2147483648")) { + try { logger(Map.of("TEAQL_LOG_MAX_FILES", files)); fail("Accepted invalid retention"); } catch (IllegalStateException expected) { } + } + assertEquals(1, RuntimeLogger.parseSize("1")); assertEquals(2048, RuntimeLogger.parseSize("2K")); assertEquals(3072, RuntimeLogger.parseSize("3KB")); + assertEquals(2097152, RuntimeLogger.parseSize("2m")); assertEquals(1048576, RuntimeLogger.parseSize("1MB")); + assertEquals(1073741824, RuntimeLogger.parseSize("1G")); assertEquals(1073741824, RuntimeLogger.parseSize("1GB")); + } + + @Test public void utf8OversizedRecordsArchiveOrderAndZeroRetention() throws Exception { + var log = logger(Map.of("TEAQL_LOG_MAX_SIZE", "256", "TEAQL_LOG_MAX_FILES", "2")); + for (int i = 0; i < 5; i++) assertTrue(log.write("INFO", "probe", "็•Œ".repeat(200), Map.of("sequence", i))); + for (int i = 0; i < 3; i++) { + var record = records(Path.of(log.path() + (i == 0 ? "" : "." + i))); + assertEquals(1, record.size()); assertEquals(4 - i, record.get(0).get("sequence").asInt()); assertEquals("็•Œ".repeat(200), record.get(0).get("message").asText()); + } + assertFalse(Files.exists(Path.of(log.path() + ".3"))); + var zero = logger(Map.of("TEAQL_LOG_MAX_SIZE", "256", "TEAQL_LOG_MAX_FILES", "0")); + zero.write("INFO", "probe", "x".repeat(400), Map.of()); + try (var files = Files.list(zero.path().getParent())) { assertEquals(1, files.count()); } + assertEquals(1, records(zero.path()).size()); + } + + @Test public void utcDayRotationAndRestart() throws Exception { + for (boolean restart : List.of(false, true)) { + var base = temporary.newFolder("day-" + restart).toPath(); + var options = new RuntimeLogger.Options(null, List.of(), Map.of(), base); + var log = new RuntimeLogger(options); byte[] before = Files.readAllBytes(log.path()); + Files.setLastModifiedTime(log.path(), FileTime.from(Instant.now().minusSeconds(86400))); + if (restart) log = new RuntimeLogger(options); else assertTrue(log.write("INFO", "next.day", "Next day", Map.of())); + assertArrayEquals(before, Files.readAllBytes(Path.of(log.path() + ".1"))); assertEquals(1, records(log.path()).size()); + } + } + + @Test public void ordinaryUtf8RecordsRespectWholeLineByteLimit() throws Exception { + var log = logger(Map.of("TEAQL_LOG_MAX_SIZE", "1K", "TEAQL_LOG_MAX_FILES", "1000")); + for (int i = 0; i < 40; i++) assertTrue(log.write("INFO", "regular", "็•Œ".repeat(30), Map.of("index", i))); + Path[] files; + try (var paths = Files.list(log.path().getParent())) { files = paths.toArray(Path[]::new); } + assertTrue(files.length > 1); + for (Path file : files) assertTrue(Files.size(file) > 0 && Files.size(file) <= 1024); + var all = records(files); assertEquals(41, all.size()); + var ids = new HashSet(); + for (var row : all) if (row.has("index")) assertTrue(ids.add(row.get("index").asInt())); + assertEquals(40, ids.size()); + } + + @Test public void rotationFailurePreservesForeignArchiveAndBusinessOutcome() throws Exception { + var log = logger(Map.of()); byte[] before = Files.readAllBytes(log.path()); + Path archive = Path.of(log.path() + ".1"); Files.createDirectory(archive); + Files.writeString(archive.resolve("marker"), "preserve"); + Files.setLastModifiedTime(log.path(), FileTime.from(Instant.now().minusSeconds(86400))); + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + var context = new DefaultUserContext(runtime); context.putToLocalCache("probe", 42, 60); + assertEquals(Integer.valueOf(42), context.getFromLocalCache("probe", Integer.class)); + var original = new java.io.IOException("private-original-error"); + RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), new RuntimeTelemetry.Operation("query", "probe", Map.of())).failure(original); + assertEquals("private-original-error", original.getMessage()); + assertFalse(log.write("INFO", "probe", "Completed", Map.of())); + assertArrayEquals(before, Files.readAllBytes(log.path())); + assertEquals("preserve", Files.readString(archive.resolve("marker"))); + } + + @Test public void nativeCheckAndIntentRejectionsAreWarnings() throws Exception { + var log = logger(Map.of()); + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + var context = new DefaultUserContext(runtime); + RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), new RuntimeTelemetry.Operation("mutation", "probe", Map.of())) + .failure(new io.teaql.core.checker.CheckException("private-validation-value")); + var scope = RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), new RuntimeTelemetry.Operation("query", "probe", Map.of())); + try { io.teaql.core.QueryIntent.of("read probe", ""); fail("Intent must be rejected"); } + catch (io.teaql.core.RequestIntentException expected) { scope.failure(expected); } + var all = records(log.path()); assertEquals(3, all.size()); + for (int i = 1; i < all.size(); i++) { assertEquals("WARN", all.get(i).get("level").asText()); assertEquals("validation", all.get(i).get("error_code").asText()); } + assertFalse(Files.readString(log.path()).contains("private-validation-value")); + } + + @Test public void sqlFileKeepsAvailableTraceCorrelation() throws Exception { + var log = logger(Map.of()); + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + var metadata = new ExecutionMetadata(); metadata.setOperation(DataServiceOperation.QUERY); metadata.setExecutionOutcome("success"); + metadata.setSqlExecutionCorrelation(new io.teaql.core.SqlExecutionCorrelation("0123456789abcdef0123456789abcdef", "0123456789abcdef")); + runtime.recordExecutionMetadata(new DefaultUserContext(runtime), metadata); + var all = records(log.path()); assertEquals(2, all.size()); + assertEquals("0123456789abcdef0123456789abcdef", all.get(1).get("trace_id").asText()); + } + + @Test public void nativePolicyRejectionKeepsBusinessAndSdkErrorIdentity() throws Exception { + var log = logger(Map.of()); + var captured = new java.util.concurrent.atomic.AtomicReference(); + RuntimeTelemetry adapter = operation -> new RuntimeTelemetry.Scope() { + @Override public void success(Map attributes) { } + @Override public void failure(Throwable error) { captured.set(error); } + }; + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).telemetry(adapter).build(); + var rejection = RuntimeLoggingTelemetry.policyRejection("private-policy-value"); + assertEquals(io.teaql.core.TeaQLRuntimeException.class, rejection.getClass()); + var operation = new RuntimeTelemetry.Operation("mutation", "probe", Map.of()); + RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), operation).failure(rejection); + assertSame(rejection, captured.get()); assertEquals("private-policy-value", rejection.getMessage()); + var unrelated = new io.teaql.core.TeaQLRuntimeException("[MUTATION POLICY DENIED] private-unrelated-value"); + RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), operation).failure(unrelated); + assertSame(unrelated, captured.get()); assertSame(adapter, runtime.getTelemetry()); + var all = records(log.path()); assertEquals(3, all.size()); + assertEquals("WARN", all.get(1).get("level").asText()); assertEquals("ERROR", all.get(2).get("level").asText()); + assertFalse(Files.readString(log.path()).contains("private-")); + } + + @Test public void startupFailureAndPostStartupPreserveBusinessResultAndError() throws Exception { + Path active = temporary.getRoot().toPath().resolve("logs/teaql.log"); Files.createDirectories(active); + try { logger(Map.of()); fail("Startup must fail"); } catch (IllegalStateException expected) { } + Files.delete(active); var log = logger(Map.of()); + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).build(); + var context = new DefaultUserContext(runtime); context.putToLocalCache("private-cache-key", 42, 60); + Files.delete(active); Files.createDirectory(active); + assertEquals(Integer.valueOf(42), context.getFromLocalCache("private-cache-key", Integer.class)); + var original = new IllegalStateException("private-original-message"); + var scope = RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), new RuntimeTelemetry.Operation("query", "probe", Map.of())); + scope.failure(original); assertEquals("private-original-message", original.getMessage()); + assertFalse(log.write("INFO", "probe", "Completed", Map.of())); + } + + private static class ValidationException extends RuntimeException { ValidationException(String text) { super(text); } } + @Test public void nativeOutcomeSeveritySafeSqlAndExtensionIdentity() throws Exception { + var log = logger(Map.of()); RuntimeTelemetry adapter = RuntimeTelemetry.NOOP; + var runtime = TeaQLRuntime.builder().metadata(new TeaQLRuntimeTest.DummyMetaFactory()).runtimeLogger(log).telemetry(adapter).build(); + assertSame(adapter, runtime.getTelemetry()); + for (Throwable error : Arrays.asList(null, new ValidationException("private-error-message"), new java.io.IOException("private-error-message"))) { + var scope = RuntimeTelemetry.startSafely(runtime.getOperationTelemetry(), new RuntimeTelemetry.Operation("query", "probe", Map.of())); + if (error == null) scope.success(); else scope.failure(error); scope.success(); + } + var metadata = new ExecutionMetadata(); metadata.setOperation(DataServiceOperation.QUERY); metadata.setDebugQuery("unprojected-sql-secret"); metadata.setExecutionOutcome("retrying"); + runtime.recordExecutionMetadata(new DefaultUserContext(runtime), metadata); + var all = records(log.path()); assertEquals(5, all.size()); + assertEquals("INFO", all.get(1).get("level").asText()); assertEquals("WARN", all.get(2).get("level").asText()); + assertEquals("ERROR", all.get(3).get("level").asText()); assertEquals("WARN", all.get(4).get("level").asText()); assertEquals("SQL", all.get(4).get("category").asText()); + String text = Files.readString(log.path()); assertFalse(text.contains("private-error-message")); assertFalse(text.contains("unprojected-sql-secret")); + } + + private void assertConcurrent(int events, int starts) throws Exception { + List all; + try (var paths = Files.list(temporary.getRoot().toPath().resolve("logs"))) { all = records(paths.toArray(Path[]::new)); } + Set ids = new HashSet<>(); + for (JsonNode record : all) if (record.has("id")) assertTrue(ids.add(record.get("id").asText())); + assertEquals(events, ids.size()); assertEquals(events + starts, all.size()); + } + + @Test public void concurrentStartupRotationAndReopening() throws Exception { + var pool = Executors.newFixedThreadPool(8); + try { + List> children = new ArrayList<>(); + for (int worker = 0; worker < 8; worker++) { int id = worker; + children.add(pool.submit(() -> { var log = logger(Map.of("TEAQL_LOG_MAX_SIZE", "1K", "TEAQL_LOG_MAX_FILES", "1000")); + for (int i = 0; i < 30; i++) assertTrue(log.write("INFO", "concurrent", "Completed", Map.of("id", id + "/" + i))); })); + } + for (var child : children) child.get(60, TimeUnit.SECONDS); + } finally { pool.shutdownNow(); assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS)); } + assertConcurrent(240, 8); + } + + public static class Writer { + public static void main(String[] arguments) { + var log = new RuntimeLogger(new RuntimeLogger.Options(null, List.of(), Map.of("TEAQL_LOG_MAX_SIZE", "1K", "TEAQL_LOG_MAX_FILES", "1000"), Path.of(arguments[0]))); + for (int i = 0; i < 30; i++) if (!log.write("INFO", "concurrent", "Completed", Map.of("id", arguments[1] + "/" + i))) System.exit(1); + } + } + @Test public void independentProcessesCoordinateWholeLines() throws Exception { + List children = new ArrayList<>(); + try { + // Surefire keeps production modules off java.class.path in a + // JPMS run. This independent probe loads both sets on its own + // classpath, without inheriting Surefire's patched-module flags. + String classPath = System.getProperty("java.class.path") + java.io.File.pathSeparator + + System.getProperty("jdk.module.path", ""); + for (int worker = 0; worker < 4; worker++) { + var command = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-cp", classPath, Writer.class.getName(), temporary.getRoot().toString(), Integer.toString(worker)).inheritIO(); + command.environment().clear(); children.add(command.start()); + } + for (Process child : children) { assertTrue(child.waitFor(60, TimeUnit.SECONDS)); assertEquals(0, child.exitValue()); } + } finally { for (Process child : children) { if (child.isAlive()) child.destroyForcibly(); child.waitFor(); } } + assertConcurrent(120, 4); + } +} diff --git a/teaql-snowflake/pom.xml b/teaql-snowflake/pom.xml index 867b3029e..5e5199b3e 100644 --- a/teaql-snowflake/pom.xml +++ b/teaql-snowflake/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-snowflake teaql-snowflake diff --git a/teaql-sql-portable/pom.xml b/teaql-sql-portable/pom.xml index 9ea06430f..fb1452f6f 100644 --- a/teaql-sql-portable/pom.xml +++ b/teaql-sql-portable/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-sql-portable diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index b5b9255c8..c75de7f71 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-sqlite teaql-sqlite diff --git a/teaql-tfp-endpoint/pom.xml b/teaql-tfp-endpoint/pom.xml index 94284e74c..76c297c2f 100644 --- a/teaql-tfp-endpoint/pom.xml +++ b/teaql-tfp-endpoint/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-tool-http/pom.xml b/teaql-tool-http/pom.xml index 063b24817..2adaf8bdb 100644 --- a/teaql-tool-http/pom.xml +++ b/teaql-tool-http/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-tool-http diff --git a/teaql-utils-json/pom.xml b/teaql-utils-json/pom.xml index b1c86aaac..4604dc6ce 100644 --- a/teaql-utils-json/pom.xml +++ b/teaql-utils-json/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 teaql-utils-json diff --git a/teaql-utils-reflection/pom.xml b/teaql-utils-reflection/pom.xml index ec3e7f6e2..03eeaf068 100644 --- a/teaql-utils-reflection/pom.xml +++ b/teaql-utils-reflection/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-utils-spring/pom.xml b/teaql-utils-spring/pom.xml index 0b7a030c1..b181e4233 100644 --- a/teaql-utils-spring/pom.xml +++ b/teaql-utils-spring/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/teaql-utils/pom.xml b/teaql-utils/pom.xml index 9382fc945..837a1c43d 100644 --- a/teaql-utils/pom.xml +++ b/teaql-utils/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.557-RELEASE + 6.3.0 ../pom.xml diff --git a/tools/verify_maven_release.sh b/tools/verify_maven_release.sh index 956477ad1..af1bb6dc7 100755 --- a/tools/verify_maven_release.sh +++ b/tools/verify_maven_release.sh @@ -35,8 +35,8 @@ mapfile -t modules < <( sed -n '//,/<\/modules>/s:.*\([^<]*\).*:\1:p' pom.xml \ | grep -v '^examples/' ) -if [[ "${#modules[@]}" -ne 33 ]]; then - echo "Expected 33 child modules, found ${#modules[@]}" >&2 +if [[ "${#modules[@]}" -ne 34 ]]; then + echo "Expected 34 child modules including teaql-opentelemetry, found ${#modules[@]}" >&2 exit 1 fi