Add optional Blob and secure-document modules with governed views

.github/workflows/release.yml

@@ -4,7 +4,6 @@ on:
   push:
     tags:
       - "v*"
-  workflow_dispatch:
 
 concurrency:
   group: teaql-java-maven-release
@@ -39,18 +38,22 @@ jobs:
       - name: Verify tag points at the release commit
         if: github.event_name == 'push'
         run: test "$(git rev-list -n 1 "$GITHUB_REF_NAME")" = "$GITHUB_SHA"
+      - name: Require a verified signed release tag
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python3 tools/verify_signed_release_tag.py
       - name: Test all modules
         run: mvn -B test
+      - name: Check release completeness verifier
+        run: |
+          python3 tools/test_verify_maven_release.py
+          python3 tools/test_verify_maven_release_absence.py
+          python3 tools/test_verify_signed_release_tag.py
       - name: Refuse to overwrite an existing release
         if: github.event_name == 'push'
         env:
           VERSION: ${{ steps.release.outputs.version }}
-        run: |
-          url="https://maven.teaql.io/repository/maven-releases/io/teaql/teaql-core/$VERSION/teaql-core-$VERSION.pom"
-          if curl --fail --silent --show-error --head "$url" >/dev/null 2>&1; then
-            echo "Release already exists and will not be overwritten: $VERSION" >&2
-            exit 1
-          fi
+        run: tools/verify_maven_release_absence.sh "$VERSION"
       - name: Publish all Maven modules
         if: github.event_name == 'push'
         run: mvn -B deploy -DskipTests -pl '!examples/conformance/lib,!examples/conformance'

.gitignore

@@ -37,6 +37,10 @@ build/
 !tools/test_error_catalog.py
 !tools/test_schema_context_contract.py
 !tools/test_dialect_integration_guide.py
+!tools/test_verify_maven_release.py
+!tools/test_verify_maven_release_absence.py
+!tools/verify_signed_release_tag.py
+!tools/test_verify_signed_release_tag.py
 __pycache__/
 
 # Sensitive

examples/blob-view-preparation/README.md

@@ -0,0 +1,27 @@
+# Blob and final-view CLI
+
+Build from the repository root:
+
+```sh
+mvn -pl examples/blob-view-preparation -am package
+```
+
+Run `io.teaql.examples.blobview.BlobViewExample` using the produced example JAR and
+runtime dependencies. Inject TEAQL_BLOB_TEST_PG_URL, TEAQL_BLOB_TEST_PG_USER and
+TEAQL_BLOB_TEST_PG_PASSWORD for a dedicated PostgreSQL test database, plus
+TEAQL_VIEW_PDF pointing to the shared real PDF fixture. Do not put passwords in
+command-line arguments. The example creates/drops only a unique owned schema and
+removes its exact temporary file/parser directory on exit.
+
+The host explicitly provisions catalog/command schemas and starts a file-provider
+gateway bound to a temporary loopback port. No Web framework or DI container is
+used. Policies grant this isolated CLI actor its own scope; replace them with real
+current authorization in an application. The example has no secure-document module.
+
+It performs actual upload/content verification, one atomic native SQL+Blob command,
+final view preparation and HTTP retrieval using the returned complete URL. It
+checks that entities/SQL still contain the Key, nested presentation reuses the same
+PreparedView, and storage unavailability after commit preserves a durable receipt
+without running the command action again. Credentials and signed URLs are not
+printed. The task evidence runner records exact source-built JAR hashes and Java
+commands, with no JUnit/test classes on the standalone classpath.

examples/blob-view-preparation/pom.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent><groupId>io.teaql</groupId><artifactId>teaql-java-parent</artifactId><version>1.555-RELEASE</version><relativePath>../../pom.xml</relativePath></parent>
+    <artifactId>teaql-blob-view-preparation-example</artifactId>
+    <description>Standalone PostgreSQL/File Blob and final view consumer, without a Web framework or document module</description>
+    <dependencies>
+        <dependency><groupId>io.teaql</groupId><artifactId>teaql-blob-reference</artifactId></dependency>
+        <dependency><groupId>io.teaql</groupId><artifactId>teaql-blob-store-file</artifactId></dependency>
+        <dependency><groupId>io.teaql</groupId><artifactId>teaql-blob-content-tika</artifactId></dependency>
+        <dependency><groupId>io.teaql</groupId><artifactId>teaql-provider-jdbc</artifactId></dependency>
+        <dependency><groupId>org.postgresql</groupId><artifactId>postgresql</artifactId><version>42.7.14</version></dependency>
+    </dependencies>
+</project>

examples/blob-view-preparation/src/main/java/io/teaql/examples/blobview/BlobViewExample.java

@@ -0,0 +1,132 @@
+package io.teaql.examples.blobview;
+
+import io.teaql.blob.reference.*;
+import io.teaql.blob.store.file.*;
+import io.teaql.blob.content.tika.TikaBlobContentVerifier;
+import io.teaql.core.*;
+import io.teaql.core.blob.*;
+import io.teaql.core.meta.*;
+import io.teaql.core.sql.*;
+import io.teaql.core.view.*;
+import io.teaql.dataservice.sql.SqlDataServiceExecutor;
+import io.teaql.provider.jdbc.JdbcSqlExecutor;
+import io.teaql.runtime.*;
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+import java.security.SecureRandom;
+import java.sql.*;
+import java.time.*;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+import javax.sql.DataSource;
+
+/** Real library-only CLI. The optional storage provider owns its temporary HTTP listener. */
+public final class BlobViewExample {
+    private static final BlobScope SCOPE=new BlobScope("example","tenant-a");
+    private static final SecureRandom RANDOM=new SecureRandom();
+    public static void main(String[] arguments) {
+        try {
+            Path root=Files.createTempDirectory("teaql-owned-blob-view-");
+            try { run(root); }
+            finally { try (var paths=Files.walk(root)) { for (var path:paths.sorted(Comparator.reverseOrder()).toList()) Files.deleteIfExists(path); } }
+        } catch (Exception failed) {
+            String code=failed instanceof ViewPreparationException view?view.code().name():failed.getClass().getSimpleName();
+            System.err.println("FAIL standalone Blob view: "+code); System.exit(1);
+        }
+    }
+    private static void run(Path root) throws Exception {
+        Clock clock=Clock.systemUTC(); byte[] signingKey=new byte[32]; RANDOM.nextBytes(signingKey);
+        var files=new FileBlobStore(root.resolve("objects"));
+        try (var database=new OwnedDatabase(); var gateway=new FileBlobHttpGateway(files,new InetSocketAddress("127.0.0.1",0),
+                URI.create("http://127.0.0.1:0/blob-read"),"example",Map.of("example",signingKey),clock,Duration.ofMinutes(10))) {
+            var sql=database.sql; var catalog=new PostgresBlobCatalog(sql); var journal=new PostgresBlobCommandJournal(sql);
+            sql.execute("CREATE TABLE asset_data(id BIGINT PRIMARY KEY,version BIGINT NOT NULL,title VARCHAR(100),document VARCHAR(128))");
+            catalog.migrate(); journal.migrate(); files.initialize(); gateway.start();
+            var scopes=(BlobBindingAuthorization.ScopeProvider)context -> context.getAttribute("scope",BlobScope.class);
+            var module=BlobReferenceModule.builder().catalog(catalog).store(gateway).clock(clock)
+                    .verifier(new TikaBlobContentVerifier(root.resolve("verify"),TikaBlobContentVerifier.Limits.defaults()))
+                    .bindingAuthorization(new BlobBindingAuthorization(scopes,(context,target,kind) -> true,(context,target,blob) -> true))
+                    .uploadAuthorization(new BlobUploadAuthorization(scopes,(context,target) -> true,(context,target,action,info) -> true))
+                    .readAuthorization(new BlobReadAuthorization(scopes,(context,target) -> true,(context,target,blob) -> true))
+                    .maintenancePermission((context,scope) -> false)
+                    .commands(journal,new BlobCommandAuthorization(scopes,context -> context.getAttribute("principal",String.class),(context,request) -> true,(context,receipt) -> true)).build();
+            try (var runtime=module.installInto(TeaQLRuntime.builder().metadata(metadata()).dataService("default",new SqlDataServiceExecutor("default",sql))
+                    .viewAuthorization(viewPolicy()).viewClock(clock).telemetry(RuntimeTelemetry.NOOP).executionLogging(false)).build()) {
+                runtime.install(RuntimeModule.of().withRequiredCapabilities("blob.v1","blob.command.v1","view.v1")); runtime.validateReadiness();
+                var actor=new DefaultUserContext(runtime); actor.putAttribute("scope",SCOPE); actor.putAttribute("principal","cli");
+                byte[] bytes=Files.readAllBytes(Path.of(required("TEAQL_VIEW_PDF"))); var upload=actor.capability(BlobRuntime.class).upload();
+                var session=upload.begin(actor,new BlobFieldTarget("Asset",null,"document"),new BlobRuntime.Uploads.FileInfo("sample.pdf",null,null),"example-upload");
+                upload.write(actor,session.uploadId(),new ByteArrayInputStream(bytes)); BlobRef key=upload.complete(actor,session.uploadId());
+                var entity=new Asset(); entity.__internalInitializeNewEntityId(42L); entity.updateProperty("title","example"); entity.updateProperty("document",key); entity.setComment("standalone Blob view example");
+                AtomicInteger actions=new AtomicInteger(); var commands=actor.capability(BlobCommandService.class);
+                var request=new BlobCommandService.Request("Asset.create/v1","example-command",("Asset:42:"+key.key()).getBytes(java.nio.charset.StandardCharsets.UTF_8));
+                commands.execute(actor,request,() -> { actions.incrementAndGet(); runtime.saveGraph(actor,entity); return List.of(new BlobCommandService.EntityRevision("Asset",42,entity.getVersion())); });
+                var options=ViewOptions.afterCommit(); var prepared=runtime.prepareView(actor,Map.of("data",entity),options);
+                var body=(Map<?,?>)((Map<?,?>)prepared.value()).get("data"); URI url=URI.create((String)body.get("document"));
+                var connection=(HttpURLConnection)url.toURL().openConnection(); connection.setConnectTimeout(5000); connection.setReadTimeout(5000);
+                try (var stream=connection.getInputStream()) { require(Arrays.equals(bytes,stream.readAllBytes())); require("application/pdf".equals(connection.getContentType())); }
+                finally { connection.disconnect(); }
+                require(prepared==runtime.prepareView(actor,prepared,options)); require(key.equals(entity.document));
+                require(key.key().equals(database.independent.queryForList("SELECT document FROM asset_data WHERE id=42",new Object[0]).get(0).get("document")));
+                gateway.close();
+                try { runtime.prepareView(actor,entity,ViewOptions.afterCommit()); throw new IllegalStateException("Expected unavailable storage"); }
+                catch (ViewPreparationException failed) { require(failed.businessCommitted()); }
+                require(commands.resolve(actor,request).isPresent()); require(actions.get()==1);
+                require(((Number)database.independent.queryForList("SELECT version FROM asset_data WHERE id=42",new Object[0]).get(0).get("version")).longValue()==1);
+                System.out.println("PASS Blob view: real upload/verification, atomic command, complete HTTP URL, stable SQL Key, presentation reuse and committed-failure recovery");
+            }
+        }
+    }
+    private static EntityMetaFactory metadata() {
+        var metadata=new SimpleEntityMetaFactory(); var descriptor=new SQLEntityDescriptor(); descriptor.setType("Asset"); descriptor.setTargetType(Asset.class); descriptor.setEntitySupplier(Asset::new);
+        ((GenericSQLProperty)descriptor.addSimpleProperty("id",Long.class)).setColumnType("BIGINT");
+        ((GenericSQLProperty)descriptor.addSimpleProperty("version",Long.class)).setColumnType("BIGINT");
+        ((GenericSQLProperty)descriptor.addSimpleProperty("title",String.class)).setColumnType("VARCHAR(100)");
+        var document=(BlobSQLProperty)descriptor.addSimpleProperty("document",BlobRef.class,BlobSQLProperty::new);
+        document.setTableName("asset_data"); document.setColumnType("VARCHAR(128)"); document.withBlobFieldSpec(BlobFieldSpec.of("Asset","document",List.of("pdf"),true,"default")); metadata.register(descriptor); return metadata;
+    }
+    private static ViewAuthorization viewPolicy() {
+        return new ViewAuthorization() {
+            @Override public Identity identity(UserContext context) { var scope=context.getAttribute("scope",BlobScope.class); return new Identity(scope.realm(),context.getAttribute("principal",String.class),scope.boundary()); }
+            @Override public boolean canReadEntity(UserContext context,EntityTarget target,String path) { return true; }
+            @Override public FieldDecision field(UserContext context,EntityTarget target,String field,String path,ViewOptions.Mode mode) { return FieldDecision.show(); }
+        };
+    }
+    private static void require(boolean condition) { if (!condition) throw new IllegalStateException("Standalone view invariant failed"); }
+    private static String required(String name) { String value=System.getenv(name); if (value==null || value.isBlank()) throw new IllegalStateException("Missing CLI configuration: "+name); return value; }
+    public static final class Asset extends BaseEntity {
+        private BlobRef document; private String title;
+        @Override public String typeName() { return "Asset"; }
+        @Override public void __internalSet(String field,Object value) { switch (field) {
+            case "document" -> { document=(BlobRef)value; markPropertyLoaded(field); } case "title" -> { title=(String)value; markPropertyLoaded(field); } default -> super.__internalSet(field,value);
+        } }
+        @Override public Object __internalGet(String field) { return switch (field) { case "document" -> document; case "title" -> title; default -> super.__internalGet(field); }; }
+    }
+    private static final class OwnedDatabase implements AutoCloseable {
+        final String schema="r4_view_cli_"+UUID.randomUUID().toString().replace("-","");
+        final String url=required("TEAQL_BLOB_TEST_PG_URL"),user=required("TEAQL_BLOB_TEST_PG_USER"),password=required("TEAQL_BLOB_TEST_PG_PASSWORD");
+        final JdbcSqlExecutor sql,independent;
+        OwnedDatabase() throws SQLException {
+            try (Connection connection=DriverManager.getConnection(url,user,password); Statement statement=connection.createStatement()) { statement.execute("CREATE SCHEMA "+schema); }
+            var source=new DataSource() {
+                @Override public Connection getConnection() throws SQLException {
+                    Connection connection=DriverManager.getConnection(url,user,password);
+                    try (Statement statement=connection.createStatement()) { statement.execute("SET search_path TO "+schema); statement.execute("SET lock_timeout='5s'"); statement.execute("SET statement_timeout='10s'"); }
+                    catch (SQLException failed) { connection.close(); throw failed; } return connection;
+                }
+                @Override public Connection getConnection(String user,String password) throws SQLException { throw new SQLFeatureNotSupportedException(); }
+                @Override public PrintWriter getLogWriter() { return null; }
+                @Override public void setLogWriter(PrintWriter writer) {}
+                @Override public void setLoginTimeout(int seconds) {}
+                @Override public int getLoginTimeout() { return 0; }
+                @Override public Logger getParentLogger() { return Logger.getLogger("teaql.example.blobview"); }
+                @Override public <T> T unwrap(Class<T> type) throws SQLException { throw new SQLFeatureNotSupportedException(); }
+                @Override public boolean isWrapperFor(Class<?> type) { return false; }
+            };
+            sql=new JdbcSqlExecutor(source); independent=new JdbcSqlExecutor(source);
+        }
+        @Override public void close() throws SQLException { try (Connection connection=DriverManager.getConnection(url,user,password); Statement statement=connection.createStatement()) { statement.execute("DROP SCHEMA "+schema+" CASCADE"); } }
+    }
+}

examples/business-id-runtime/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
-            <version>42.7.3</version>
+            <version>42.7.14</version>
             <scope>test</scope>
         </dependency>
         <dependency>

examples/order-management/README.md

@@ -15,6 +15,33 @@ an unauthorized high-value order as one complete graph, and queries SQLite to
 prove that zero order rows were written. The second run demonstrates
 idempotency and repeats the policy proof.
 
+## Generated Order document round trip
+
+The application-owned `GeneratedOrderDocumentService` uses the generated
+`CustomerOrder` and `OrderLine` queries and saves with a context-bound `tqd1`
+document. It allows changing item quantities and removing listed items after
+checking the current customer grant, owner, aggregate and item versions, and the
+complete issued relation. An application-owned command receipt shares the same
+SQLite JDBC transaction as the generated entity saves. The terminal entry point
+creates a temporary database and key, opens and accepts an Order document, then
+checks the saved generated entities and single receipt. It requires no web server.
+
+Install the current TeaQL SQLite and secure-document modules from the repository
+root, then run the Java 21 example:
+
+```bash
+mvn -pl teaql-sqlite,teaql-secure-document-reference -am -DskipTests install
+cd examples/order-management
+mvn -pl java-app-console -am install
+mvn -pl java-app-console exec:java -Dexec.mainClass=com.teaql.example.GeneratedDocumentCli
+```
+
+The generated consumer tests use real SQLite and cover a successful update and
+removal, idempotent replay, rejected projection and stale revision, rollback when
+an item write or receipt insert fails, and a presentation failure after commit.
+This example covers only the listed Order fields and mutations; adding another
+generated model requires its own application-owned document adapter.
+
 Read `java-app-console/.../OrderManagementApp.java` first (handwritten), then `java-lib-core/lib/.../Q.java`, `CustomerOrderRequest.java`, and `CustomerOrder.java` (generated). Java is the naming and governance gold standard: `comment(...)` may appear anywhere before `purpose(...)`; only the purposed request exposes execute methods.
 
 ## Verify the first result

examples/order-management/java-app-console/pom.xml

@@ -12,6 +12,7 @@
     <maven.compiler.release>21</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <teaql.version>1.555-RELEASE</teaql.version>
+    <exec.mainClass>com.teaql.example.OrderManagementApp</exec.mainClass>
   </properties>
   <dependencies>
     <dependency>
@@ -24,6 +25,22 @@
       <artifactId>teaql-provider-jdbc</artifactId>
       <version>${teaql.version}</version>
     </dependency>
+    <dependency>
+      <groupId>io.teaql</groupId>
+      <artifactId>teaql-secure-document-reference</artifactId>
+      <version>${teaql.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.22.3</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.xerial</groupId>
       <artifactId>sqlite-jdbc</artifactId>
@@ -36,7 +53,7 @@
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>exec-maven-plugin</artifactId>
         <version>3.5.0</version>
-        <configuration><mainClass>com.teaql.example.OrderManagementApp</mainClass></configuration>
+        <configuration><mainClass>${exec.mainClass}</mainClass></configuration>
       </plugin>
     </plugins>
   </build>