From 46bec9d81714e358e0ba81b51559249396c99729 Mon Sep 17 00:00:00 2001 From: James McVay Date: Wed, 10 Jun 2026 23:19:47 +0200 Subject: [PATCH 01/18] Add OpenCL GPU frontend: threshold, connected components, gradient clusters Runs the detector frontend on an Intel iGPU via OpenCL, gated behind APRILTAG_OPENCL=1 with silent CPU fallback. The threshold stage is byte-identical to the CPU implementation; connected components and cluster extraction replicate the CPU edge rules exactly (validated 108/108 over a 9-image x 4-family x 3-decimation corpus, bit-exact with APRILTAG_OPENCL_EXACT=1). On a Core Ultra 5 225H / Arc 140T, apriltag_detector_detect drops from 52 ms / 255 core-ms to 37 ms / 133 core-ms on a 3088x2064 frame with 32 tagStandard52h13 tags. Co-Authored-By: Claude Fable 5 --- CMakeLists.txt | 10 + apriltag_quad_thresh.c | 31 +- ocl_harness/README.md | 21 + ocl_harness/cluster_harness.c | 381 +++++++++++++ ocl_harness/corpus_harness.c | 161 ++++++ ocl_harness/detect_harness.c | 168 ++++++ ocl_harness/parity_harness.c | 123 +++++ ocl_harness/profile_harness.c | 44 ++ ocl_harness/uf_harness.c | 247 +++++++++ ocl_threshold.c | 995 ++++++++++++++++++++++++++++++++++ ocl_threshold.h | 24 + 11 files changed, 2201 insertions(+), 4 deletions(-) create mode 100644 ocl_harness/README.md create mode 100644 ocl_harness/cluster_harness.c create mode 100644 ocl_harness/corpus_harness.c create mode 100644 ocl_harness/detect_harness.c create mode 100644 ocl_harness/parity_harness.c create mode 100644 ocl_harness/profile_harness.c create mode 100644 ocl_harness/uf_harness.c create mode 100644 ocl_threshold.c create mode 100644 ocl_threshold.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b238324..fd0cbbcb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,11 @@ endif() aux_source_directory(common COMMON_SRC) set(APRILTAG_SRCS apriltag.c apriltag_pose.c apriltag_quad_thresh.c) +find_package(OpenCL QUIET) +if(OpenCL_FOUND) + list(APPEND APRILTAG_SRCS ocl_threshold.c) +endif() + # Library file(GLOB TAG_FILES ${CMAKE_CURRENT_SOURCE_DIR}/tag*.c) add_library(${PROJECT_NAME} ${APRILTAG_SRCS} ${COMMON_SRC} ${TAG_FILES}) @@ -94,6 +99,11 @@ if (UNIX) target_link_libraries(${PROJECT_NAME} PUBLIC m) endif() +if(OpenCL_FOUND) + target_compile_definitions(${PROJECT_NAME} PRIVATE APRILTAG_HAVE_OPENCL) + target_link_libraries(${PROJECT_NAME} PRIVATE OpenCL::OpenCL) +endif() + set_target_properties(${PROJECT_NAME} PROPERTIES SOVERSION 3 VERSION ${PROJECT_VERSION}) set_target_properties(${PROJECT_NAME} PROPERTIES DEBUG_POSTFIX "d") set_property(TARGET ${PROJECT_NAME} PROPERTY C_STANDARD 99) diff --git a/apriltag_quad_thresh.c b/apriltag_quad_thresh.c index e7841085..afd77428 100644 --- a/apriltag_quad_thresh.c +++ b/apriltag_quad_thresh.c @@ -44,6 +44,10 @@ either expressed or implied, of the Regents of The University of Michigan. #include "common/postscript_utils.h" #include "common/math_util.h" +#ifdef APRILTAG_HAVE_OPENCL +#include "ocl_threshold.h" +#endif + #ifdef _WIN32 static inline long int random(void) { @@ -1218,6 +1222,14 @@ void do_threshold_task(void *p) image_u8_t *threshold(apriltag_detector_t *td, image_u8_t *im) { +#ifdef APRILTAG_HAVE_OPENCL + image_u8_t *oclThreshim = oclThreshold(td, im); + if (oclThreshim != NULL) { + timeprofile_stamp(td->tp, "threshold"); + return oclThreshim; + } +#endif + int w = im->width, h = im->height, s = im->stride; assert(w < 32768); assert(h < 32768); @@ -1879,7 +1891,17 @@ zarray_t *apriltag_quad_thresh(apriltag_detector_t *td, image_u8_t *im) int w = im->width, h = im->height; - image_u8_t *threshim = threshold(td, im); + image_u8_t *threshim = NULL; + zarray_t* clusters = NULL; +#ifdef APRILTAG_HAVE_OPENCL + clusters = oclFrontend(td, im); + if (clusters != NULL) { + timeprofile_stamp(td->tp, "threshold"); + timeprofile_stamp(td->tp, "unionfind"); + } +#endif + if (clusters == NULL) { + threshim = threshold(td, im); int ts = threshim->stride; if (td->debug) @@ -1931,7 +1953,7 @@ zarray_t *apriltag_quad_thresh(apriltag_detector_t *td, image_u8_t *im) timeprofile_stamp(td->tp, "unionfind"); - zarray_t* clusters = gradient_clusters(td, threshim, w, h, ts, uf); + clusters = gradient_clusters(td, threshim, w, h, ts, uf); if (td->debug) { image_u8x3_t *d = image_u8x3_create(w, h); @@ -1964,9 +1986,10 @@ zarray_t *apriltag_quad_thresh(apriltag_detector_t *td, image_u8_t *im) image_u8x3_write_pnm(d, "debug_clusters.pnm"); image_u8x3_destroy(d); } + } - - image_u8_destroy(threshim); + if (threshim != NULL) + image_u8_destroy(threshim); timeprofile_stamp(td->tp, "make clusters"); //////////////////////////////////////////////////////// diff --git a/ocl_harness/README.md b/ocl_harness/README.md new file mode 100644 index 00000000..6d4e87e8 --- /dev/null +++ b/ocl_harness/README.md @@ -0,0 +1,21 @@ +# GPU frontend validation harnesses + +Standalone benchmarks and equivalence tests for the OpenCL frontend +(`ocl_threshold.c`). Build against the library, e.g.: + + gcc -O2 -o detbench ocl_harness/detect_harness.c -I. -Lbuild -lapriltag -lm -lpthread + +Run with `OCL_ICD_VENDORS` pointing at the Intel OpenCL ICD and +`LD_LIBRARY_PATH` at the built library. Environment flags: + +- `APRILTAG_OPENCL=1` enables the GPU frontend (silent CPU fallback). +- `APRILTAG_OPENCL_EXACT=1` bit-exact output vs the CPU path (validation). +- `APRILTAG_OPENCL_PROFILE=1` per-kernel GPU timings on stderr. +- `APRILTAG_OPENCL_DEBUG=1` fallback diagnostics on stderr. + +- `detect_harness.c` — full-detect CPU/GPU benchmark + detection equivalence. +- `corpus_harness.c` — multi-image × family × decimation equivalence sweep. +- `parity_harness.c` — threshold stage byte-parity + CPU accounting. +- `uf_harness.c` — GPU connected-components equivalence vs CPU unionfind. +- `cluster_harness.c` — GPU cluster extraction equivalence vs gradient_clusters. +- `profile_harness.c` — per-stage timeprofile of apriltag_detector_detect. diff --git a/ocl_harness/cluster_harness.c b/ocl_harness/cluster_harness.c new file mode 100644 index 00000000..37b6e657 --- /dev/null +++ b/ocl_harness/cluster_harness.c @@ -0,0 +1,381 @@ +#include +#include +#include +#include +#include +#include + +#define CL_TARGET_OPENCL_VERSION 300 +#include + +#include "apriltag.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/unionfind.h" +#include "common/zarray.h" + +image_u8_t *threshold(apriltag_detector_t *td, image_u8_t *im); +unionfind_t *connected_components(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts); +zarray_t *gradient_clusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts, unionfind_t *uf); + +struct pt { + uint16_t x, y; + int16_t gx, gy; + float slope; +}; + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +static double processCpuMs(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec * 1000.0 + usage.ru_utime.tv_usec / 1000.0 + + usage.ru_stime.tv_sec * 1000.0 + usage.ru_stime.tv_usec / 1000.0; +} + +// CCL kernels are identical to uf_harness.c; extractPairs replicates +// do_gradient_clusters' emit rules, including the connected_last dedup +// (computable per-pixel from the left neighbour's would-emit state). +static const char *clSource = + "inline uint findRoot(__global volatile uint *labels, uint i) {\n" + " uint l = labels[i];\n" + " while (l != i) { i = l; l = labels[i]; }\n" + " return i;\n" + "}\n" + "inline void mergeRoots(__global volatile uint *labels, uint a, uint b) {\n" + " while (1) {\n" + " a = findRoot(labels, a);\n" + " b = findRoot(labels, b);\n" + " if (a == b) return;\n" + " uint hi = max(a, b), lo = min(a, b);\n" + " uint old = atomic_min(&labels[hi], lo);\n" + " if (old == hi) return;\n" + " a = lo; b = old;\n" + " }\n" + "}\n" + "__kernel void initLabels(__global const uchar *im, int s, int w, int h,\n" + " __global uint *labels) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " labels[y * w + x] = (im[y * s + x] == 127) ? 0xFFFFFFFFu : (uint)(y * w + x);\n" + "}\n" + "__kernel void mergeEdges(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x < 1 || x >= w - 1 || y >= h) return;\n" + " uchar v = im[y * s + x];\n" + " if (v == 127) return;\n" + " uint idx = (uint)(y * w + x);\n" + " uchar vLeft = im[y * s + x - 1];\n" + " if (vLeft == v) mergeRoots(labels, idx, idx - 1);\n" + " if (y == 0) return;\n" + " uchar vUpLeft = im[(y - 1) * s + x - 1];\n" + " uchar vUp = im[(y - 1) * s + x];\n" + " uchar vUpRight = im[(y - 1) * s + x + 1];\n" + " if ((x == 1 || !((vLeft == vUpLeft) && (vUpLeft == vUp))) && vUp == v)\n" + " mergeRoots(labels, idx, idx - (uint)w);\n" + " if (v == 255) {\n" + " if ((x == 1 || !(vLeft == vUpLeft || vUp == vUpLeft)) && vUpLeft == v)\n" + " mergeRoots(labels, idx, idx - (uint)w - 1);\n" + " if (!(vUp == vUpRight) && vUpRight == v)\n" + " mergeRoots(labels, idx, idx - (uint)w + 1);\n" + " }\n" + "}\n" + "__kernel void compressLabels(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " if (im[y * s + x] == 127) return;\n" + " uint idx = (uint)(y * w + x);\n" + " labels[idx] = findRoot(labels, idx);\n" + "}\n" + "__kernel void countSizes(__global const uchar *im, int s, int w, int h,\n" + " __global const uint *labels, __global volatile uint *sizes) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " if (im[y * s + x] == 127) return;\n" + " atomic_inc(&sizes[labels[y * w + x]]);\n" + "}\n" + "inline int wouldEmit(__global const uchar *im, __global const uint *labels,\n" + " __global const uint *sizes, int s, int w, uint minCluster,\n" + " int x, int y, int dx, int dy) {\n" + " uchar v0 = im[y * s + x];\n" + " if (v0 == 127) return 0;\n" + " if (sizes[labels[y * w + x]] < minCluster) return 0;\n" + " uchar v1 = im[(y + dy) * s + x + dx];\n" + " if ((int)v0 + (int)v1 != 255) return 0;\n" + " if (sizes[labels[(y + dy) * w + x + dx]] < minCluster) return 0;\n" + " return 1;\n" + "}\n" + "inline void emitPair(__global const uchar *im, __global const uint *labels,\n" + " int s, int w, int x, int y, int dx, int dy,\n" + " __global volatile uint *counter, __global ulong2 *records, uint capacity) {\n" + " uchar v0 = im[y * s + x];\n" + " uchar v1 = im[(y + dy) * s + x + dx];\n" + " uint rep0 = labels[y * w + x];\n" + " uint rep1 = labels[(y + dy) * w + x + dx];\n" + " ulong key = (rep0 < rep1) ? (((ulong)rep1 << 32) | rep0) : (((ulong)rep0 << 32) | rep1);\n" + " int grad = (int)v1 - (int)v0;\n" + " ushort px = (ushort)(2 * x + dx), py = (ushort)(2 * y + dy);\n" + " ushort pgx = (ushort)(short)(dx * grad), pgy = (ushort)(short)(dy * grad);\n" + " ulong packed = ((ulong)px << 48) | ((ulong)py << 32) | ((ulong)pgx << 16) | (ulong)pgy;\n" + " uint slot = atomic_inc(counter);\n" + " if (slot < capacity) records[slot] = (ulong2)(key, packed);\n" + "}\n" + "__kernel void extractPairs(__global const uchar *im, int s, int w, int h,\n" + " __global const uint *labels, __global const uint *sizes,\n" + " uint minCluster, __global volatile uint *counter,\n" + " __global ulong2 *records, uint capacity) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x < 1 || x >= w - 1 || y < 1 || y >= h - 1) return;\n" + " uchar v0 = im[y * s + x];\n" + " if (v0 == 127) return;\n" + " if (sizes[labels[y * w + x]] < minCluster) return;\n" + " if (wouldEmit(im, labels, sizes, s, w, minCluster, x, y, 1, 0))\n" + " emitPair(im, labels, s, w, x, y, 1, 0, counter, records, capacity);\n" + " if (wouldEmit(im, labels, sizes, s, w, minCluster, x, y, 0, 1))\n" + " emitPair(im, labels, s, w, x, y, 0, 1, counter, records, capacity);\n" + " int prevEmitted = (x > 1) && wouldEmit(im, labels, sizes, s, w, minCluster, x - 1, y, 1, 1);\n" + " if (!prevEmitted && wouldEmit(im, labels, sizes, s, w, minCluster, x, y, -1, 1))\n" + " emitPair(im, labels, s, w, x, y, -1, 1, counter, records, capacity);\n" + " if (wouldEmit(im, labels, sizes, s, w, minCluster, x, y, 1, 1))\n" + " emitPair(im, labels, s, w, x, y, 1, 1, counter, records, capacity);\n" + "}\n"; + +static size_t roundUp(size_t value, size_t multiple) { + return ((value + multiple - 1) / multiple) * multiple; +} + +static int compareU64(const void *a, const void *b) { + uint64_t da = *(const uint64_t *)a, db = *(const uint64_t *)b; + return (da > db) - (da < db); +} + +static int compareRecords(const void *a, const void *b) { + const uint64_t *ra = (const uint64_t *)a, *rb = (const uint64_t *)b; + if (ra[0] != rb[0]) + return (ra[0] > rb[0]) - (ra[0] < rb[0]); + return (ra[1] > rb[1]) - (ra[1] < rb[1]); +} + +typedef struct { + uint64_t *pts; + size_t count; +} Blob; + +static int compareBlobs(const void *a, const void *b) { + const Blob *ba = (const Blob *)a, *bb = (const Blob *)b; + if (ba->count != bb->count) + return (ba->count > bb->count) - (ba->count < bb->count); + return memcmp(ba->pts, bb->pts, ba->count * 8); +} + +static uint64_t packPt(const struct pt *p) { + return ((uint64_t)p->x << 48) | ((uint64_t)p->y << 32) | + ((uint64_t)(uint16_t)p->gx << 16) | (uint64_t)(uint16_t)p->gy; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s image.pgm\n", argv[0]); + return 1; + } + image_u8_t *im = image_u8_create_from_pnm(argv[1]); + if (im == NULL) + return 1; + + apriltag_family_t *family = tagStandard52h13_create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = 8; + td->quad_decimate = 1.0f; + td->quad_sigma = 0.0f; + unsetenv("APRILTAG_OPENCL"); + zarray_t *warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + + image_u8_t *threshim = threshold(td, im); + const int w = im->width, h = im->height, s = threshim->stride; + const size_t pixelCount = (size_t)w * (size_t)h; + const uint32_t minCluster = (uint32_t)td->qtp.min_cluster_pixels; + printf("image %dx%d stride %d, min_cluster_pixels %u\n", w, h, s, minCluster); + + unionfind_t *uf = connected_components(td, threshim, w, h, s); + + // CPU clusters timing + reference content + zarray_t *cpuClusters = NULL; + { + double walls[8]; + double cpuStart = processCpuMs(); + for (int i = 0; i < 8; i++) { + if (cpuClusters != NULL) { + for (int c = 0; c < zarray_size(cpuClusters); c++) { + zarray_t *cl; + zarray_get(cpuClusters, c, &cl); + zarray_destroy(cl); + } + zarray_destroy(cpuClusters); + } + double start = nowMs(); + cpuClusters = gradient_clusters(td, threshim, w, h, s, uf); + walls[i] = nowMs() - start; + } + double cpuPer = (processCpuMs() - cpuStart) / 8; + double best = walls[0]; + for (int i = 1; i < 8; i++) + if (walls[i] < best) + best = walls[i]; + printf("CPU clusters (8t): best wall %6.2f ms cpu %6.2f core-ms per call clusters %d\n", + best, cpuPer, zarray_size(cpuClusters)); + } + + // GPU setup + cl_platform_id platform; + cl_device_id device; + clGetPlatformIDs(1, &platform, NULL); + clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device, NULL); + cl_int err; + cl_context ctx = clCreateContext(NULL, 1, &device, NULL, NULL, &err); + cl_command_queue queue = clCreateCommandQueueWithProperties(ctx, device, NULL, &err); + cl_program program = clCreateProgramWithSource(ctx, 1, &clSource, NULL, &err); + if (clBuildProgram(program, 1, &device, "", NULL, NULL) != CL_SUCCESS) { + char log[8192] = { 0 }; + clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, sizeof(log) - 1, log, NULL); + fprintf(stderr, "build failed:\n%s\n", log); + return 1; + } + cl_kernel kInit = clCreateKernel(program, "initLabels", &err); + cl_kernel kMerge = clCreateKernel(program, "mergeEdges", &err); + cl_kernel kCompress = clCreateKernel(program, "compressLabels", &err); + cl_kernel kSizes = clCreateKernel(program, "countSizes", &err); + cl_kernel kExtract = clCreateKernel(program, "extractPairs", &err); + + const size_t imageBytes = (size_t)s * (size_t)h; + const cl_uint capacity = 8 * 1024 * 1024; + cl_mem bufIm = clCreateBuffer(ctx, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR, imageBytes, threshim->buf, &err); + cl_mem bufLabels = clCreateBuffer(ctx, CL_MEM_READ_WRITE, pixelCount * 4, NULL, &err); + cl_mem bufSizes = clCreateBuffer(ctx, CL_MEM_READ_WRITE, pixelCount * 4, NULL, &err); + cl_mem bufCounter = clCreateBuffer(ctx, CL_MEM_READ_WRITE, 4, NULL, &err); + cl_mem bufRecords = clCreateBuffer(ctx, CL_MEM_READ_WRITE, (size_t)capacity * 16, NULL, &err); + if (err != CL_SUCCESS) { + fprintf(stderr, "buffer alloc failed\n"); + return 1; + } + + const cl_int cw = w, ch = h, cs = s; + cl_kernel layoutKernels[4] = { kInit, kMerge, kCompress, kSizes }; + for (int k = 0; k < 4; k++) { + clSetKernelArg(layoutKernels[k], 0, sizeof(cl_mem), &bufIm); + clSetKernelArg(layoutKernels[k], 1, sizeof(cl_int), &cs); + clSetKernelArg(layoutKernels[k], 2, sizeof(cl_int), &cw); + clSetKernelArg(layoutKernels[k], 3, sizeof(cl_int), &ch); + clSetKernelArg(layoutKernels[k], 4, sizeof(cl_mem), &bufLabels); + } + clSetKernelArg(kSizes, 5, sizeof(cl_mem), &bufSizes); + clSetKernelArg(kExtract, 0, sizeof(cl_mem), &bufIm); + clSetKernelArg(kExtract, 1, sizeof(cl_int), &cs); + clSetKernelArg(kExtract, 2, sizeof(cl_int), &cw); + clSetKernelArg(kExtract, 3, sizeof(cl_int), &ch); + clSetKernelArg(kExtract, 4, sizeof(cl_mem), &bufLabels); + clSetKernelArg(kExtract, 5, sizeof(cl_mem), &bufSizes); + clSetKernelArg(kExtract, 6, sizeof(cl_uint), &minCluster); + clSetKernelArg(kExtract, 7, sizeof(cl_mem), &bufCounter); + clSetKernelArg(kExtract, 8, sizeof(cl_mem), &bufRecords); + clSetKernelArg(kExtract, 9, sizeof(cl_uint), &capacity); + + const size_t global[2] = { roundUp((size_t)w, 16), roundUp((size_t)h, 16) }; + const cl_uint zero = 0; + + double gpuWallBest = 1e9; + double gpuCpuStart = processCpuMs(); + for (int i = 0; i < 20; i++) { + double start = nowMs(); + clEnqueueFillBuffer(queue, bufSizes, &zero, 4, 0, pixelCount * 4, 0, NULL, NULL); + clEnqueueFillBuffer(queue, bufCounter, &zero, 4, 0, 4, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kInit, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kMerge, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kCompress, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kSizes, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kExtract, 2, NULL, global, NULL, 0, NULL, NULL); + clFinish(queue); + double elapsed = nowMs() - start; + if (elapsed < gpuWallBest) + gpuWallBest = elapsed; + } + double gpuCpuPer = (processCpuMs() - gpuCpuStart) / 20; + + cl_uint recordCount = 0; + clEnqueueReadBuffer(queue, bufCounter, CL_TRUE, 0, 4, &recordCount, 0, NULL, NULL); + printf("GPU frontend chain: best wall %6.2f ms cpu %6.2f core-ms per call records %u\n", + gpuWallBest, gpuCpuPer, recordCount); + if (recordCount > capacity) { + fprintf(stderr, "record overflow\n"); + return 1; + } + + uint64_t *records = malloc((size_t)recordCount * 16); + clEnqueueReadBuffer(queue, bufRecords, CL_TRUE, 0, (size_t)recordCount * 16, records, 0, NULL, NULL); + + double sortStart = nowMs(); + qsort(records, recordCount, 16, compareRecords); + printf("validation-only CPU qsort of records: %.2f ms\n", nowMs() - sortStart); + + // GPU blobs: group sorted records by key + Blob *gpuBlobs = malloc(sizeof(Blob) * (recordCount > 0 ? recordCount : 1)); + size_t gpuBlobCount = 0; + size_t groupStart = 0; + for (size_t i = 1; i <= recordCount; i++) { + if (i == recordCount || records[2 * i] != records[2 * groupStart]) { + size_t count = i - groupStart; + uint64_t *pts = malloc(count * 8); + for (size_t j = 0; j < count; j++) + pts[j] = records[2 * (groupStart + j) + 1]; + gpuBlobs[gpuBlobCount].pts = pts; + gpuBlobs[gpuBlobCount].count = count; + gpuBlobCount++; + groupStart = i; + } + } + + // CPU blobs + size_t cpuBlobCount = (size_t)zarray_size(cpuClusters); + Blob *cpuBlobs = malloc(sizeof(Blob) * (cpuBlobCount > 0 ? cpuBlobCount : 1)); + for (size_t c = 0; c < cpuBlobCount; c++) { + zarray_t *cl; + zarray_get(cpuClusters, (int)c, &cl); + size_t count = (size_t)zarray_size(cl); + uint64_t *pts = malloc(count * 8); + for (size_t j = 0; j < count; j++) { + struct pt *p; + zarray_get_volatile(cl, (int)j, &p); + pts[j] = packPt(p); + } + qsort(pts, count, 8, compareU64); + cpuBlobs[c].pts = pts; + cpuBlobs[c].count = count; + } + + qsort(gpuBlobs, gpuBlobCount, sizeof(Blob), compareBlobs); + qsort(cpuBlobs, cpuBlobCount, sizeof(Blob), compareBlobs); + + if (gpuBlobCount != cpuBlobCount) { + printf("equivalence: CLUSTER COUNT MISMATCH cpu=%zu gpu=%zu\n", cpuBlobCount, gpuBlobCount); + return 2; + } + size_t mismatches = 0; + for (size_t i = 0; i < cpuBlobCount; i++) { + if (cpuBlobs[i].count != gpuBlobs[i].count || + memcmp(cpuBlobs[i].pts, gpuBlobs[i].pts, cpuBlobs[i].count * 8) != 0) + mismatches++; + } + if (mismatches == 0) + printf("equivalence: IDENTICAL cluster content (%zu clusters)\n", cpuBlobCount); + else + printf("equivalence: %zu of %zu clusters differ\n", mismatches, cpuBlobCount); + + return mismatches == 0 ? 0 : 2; +} diff --git a/ocl_harness/corpus_harness.c b/ocl_harness/corpus_harness.c new file mode 100644 index 00000000..d837f602 --- /dev/null +++ b/ocl_harness/corpus_harness.c @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include + +#include "apriltag.h" +#include "tag16h5.h" +#include "tag36h11.h" +#include "tagCustom48h12.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/zarray.h" + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +static double processCpuMs(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec * 1000.0 + usage.ru_utime.tv_usec / 1000.0 + + usage.ru_stime.tv_sec * 1000.0 + usage.ru_stime.tv_usec / 1000.0; +} + +typedef struct { + const char *name; + apriltag_family_t *(*create)(void); + void (*destroy)(apriltag_family_t *); +} FamilyEntry; + +static const FamilyEntry families[] = { + { "tagStandard52h13", tagStandard52h13_create, tagStandard52h13_destroy }, + { "tag36h11", tag36h11_create, tag36h11_destroy }, + { "tagCustom48h12", tagCustom48h12_create, tagCustom48h12_destroy }, + { "tag16h5", tag16h5_create, tag16h5_destroy }, +}; + +static int compareDetections(const void *a, const void *b) { + const apriltag_detection_t *da = *(apriltag_detection_t *const *)a; + const apriltag_detection_t *db = *(apriltag_detection_t *const *)b; + if (da->id != db->id) + return (da->id > db->id) - (da->id < db->id); + if (da->c[0] != db->c[0]) + return (da->c[0] > db->c[0]) - (da->c[0] < db->c[0]); + return (da->c[1] > db->c[1]) - (da->c[1] < db->c[1]); +} + +static zarray_t *runDetect(apriltag_detector_t *td, image_u8_t *im, int gpu, + double *wallMs, double *cpuMs) { + if (gpu) + setenv("APRILTAG_OPENCL", "1", 1); + else + unsetenv("APRILTAG_OPENCL"); + zarray_t *warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + double cpuStart = processCpuMs(); + double start = nowMs(); + zarray_t *detections = apriltag_detector_detect(td, im); + *wallMs = nowMs() - start; + *cpuMs = processCpuMs() - cpuStart; + return detections; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s image.pgm [image.pgm ...]\n", argv[0]); + return 1; + } + const double decimates[] = { 1.0, 1.5, 2.0 }; + int totalCases = 0, passedCases = 0; + double cpuWallSum = 0, cpuCpuSum = 0, gpuWallSum = 0, gpuCpuSum = 0; + double worstCornerDelta = 0.0; + + for (int imgIdx = 1; imgIdx < argc; imgIdx++) { + image_u8_t *im = image_u8_create_from_pnm(argv[imgIdx]); + if (im == NULL) { + fprintf(stderr, "failed to load %s\n", argv[imgIdx]); + return 1; + } + const char *shortName = strrchr(argv[imgIdx], '/'); + shortName = shortName != NULL ? shortName + 1 : argv[imgIdx]; + + for (size_t decIdx = 0; decIdx < 3; decIdx++) { + for (size_t famIdx = 0; famIdx < sizeof(families) / sizeof(families[0]); famIdx++) { + apriltag_family_t *family = families[famIdx].create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = 8; + td->quad_decimate = (float)decimates[decIdx]; + td->quad_sigma = 0.0f; + + double cpuWall, cpuCpu, gpuWall, gpuCpu; + zarray_t *cpuDet = runDetect(td, im, 0, &cpuWall, &cpuCpu); + zarray_t *gpuDet = runDetect(td, im, 1, &gpuWall, &gpuCpu); + cpuWallSum += cpuWall; + cpuCpuSum += cpuCpu; + gpuWallSum += gpuWall; + gpuCpuSum += gpuCpu; + + int cpuCount = zarray_size(cpuDet); + int gpuCount = zarray_size(gpuDet); + double maxDelta = 0.0; + int idMismatch = 0, hammingMismatch = 0; + if (cpuCount == gpuCount && cpuCount > 0) { + apriltag_detection_t **cs = malloc(sizeof(void *) * cpuCount); + apriltag_detection_t **gs = malloc(sizeof(void *) * cpuCount); + for (int i = 0; i < cpuCount; i++) { + zarray_get(cpuDet, i, &cs[i]); + zarray_get(gpuDet, i, &gs[i]); + } + qsort(cs, cpuCount, sizeof(void *), compareDetections); + qsort(gs, cpuCount, sizeof(void *), compareDetections); + for (int i = 0; i < cpuCount; i++) { + if (cs[i]->id != gs[i]->id) { + idMismatch++; + continue; + } + if (cs[i]->hamming != gs[i]->hamming) + hammingMismatch++; + for (int corner = 0; corner < 4; corner++) + for (int axis = 0; axis < 2; axis++) { + double d = fabs(cs[i]->p[corner][axis] - gs[i]->p[corner][axis]); + if (d > maxDelta) + maxDelta = d; + } + } + free(cs); + free(gs); + } + int pass = (cpuCount == gpuCount) && (idMismatch == 0) && + (hammingMismatch == 0) && (maxDelta < 0.05); + totalCases++; + if (pass) + passedCases++; + if (maxDelta > worstCornerDelta) + worstCornerDelta = maxDelta; + printf("%-22s dec %.1f %-18s cpu %3d gpu %3d maxd %.4f %s\n", + shortName, decimates[decIdx], families[famIdx].name, + cpuCount, gpuCount, maxDelta, pass ? "PASS" : "FAIL"); + if (!pass) + printf(" DETAIL: idMismatch=%d hammingMismatch=%d\n", idMismatch, hammingMismatch); + + apriltag_detections_destroy(cpuDet); + apriltag_detections_destroy(gpuDet); + apriltag_detector_destroy(td); + families[famIdx].destroy(family); + } + } + image_u8_destroy(im); + } + + printf("\nSUMMARY: %d/%d cases passed, worst corner delta %.4f px\n", + passedCases, totalCases, worstCornerDelta); + printf("SUMMARY: corpus totals cpu %.0f ms wall / %.0f core-ms gpu %.0f ms wall / %.0f core-ms\n", + cpuWallSum, cpuCpuSum, gpuWallSum, gpuCpuSum); + return passedCases == totalCases ? 0 : 2; +} diff --git a/ocl_harness/detect_harness.c b/ocl_harness/detect_harness.c new file mode 100644 index 00000000..b4586ddf --- /dev/null +++ b/ocl_harness/detect_harness.c @@ -0,0 +1,168 @@ +#include +#include +#include +#include +#include +#include + +#include "apriltag.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/timeprofile.h" +#include "common/zarray.h" + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +static double processCpuMs(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec * 1000.0 + usage.ru_utime.tv_usec / 1000.0 + + usage.ru_stime.tv_sec * 1000.0 + usage.ru_stime.tv_usec / 1000.0; +} + +static int compareDetections(const void *a, const void *b) { + const apriltag_detection_t *da = *(apriltag_detection_t *const *)a; + const apriltag_detection_t *db = *(apriltag_detection_t *const *)b; + if (da->id != db->id) + return (da->id > db->id) - (da->id < db->id); + return (da->c[0] > db->c[0]) - (da->c[0] < db->c[0]); +} + +static zarray_t *benchDetect(const char *label, apriltag_detector_t *td, image_u8_t *im, int iters) { + zarray_t *kept = NULL; + double walls[32]; + double cpuStart = processCpuMs(); + for (int i = 0; i < iters; i++) { + if (kept != NULL) + apriltag_detections_destroy(kept); + double start = nowMs(); + kept = apriltag_detector_detect(td, im); + walls[i] = nowMs() - start; + } + double cpuPer = (processCpuMs() - cpuStart) / iters; + double median; + { + double sorted[32]; + memcpy(sorted, walls, sizeof(double) * iters); + for (int i = 0; i < iters; i++) + for (int j = i + 1; j < iters; j++) + if (sorted[j] < sorted[i]) { + double t = sorted[i]; + sorted[i] = sorted[j]; + sorted[j] = t; + } + median = sorted[iters / 2]; + } + printf("%-10s wall %6.2f ms cpu %6.2f core-ms detections %d\n", + label, median, cpuPer, zarray_size(kept)); + printf("--- %s stage profile (last iteration) ---\n", label); + timeprofile_display(td->tp); + return kept; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s image.pgm\n", argv[0]); + return 1; + } + image_u8_t *im = image_u8_create_from_pnm(argv[1]); + if (im == NULL) + return 1; + + apriltag_family_t *family = tagStandard52h13_create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = 8; + td->quad_decimate = 1.0f; + td->quad_sigma = 0.0f; + + unsetenv("APRILTAG_OPENCL"); + zarray_t *warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + zarray_t *cpuDetections = benchDetect("CPU", td, im, 12); + + setenv("APRILTAG_OPENCL", "1", 1); + warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + zarray_t *gpuDetections = benchDetect("GPU", td, im, 12); + zarray_t *gpuDetections2 = benchDetect("GPU run2", td, im, 12); + + int cpuCount = zarray_size(cpuDetections); + int gpuCount = zarray_size(gpuDetections); + if (cpuCount != gpuCount) { + printf("RESULT: DETECTION COUNT MISMATCH cpu=%d gpu=%d\n", cpuCount, gpuCount); + return 2; + } + + apriltag_detection_t **cpuSorted = malloc(sizeof(void *) * cpuCount); + apriltag_detection_t **gpuSorted = malloc(sizeof(void *) * gpuCount); + for (int i = 0; i < cpuCount; i++) { + zarray_get(cpuDetections, i, &cpuSorted[i]); + zarray_get(gpuDetections, i, &gpuSorted[i]); + } + qsort(cpuSorted, cpuCount, sizeof(void *), compareDetections); + qsort(gpuSorted, gpuCount, sizeof(void *), compareDetections); + + int idMismatches = 0, hammingMismatches = 0; + double maxCornerDelta = 0.0, maxCenterDelta = 0.0; + for (int i = 0; i < cpuCount; i++) { + if (cpuSorted[i]->id != gpuSorted[i]->id) { + idMismatches++; + continue; + } + if (cpuSorted[i]->hamming != gpuSorted[i]->hamming) + hammingMismatches++; + for (int corner = 0; corner < 4; corner++) { + for (int axis = 0; axis < 2; axis++) { + double delta = fabs(cpuSorted[i]->p[corner][axis] - gpuSorted[i]->p[corner][axis]); + if (delta > maxCornerDelta) + maxCornerDelta = delta; + } + } + for (int axis = 0; axis < 2; axis++) { + double delta = fabs(cpuSorted[i]->c[axis] - gpuSorted[i]->c[axis]); + if (delta > maxCenterDelta) + maxCenterDelta = delta; + } + } + + printf("RESULT: %d detections both modes, id mismatches %d, hamming mismatches %d\n", + cpuCount, idMismatches, hammingMismatches); + printf("RESULT: max corner delta %.6f px, max center delta %.6f px\n", + maxCornerDelta, maxCenterDelta); + + // GPU run-to-run determinism: same detections across two GPU runs? + double gpuRunDelta = 0.0; + int gpuRunIdMismatch = (zarray_size(gpuDetections2) != gpuCount); + if (!gpuRunIdMismatch) { + apriltag_detection_t **gpu2Sorted = malloc(sizeof(void *) * gpuCount); + for (int i = 0; i < gpuCount; i++) + zarray_get(gpuDetections2, i, &gpu2Sorted[i]); + qsort(gpu2Sorted, gpuCount, sizeof(void *), compareDetections); + for (int i = 0; i < gpuCount; i++) { + if (gpuSorted[i]->id != gpu2Sorted[i]->id) { + gpuRunIdMismatch = 1; + break; + } + for (int corner = 0; corner < 4; corner++) + for (int axis = 0; axis < 2; axis++) { + double delta = fabs(gpuSorted[i]->p[corner][axis] - gpu2Sorted[i]->p[corner][axis]); + if (delta > gpuRunDelta) + gpuRunDelta = delta; + } + } + } + printf("RESULT: gpu run-to-run max corner delta %.6f px%s\n", + gpuRunDelta, gpuRunIdMismatch ? " (ID MISMATCH)" : ""); + + // 0.05 px gate: point-order differences within clusters shift fitted + // corners at this scale; vide reprojection errors are an order of + // magnitude larger. + int pass = (idMismatches == 0) && (hammingMismatches == 0) && (maxCornerDelta < 0.05) && !gpuRunIdMismatch; + printf("RESULT: %s\n", pass ? "PASS" : "FAIL"); + return pass ? 0 : 2; +} diff --git a/ocl_harness/parity_harness.c b/ocl_harness/parity_harness.c new file mode 100644 index 00000000..bda7c7df --- /dev/null +++ b/ocl_harness/parity_harness.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include + +#include "apriltag.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/zarray.h" + +image_u8_t *threshold(apriltag_detector_t *td, image_u8_t *im); + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +static double processCpuMs(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + double user = usage.ru_utime.tv_sec * 1000.0 + usage.ru_utime.tv_usec / 1000.0; + double sys = usage.ru_stime.tv_sec * 1000.0 + usage.ru_stime.tv_usec / 1000.0; + return user + sys; +} + +static int compareDoubles(const void *a, const void *b) { + double da = *(const double *)a, db = *(const double *)b; + return (da > db) - (da < db); +} + +static void benchThreshold(const char *label, apriltag_detector_t *td, image_u8_t *im, int iters) { + double times[64]; + double cpuStart = processCpuMs(); + for (int i = 0; i < iters; i++) { + double start = nowMs(); + image_u8_t *result = threshold(td, im); + times[i] = nowMs() - start; + image_u8_destroy(result); + } + double cpuPerCall = (processCpuMs() - cpuStart) / iters; + qsort(times, iters, sizeof(double), compareDoubles); + printf("%-14s wall %6.2f ms cpu %6.2f core-ms per call\n", label, times[iters / 2], cpuPerCall); +} + +static void benchDetect(const char *label, apriltag_detector_t *td, image_u8_t *im, int iters) { + double times[64]; + int tagCount = 0; + double cpuStart = processCpuMs(); + for (int i = 0; i < iters; i++) { + double start = nowMs(); + zarray_t *detections = apriltag_detector_detect(td, im); + times[i] = nowMs() - start; + tagCount = zarray_size(detections); + apriltag_detections_destroy(detections); + } + double cpuPerCall = (processCpuMs() - cpuStart) / iters; + qsort(times, iters, sizeof(double), compareDoubles); + printf("%-14s wall %6.2f ms cpu %6.2f core-ms per call tags %d\n", + label, times[iters / 2], cpuPerCall, tagCount); +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s image.pgm\n", argv[0]); + return 1; + } + image_u8_t *im = image_u8_create_from_pnm(argv[1]); + if (im == NULL) { + fprintf(stderr, "failed to load %s\n", argv[1]); + return 1; + } + + apriltag_family_t *family = tagStandard52h13_create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = 8; + td->quad_decimate = 1.0f; + td->quad_sigma = 0.0f; + + unsetenv("APRILTAG_OPENCL"); + zarray_t *warmupDetections = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmupDetections); + + image_u8_t *cpuResult = threshold(td, im); + setenv("APRILTAG_OPENCL", "1", 1); + image_u8_t *gpuResult = threshold(td, im); + + size_t bytes = (size_t)cpuResult->stride * cpuResult->height; + int identical = memcmp(cpuResult->buf, gpuResult->buf, bytes) == 0; + if (identical) { + printf("parity: IDENTICAL (%zu bytes)\n", bytes); + } else { + size_t diffCount = 0, firstDiff = 0; + for (size_t i = 0; i < bytes; i++) { + if (cpuResult->buf[i] != gpuResult->buf[i]) { + if (diffCount == 0) + firstDiff = i; + diffCount++; + } + } + printf("parity: MISMATCH (%zu of %zu bytes differ, first at %zu: cpu=%d gpu=%d)\n", + diffCount, bytes, firstDiff, cpuResult->buf[firstDiff], gpuResult->buf[firstDiff]); + } + image_u8_destroy(cpuResult); + image_u8_destroy(gpuResult); + + setenv("APRILTAG_OPENCL", "1", 1); + benchThreshold("thresh GPU", td, im, 30); + unsetenv("APRILTAG_OPENCL"); + benchThreshold("thresh CPU", td, im, 30); + + setenv("APRILTAG_OPENCL", "1", 1); + benchDetect("detect GPUthr", td, im, 15); + unsetenv("APRILTAG_OPENCL"); + benchDetect("detect CPU", td, im, 15); + + apriltag_detector_destroy(td); + tagStandard52h13_destroy(family); + image_u8_destroy(im); + return identical ? 0 : 2; +} diff --git a/ocl_harness/profile_harness.c b/ocl_harness/profile_harness.c new file mode 100644 index 00000000..b14e94c7 --- /dev/null +++ b/ocl_harness/profile_harness.c @@ -0,0 +1,44 @@ +#include +#include + +#include "apriltag.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/timeprofile.h" +#include "common/zarray.h" + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s image.pgm nthreads\n", argv[0]); + return 1; + } + image_u8_t *im = image_u8_create_from_pnm(argv[1]); + if (im == NULL) { + fprintf(stderr, "failed to load %s\n", argv[1]); + return 1; + } + + apriltag_family_t *family = tagStandard52h13_create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = atoi(argv[2]); + td->quad_decimate = 1.0f; + td->quad_sigma = 0.0f; + td->refine_edges = 1; + td->decode_sharpening = 0.0; + + for (int i = 0; i < 5; i++) { + zarray_t *warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + } + + zarray_t *detections = apriltag_detector_detect(td, im); + printf("detections: %d\n", zarray_size(detections)); + timeprofile_display(td->tp); + + apriltag_detections_destroy(detections); + apriltag_detector_destroy(td); + tagStandard52h13_destroy(family); + image_u8_destroy(im); + return 0; +} diff --git a/ocl_harness/uf_harness.c b/ocl_harness/uf_harness.c new file mode 100644 index 00000000..631dbfc5 --- /dev/null +++ b/ocl_harness/uf_harness.c @@ -0,0 +1,247 @@ +#include +#include +#include +#include +#include + +#define CL_TARGET_OPENCL_VERSION 300 +#include + +#include "apriltag.h" +#include "tagStandard52h13.h" +#include "common/image_u8.h" +#include "common/unionfind.h" +#include "common/zarray.h" + +image_u8_t *threshold(apriltag_detector_t *td, image_u8_t *im); +unionfind_t *connected_components(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts); + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +static double processCpuMs(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec * 1000.0 + usage.ru_utime.tv_usec / 1000.0 + + usage.ru_stime.tv_sec * 1000.0 + usage.ru_stime.tv_usec / 1000.0; +} + +// GPU connected-components: replicates the exact edge rules of +// do_unionfind_first_line / do_unionfind_line2 (including the border bounds +// x in [1, w-2] and the redundancy-skip conditions, which leak into +// component semantics at the right image border). Final labels are the +// minimum pixel index of each component, which is order-independent and +// therefore deterministic under GPU execution. +static const char *cclSource = + "inline uint findRoot(__global volatile uint *labels, uint i) {\n" + " uint l = labels[i];\n" + " while (l != i) { i = l; l = labels[i]; }\n" + " return i;\n" + "}\n" + "inline void mergeRoots(__global volatile uint *labels, uint a, uint b) {\n" + " while (1) {\n" + " a = findRoot(labels, a);\n" + " b = findRoot(labels, b);\n" + " if (a == b) return;\n" + " uint hi = max(a, b), lo = min(a, b);\n" + " uint old = atomic_min(&labels[hi], lo);\n" + " if (old == hi) return;\n" + " a = lo; b = old;\n" + " }\n" + "}\n" + "__kernel void initLabels(__global const uchar *im, int s, int w, int h,\n" + " __global uint *labels) {\n" + " int x = get_global_id(0);\n" + " int y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " labels[y * w + x] = (im[y * s + x] == 127) ? 0xFFFFFFFFu : (uint)(y * w + x);\n" + "}\n" + "__kernel void mergeEdges(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels) {\n" + " int x = get_global_id(0);\n" + " int y = get_global_id(1);\n" + " if (x < 1 || x >= w - 1 || y >= h) return;\n" + " uchar v = im[y * s + x];\n" + " if (v == 127) return;\n" + " uint idx = (uint)(y * w + x);\n" + " uchar vLeft = im[y * s + x - 1];\n" + " if (vLeft == v) mergeRoots(labels, idx, idx - 1);\n" + " if (y == 0) return;\n" + " uchar vUpLeft = im[(y - 1) * s + x - 1];\n" + " uchar vUp = im[(y - 1) * s + x];\n" + " uchar vUpRight = im[(y - 1) * s + x + 1];\n" + " if ((x == 1 || !((vLeft == vUpLeft) && (vUpLeft == vUp))) && vUp == v)\n" + " mergeRoots(labels, idx, idx - (uint)w);\n" + " if (v == 255) {\n" + " if ((x == 1 || !(vLeft == vUpLeft || vUp == vUpLeft)) && vUpLeft == v)\n" + " mergeRoots(labels, idx, idx - (uint)w - 1);\n" + " if (!(vUp == vUpRight) && vUpRight == v)\n" + " mergeRoots(labels, idx, idx - (uint)w + 1);\n" + " }\n" + "}\n" + "__kernel void compressLabels(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels) {\n" + " int x = get_global_id(0);\n" + " int y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " if (im[y * s + x] == 127) return;\n" + " uint idx = (uint)(y * w + x);\n" + " labels[idx] = findRoot(labels, idx);\n" + "}\n"; + +static size_t roundUp(size_t value, size_t multiple) { + return ((value + multiple - 1) / multiple) * multiple; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s image.pgm\n", argv[0]); + return 1; + } + image_u8_t *im = image_u8_create_from_pnm(argv[1]); + if (im == NULL) { + fprintf(stderr, "failed to load %s\n", argv[1]); + return 1; + } + + apriltag_family_t *family = tagStandard52h13_create(); + apriltag_detector_t *td = apriltag_detector_create(); + apriltag_detector_add_family(td, family); + td->nthreads = 8; + td->quad_decimate = 1.0f; + td->quad_sigma = 0.0f; + unsetenv("APRILTAG_OPENCL"); + zarray_t *warmup = apriltag_detector_detect(td, im); + apriltag_detections_destroy(warmup); + + image_u8_t *threshim = threshold(td, im); + const int w = im->width, h = im->height, s = threshim->stride; + const size_t pixelCount = (size_t)w * (size_t)h; + printf("image %dx%d stride %d\n", w, h, s); + + // --- CPU reference (the lib's parallel implementation, 8 threads) --- + unionfind_t *uf = NULL; + { + double cpuStart = processCpuMs(); + double walls[32]; + for (int i = 0; i < 20; i++) { + double start = nowMs(); + uf = connected_components(td, threshim, w, h, s); + walls[i] = nowMs() - start; + } + double cpuPer = (processCpuMs() - cpuStart) / 20; + double best = walls[0]; + for (int i = 1; i < 20; i++) + if (walls[i] < best) + best = walls[i]; + printf("CPU unionfind (8t): best wall %6.2f ms cpu %6.2f core-ms per call\n", best, cpuPer); + } + + uint32_t *cpuCanon = malloc(pixelCount * 4); + uint32_t *minOfRep = malloc(pixelCount * 4); + memset(minOfRep, 0xFF, pixelCount * 4); + for (size_t idx = 0; idx < pixelCount; idx++) { + int y = (int)(idx / w), x = (int)(idx % w); + if (threshim->buf[y * s + x] == 127) + continue; + uint32_t rep = unionfind_get_representative(uf, (uint32_t)idx); + if (idx < minOfRep[rep]) + minOfRep[rep] = (uint32_t)idx; + } + for (size_t idx = 0; idx < pixelCount; idx++) { + int y = (int)(idx / w), x = (int)(idx % w); + if (threshim->buf[y * s + x] == 127) { + cpuCanon[idx] = 0xFFFFFFFFu; + continue; + } + cpuCanon[idx] = minOfRep[unionfind_get_representative(uf, (uint32_t)idx)]; + } + + // --- GPU CCL --- + cl_platform_id platform; + cl_uint platformCount = 0; + if (clGetPlatformIDs(1, &platform, &platformCount) != CL_SUCCESS || platformCount == 0) { + fprintf(stderr, "no OpenCL platform\n"); + return 1; + } + cl_device_id device; + if (clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device, NULL) != CL_SUCCESS) { + fprintf(stderr, "no GPU device\n"); + return 1; + } + cl_int err; + cl_context ctx = clCreateContext(NULL, 1, &device, NULL, NULL, &err); + cl_command_queue queue = clCreateCommandQueueWithProperties(ctx, device, NULL, &err); + cl_program program = clCreateProgramWithSource(ctx, 1, &cclSource, NULL, &err); + if (clBuildProgram(program, 1, &device, "", NULL, NULL) != CL_SUCCESS) { + char log[8192] = { 0 }; + clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, sizeof(log) - 1, log, NULL); + fprintf(stderr, "build failed:\n%s\n", log); + return 1; + } + cl_kernel kInit = clCreateKernel(program, "initLabels", &err); + cl_kernel kMerge = clCreateKernel(program, "mergeEdges", &err); + cl_kernel kCompress = clCreateKernel(program, "compressLabels", &err); + + const size_t imageBytes = (size_t)s * (size_t)h; + cl_mem bufIm = clCreateBuffer(ctx, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR, imageBytes, threshim->buf, &err); + cl_mem bufLabels = clCreateBuffer(ctx, CL_MEM_READ_WRITE, pixelCount * 4, NULL, &err); + if (err != CL_SUCCESS) { + fprintf(stderr, "buffer alloc failed\n"); + return 1; + } + + const cl_int cw = w, ch = h, cs = s; + cl_kernel kernels[3] = { kInit, kMerge, kCompress }; + for (int k = 0; k < 3; k++) { + clSetKernelArg(kernels[k], 0, sizeof(cl_mem), &bufIm); + clSetKernelArg(kernels[k], 1, sizeof(cl_int), &cs); + clSetKernelArg(kernels[k], 2, sizeof(cl_int), &cw); + clSetKernelArg(kernels[k], 3, sizeof(cl_int), &ch); + clSetKernelArg(kernels[k], 4, sizeof(cl_mem), &bufLabels); + } + const size_t global[2] = { roundUp((size_t)w, 16), roundUp((size_t)h, 16) }; + + double gpuWallBest = 1e9; + double gpuCpuStart = processCpuMs(); + for (int i = 0; i < 20; i++) { + double start = nowMs(); + clEnqueueNDRangeKernel(queue, kInit, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kMerge, 2, NULL, global, NULL, 0, NULL, NULL); + clEnqueueNDRangeKernel(queue, kCompress, 2, NULL, global, NULL, 0, NULL, NULL); + clFinish(queue); + double elapsed = nowMs() - start; + if (elapsed < gpuWallBest) + gpuWallBest = elapsed; + } + double gpuCpuPer = (processCpuMs() - gpuCpuStart) / 20; + printf("GPU ccl: best wall %6.2f ms cpu %6.2f core-ms per call\n", gpuWallBest, gpuCpuPer); + + uint32_t *gpuLabels = malloc(pixelCount * 4); + clEnqueueReadBuffer(queue, bufLabels, CL_TRUE, 0, pixelCount * 4, gpuLabels, 0, NULL, NULL); + + size_t mismatches = 0, firstMismatch = 0, checked = 0; + for (size_t idx = 0; idx < pixelCount; idx++) { + int y = (int)(idx / w), x = (int)(idx % w); + if (threshim->buf[y * s + x] == 127) + continue; + checked++; + if (cpuCanon[idx] != gpuLabels[idx]) { + if (mismatches == 0) + firstMismatch = idx; + mismatches++; + } + } + if (mismatches == 0) { + printf("equivalence: IDENTICAL components (%zu pixels checked)\n", checked); + } else { + printf("equivalence: %zu of %zu pixels mismatch, first at idx %zu (x=%zu y=%zu) cpu=%u gpu=%u\n", + mismatches, checked, firstMismatch, firstMismatch % w, firstMismatch / w, + cpuCanon[firstMismatch], gpuLabels[firstMismatch]); + } + + return mismatches == 0 ? 0 : 2; +} diff --git a/ocl_threshold.c b/ocl_threshold.c new file mode 100644 index 00000000..b17e42c5 --- /dev/null +++ b/ocl_threshold.c @@ -0,0 +1,995 @@ +#include "ocl_threshold.h" + +#define CL_TARGET_OPENCL_VERSION 300 +#include +#include +#include +#include +#include + +#include "common/workerpool.h" + +// GPU implementation of the detector frontend: adaptive tile threshold, +// connected components, component sizes, and boundary-pair extraction, +// replicating the CPU implementations' exact semantics (see +// apriltag_quad_thresh.c). Cluster grouping uses a single GPU partition pass +// over the high bits of the component-pair key; final grouping happens on +// the CPU during the cluster build it must perform anyway. +// +// All entry points return NULL when the GPU path is disabled (APRILTAG_OPENCL +// unset) or unavailable, in which case callers run the CPU implementation. +// APRILTAG_OPENCL_PROFILE=1 prints per-kernel GPU timings to stderr. + +static const char *sourceThreshold = + "__kernel void tileMinmax(__global const uchar *im, int s, int tw,\n" + " __global uchar *tileMax, __global uchar *tileMin) {\n" + " int tx = get_global_id(0);\n" + " int ty = get_global_id(1);\n" + " if (tx >= tw) return;\n" + " uchar mx = 0, mn = 255;\n" + " int base = (ty * 4) * s + tx * 4;\n" + " for (int dy = 0; dy < 4; dy++) {\n" + " for (int dx = 0; dx < 4; dx++) {\n" + " uchar v = im[base + dy * s + dx];\n" + " mx = max(mx, v);\n" + " mn = min(mn, v);\n" + " }\n" + " }\n" + " tileMax[ty * tw + tx] = mx;\n" + " tileMin[ty * tw + tx] = mn;\n" + "}\n" + "__kernel void tileBlur(__global const uchar *tileMax, __global const uchar *tileMin,\n" + " int tw, int th,\n" + " __global uchar *blurMax, __global uchar *blurMin) {\n" + " int tx = get_global_id(0);\n" + " int ty = get_global_id(1);\n" + " if (tx >= tw || ty >= th) return;\n" + " uchar mx = 0, mn = 255;\n" + " for (int dy = -1; dy <= 1; dy++) {\n" + " if (ty + dy < 0 || ty + dy >= th) continue;\n" + " for (int dx = -1; dx <= 1; dx++) {\n" + " if (tx + dx < 0 || tx + dx >= tw) continue;\n" + " mx = max(mx, tileMax[(ty + dy) * tw + tx + dx]);\n" + " mn = min(mn, tileMin[(ty + dy) * tw + tx + dx]);\n" + " }\n" + " }\n" + " blurMax[ty * tw + tx] = mx;\n" + " blurMin[ty * tw + tx] = mn;\n" + "}\n" + // classify also seeds the CCL labels so the frontend path can skip a + // whole initLabels pass over the image. + "__kernel void classify(__global const uchar *im, int s, int w, int h,\n" + " int tw, int th,\n" + " __global const uchar *blurMax, __global const uchar *blurMin,\n" + " int minWhiteBlackDiff, __global uchar *out, __global uint *labels) {\n" + " int x = get_global_id(0);\n" + " int y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " int interior = (x < tw * 4) && (y < th * 4);\n" + " int tx = min(x >> 2, tw - 1);\n" + " int ty = min(y >> 2, th - 1);\n" + " int mn = blurMin[ty * tw + tx];\n" + " int mx = blurMax[ty * tw + tx];\n" + " uchar result;\n" + " if (interior && (mx - mn < minWhiteBlackDiff)) {\n" + " result = 127;\n" + " } else {\n" + " int thresh = mn + (mx - mn) / 2;\n" + " result = (im[y * s + x] > thresh) ? (uchar)255 : (uchar)0;\n" + " }\n" + " out[y * s + x] = result;\n" + " labels[y * w + x] = (result == 127) ? 0xFFFFFFFFu : (uint)(y * w + x);\n" + "}\n"; + +static const char *sourceCcl = + "inline uint findRoot(__global volatile uint *labels, uint i) {\n" + " uint l = labels[i];\n" + " while (l != i) { i = l; l = labels[i]; }\n" + " return i;\n" + "}\n" + "inline void mergeRoots(__global volatile uint *labels, uint a, uint b) {\n" + " while (1) {\n" + " a = findRoot(labels, a);\n" + " b = findRoot(labels, b);\n" + " if (a == b) return;\n" + " uint hi = max(a, b), lo = min(a, b);\n" + " uint old = atomic_min(&labels[hi], lo);\n" + " if (old == hi) return;\n" + " a = lo; b = old;\n" + " }\n" + "}\n" + "__kernel void initLabels(__global const uchar *im, int s, int w, int h,\n" + " __global uint *labels) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " labels[y * w + x] = (im[y * s + x] == 127) ? 0xFFFFFFFFu : (uint)(y * w + x);\n" + "}\n" + "__kernel void mergeEdges(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x < 1 || x >= w - 1 || y >= h) return;\n" + " uchar v = im[y * s + x];\n" + " if (v == 127) return;\n" + " uint idx = (uint)(y * w + x);\n" + " uchar vLeft = im[y * s + x - 1];\n" + " if (vLeft == v) mergeRoots(labels, idx, idx - 1);\n" + " if (y == 0) return;\n" + " uchar vUpLeft = im[(y - 1) * s + x - 1];\n" + " uchar vUp = im[(y - 1) * s + x];\n" + " uchar vUpRight = im[(y - 1) * s + x + 1];\n" + " if ((x == 1 || !((vLeft == vUpLeft) && (vUpLeft == vUp))) && vUp == v)\n" + " mergeRoots(labels, idx, idx - (uint)w);\n" + " if (v == 255) {\n" + " if ((x == 1 || !(vLeft == vUpLeft || vUp == vUpLeft)) && vUpLeft == v)\n" + " mergeRoots(labels, idx, idx - (uint)w - 1);\n" + " if (!(vUp == vUpRight) && vUpRight == v)\n" + " mergeRoots(labels, idx, idx - (uint)w + 1);\n" + " }\n" + "}\n"; + +static const char *sourceCompress = + // Path-compress every label and accumulate component sizes with one + // atomic per horizontal same-root run instead of one per pixel: large + // uniform regions otherwise serialize millions of atomics on one root. + "__kernel void compressAndCount(__global const uchar *im, int s, int w, int h,\n" + " __global volatile uint *labels, __global volatile uint *sizes) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " int lid = get_local_id(0);\n" + " __local uint roots[128];\n" + " uint root = 0xFFFFFFFFu;\n" + " if (x < w && y < h && im[y * s + x] != 127) {\n" + " root = findRoot(labels, (uint)(y * w + x));\n" + " labels[y * w + x] = root;\n" + " }\n" + " roots[lid] = root;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " if (root == 0xFFFFFFFFu) return;\n" + " if (lid > 0 && roots[lid - 1] == root) return;\n" + " uint run = 1;\n" + " uint lsz = (uint)get_local_size(0);\n" + " while ((uint)lid + run < lsz && roots[lid + run] == root) run++;\n" + " atomic_add(&sizes[root], run);\n" + "}\n" + // One coalesced byte per pixel replaces the extract kernel's repeated + // scattered label+size lookups. + "__kernel void buildBigMap(__global const uchar *im, int s, int w, int h,\n" + " __global const uint *labels, __global const uint *sizes,\n" + " uint minCluster, __global uchar *bigMap) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x >= w || y >= h) return;\n" + " uint idx = (uint)(y * w + x);\n" + " uchar ok = 0;\n" + " if (im[y * s + x] != 127)\n" + " ok = (sizes[labels[idx]] >= minCluster) ? (uchar)1 : (uchar)0;\n" + " bigMap[idx] = ok;\n" + "}\n"; + +static const char *sourceExtract = + "inline int wouldEmit(__global const uchar *im, __global const uchar *bigMap,\n" + " int s, int w, int x, int y, int dx, int dy) {\n" + " if (bigMap[y * w + x] == 0) return 0;\n" + " uchar v0 = im[y * s + x];\n" + " uchar v1 = im[(y + dy) * s + x + dx];\n" + " if ((int)v0 + (int)v1 != 255) return 0;\n" + " return bigMap[(y + dy) * w + x + dx] != 0;\n" + "}\n" + "inline void emitPair(__global const uchar *im, __global const uint *labels,\n" + " int s, int w, int x, int y, int dx, int dy,\n" + " __global volatile uint *counter, __global ulong2 *records, uint capacity) {\n" + " uchar v0 = im[y * s + x];\n" + " uchar v1 = im[(y + dy) * s + x + dx];\n" + " uint rep0 = labels[y * w + x];\n" + " uint rep1 = labels[(y + dy) * w + x + dx];\n" + " ulong key = (rep0 < rep1) ? (((ulong)rep1 << 32) | rep0) : (((ulong)rep0 << 32) | rep1);\n" + " int grad = (int)v1 - (int)v0;\n" + " ushort px = (ushort)(2 * x + dx), py = (ushort)(2 * y + dy);\n" + " ushort pgx = (ushort)(short)(dx * grad), pgy = (ushort)(short)(dy * grad);\n" + " ulong packed = ((ulong)px << 48) | ((ulong)py << 32) | ((ulong)pgx << 16) | (ulong)pgy;\n" + " uint slot = atomic_inc(counter);\n" + " if (slot < capacity) records[slot] = (ulong2)(key, packed);\n" + "}\n" + "__kernel void extractPairs(__global const uchar *im, int s, int w, int h,\n" + " __global const uint *labels, __global const uchar *bigMap,\n" + " __global volatile uint *counter,\n" + " __global ulong2 *records, uint capacity) {\n" + " int x = get_global_id(0), y = get_global_id(1);\n" + " if (x < 1 || x >= w - 1 || y < 1 || y >= h - 1) return;\n" + " if (bigMap[y * w + x] == 0) return;\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 1, 0))\n" + " emitPair(im, labels, s, w, x, y, 1, 0, counter, records, capacity);\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 0, 1))\n" + " emitPair(im, labels, s, w, x, y, 0, 1, counter, records, capacity);\n" + " int prevEmitted = (x > 1) && wouldEmit(im, bigMap, s, w, x - 1, y, 1, 1);\n" + " if (!prevEmitted && wouldEmit(im, bigMap, s, w, x, y, -1, 1))\n" + " emitPair(im, labels, s, w, x, y, -1, 1, counter, records, capacity);\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 1, 1))\n" + " emitPair(im, labels, s, w, x, y, 1, 1, counter, records, capacity);\n" + "}\n"; + +static const char *sourcePartition = + // Both kernels read the live record count from the device so the host + // never has to stall mid-chain; launched over the full capacity. + "__kernel void histKeys(__global const ulong2 *records, __global const uint *counter,\n" + " uint capacity, __global volatile uint *hist) {\n" + " __local uint localCount;\n" + " if (get_local_id(0) == 0) localCount = min(counter[0], capacity);\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " uint i = get_global_id(0);\n" + " if (i >= localCount) return;\n" + " atomic_inc(&hist[(uint)((records[i].x >> 39) & 0xFFFFul)]);\n" + "}\n" + "__kernel void scatterRecords(__global const ulong2 *records, __global const uint *counter,\n" + " uint capacity, __global volatile uint *offsets,\n" + " __global ulong2 *out) {\n" + " __local uint localCount;\n" + " if (get_local_id(0) == 0) localCount = min(counter[0], capacity);\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " uint i = get_global_id(0);\n" + " if (i >= localCount) return;\n" + " ulong2 r = records[i];\n" + " out[atomic_inc(&offsets[(uint)((r.x >> 39) & 0xFFFFul)])] = r;\n" + "}\n"; + +static const char *sourceScan = + "__kernel void scanLocal(__global const uint *hist, __global uint *offsets,\n" + " __global uint *blockSums) {\n" + " int lid = get_local_id(0);\n" + " int gid = get_global_id(0);\n" + " __local uint tmp[256];\n" + " tmp[lid] = hist[gid];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int offset = 1; offset < 256; offset <<= 1) {\n" + " uint v = (lid >= offset) ? tmp[lid - offset] : 0u;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " tmp[lid] += v;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " offsets[gid] = tmp[lid] - hist[gid];\n" + " if (lid == 255) blockSums[get_group_id(0)] = tmp[lid];\n" + "}\n" + "__kernel void scanBlocks(__global uint *blockSums) {\n" + " uint running = 0;\n" + " for (int i = 0; i < 256; i++) {\n" + " uint v = blockSums[i];\n" + " blockSums[i] = running;\n" + " running += v;\n" + " }\n" + "}\n" + "__kernel void addBlockOffsets(__global uint *offsets, __global const uint *blockSums) {\n" + " offsets[get_global_id(0)] += blockSums[get_group_id(0)];\n" + "}\n"; + +#define OCL_RECORD_CAPACITY (8u * 1024u * 1024u) +#define OCL_BIN_COUNT 65536u + +typedef struct { + uint16_t x, y; + int16_t gx, gy; + float slope; +} OclPt; + +static pthread_once_t oclInitOnce = PTHREAD_ONCE_INIT; +static pthread_mutex_t oclMutex = PTHREAD_MUTEX_INITIALIZER; +static int oclReady = 0; +static cl_context oclContext; +static cl_command_queue oclQueue; +static cl_kernel oclKernelTileMinmax; +static cl_kernel oclKernelTileBlur; +static cl_kernel oclKernelClassify; +static cl_kernel oclKernelInitLabels; +static cl_kernel oclKernelMergeEdges; +static cl_kernel oclKernelCompressAndCount; +static cl_kernel oclKernelBuildBigMap; +static cl_kernel oclKernelExtractPairs; +static cl_kernel oclKernelHistKeys; +static cl_kernel oclKernelScatterRecords; +static cl_kernel oclKernelScanLocal; +static cl_kernel oclKernelScanBlocks; +static cl_kernel oclKernelAddBlockOffsets; + +typedef struct { + int valid; + cl_int w, h, s; + const void *thresholdOutputFor; + cl_mem bufIm; + cl_mem bufOut; + cl_mem bufMaxRaw; + cl_mem bufMinRaw; + cl_mem bufMaxBlur; + cl_mem bufMinBlur; + cl_mem bufLabels; + cl_mem bufSizes; + cl_mem bufBigMap; + cl_mem bufCounter; + cl_mem bufRecords; + cl_mem bufPartitioned; + cl_mem bufHist; + cl_mem bufOffsets; + cl_mem bufBlockSums; +} OclBufferCache; + +static OclBufferCache cache; +static uint32_t histHost[OCL_BIN_COUNT]; +static uint32_t offsetsHost[OCL_BIN_COUNT]; +static uint32_t counterHost; + +// APRILTAG_OPENCL_EXACT=1 sorts cluster points into the CPU emitter's exact +// order, making output bit-identical to the CPU path (validation mode). It +// currently costs more CPU than it saves; production runs without it, where +// output is content-equivalent and corners may differ at the 0.02 px level. +// TODO: emit records row-ordered on the GPU to get exactness for free. +static int oclExactOrder; + +static cl_event profEventsArr[32]; +static const char *profNamesArr[32]; +static int profEventCount; +static int profEnabled; + +static cl_event *profSlot(const char *name) +{ + if (!profEnabled || profEventCount >= 32) + return NULL; + profNamesArr[profEventCount] = name; + return &profEventsArr[profEventCount++]; +} + +static void profReset(void) +{ + profEnabled = getenv("APRILTAG_OPENCL_PROFILE") != NULL; + profEventCount = 0; +} + +static void profPrint(void) +{ + if (!profEnabled) + return; + cl_ulong first = (cl_ulong)-1, last = 0; + for (int i = 0; i < profEventCount; i++) { + cl_ulong t0 = 0, t1 = 0; + clGetEventProfilingInfo(profEventsArr[i], CL_PROFILING_COMMAND_START, sizeof(t0), &t0, NULL); + clGetEventProfilingInfo(profEventsArr[i], CL_PROFILING_COMMAND_END, sizeof(t1), &t1, NULL); + fprintf(stderr, " %-16s %8.1f us\n", profNamesArr[i], (t1 - t0) / 1000.0); + if (t0 < first) + first = t0; + if (t1 > last) + last = t1; + clReleaseEvent(profEventsArr[i]); + } + if (profEventCount > 0) + fprintf(stderr, " %-16s %8.1f us\n", "gpu span", (last - first) / 1000.0); + profEventCount = 0; +} + +static void oclDebugLog(const char *message) +{ + if (getenv("APRILTAG_OPENCL_DEBUG") != NULL) + fprintf(stderr, "apriltag opencl: %s\n", message); +} + +static void oclInit(void) +{ + cl_platform_id platforms[8]; + cl_uint platformCount = 0; + if (clGetPlatformIDs(8, platforms, &platformCount) != CL_SUCCESS || platformCount == 0) { + oclDebugLog("no OpenCL platforms"); + return; + } + + cl_device_id device = NULL; + for (cl_uint i = 0; i < platformCount && device == NULL; i++) { + cl_uint deviceCount = 0; + if (clGetDeviceIDs(platforms[i], CL_DEVICE_TYPE_GPU, 1, &device, &deviceCount) != CL_SUCCESS) + device = NULL; + } + if (device == NULL) { + oclDebugLog("no GPU device"); + return; + } + + cl_int err = CL_SUCCESS; + oclContext = clCreateContext(NULL, 1, &device, NULL, NULL, &err); + if (err != CL_SUCCESS) + return; + const cl_queue_properties queueProps[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_PROFILING_ENABLE, 0 }; + oclQueue = clCreateCommandQueueWithProperties(oclContext, device, queueProps, &err); + if (err != CL_SUCCESS) + return; + + const char *sources[6] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourcePartition, sourceScan }; + cl_program program = clCreateProgramWithSource(oclContext, 6, sources, NULL, &err); + if (err != CL_SUCCESS) + return; + err = clBuildProgram(program, 1, &device, "", NULL, NULL); + if (err != CL_SUCCESS) { + char log[8192] = { 0 }; + clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, sizeof(log) - 1, log, NULL); + oclDebugLog(log); + clReleaseProgram(program); + return; + } + + struct { cl_kernel *handle; const char *name; } kernels[] = { + { &oclKernelTileMinmax, "tileMinmax" }, + { &oclKernelTileBlur, "tileBlur" }, + { &oclKernelClassify, "classify" }, + { &oclKernelInitLabels, "initLabels" }, + { &oclKernelMergeEdges, "mergeEdges" }, + { &oclKernelCompressAndCount, "compressAndCount" }, + { &oclKernelBuildBigMap, "buildBigMap" }, + { &oclKernelExtractPairs, "extractPairs" }, + { &oclKernelHistKeys, "histKeys" }, + { &oclKernelScatterRecords, "scatterRecords" }, + { &oclKernelScanLocal, "scanLocal" }, + { &oclKernelScanBlocks, "scanBlocks" }, + { &oclKernelAddBlockOffsets, "addBlockOffsets" }, + }; + int failed = 0; + for (size_t i = 0; i < sizeof(kernels) / sizeof(kernels[0]); i++) { + *kernels[i].handle = clCreateKernel(program, kernels[i].name, &err); + if (err != CL_SUCCESS) + failed = 1; + } + clReleaseProgram(program); + if (failed) + return; + + oclReady = 1; +} + +static void releaseBuffer(cl_mem buffer) +{ + if (buffer != NULL) + clReleaseMemObject(buffer); +} + +static void releaseCache(void) +{ + releaseBuffer(cache.bufIm); + releaseBuffer(cache.bufOut); + releaseBuffer(cache.bufMaxRaw); + releaseBuffer(cache.bufMinRaw); + releaseBuffer(cache.bufMaxBlur); + releaseBuffer(cache.bufMinBlur); + releaseBuffer(cache.bufLabels); + releaseBuffer(cache.bufSizes); + releaseBuffer(cache.bufBigMap); + releaseBuffer(cache.bufCounter); + releaseBuffer(cache.bufRecords); + releaseBuffer(cache.bufPartitioned); + releaseBuffer(cache.bufHist); + releaseBuffer(cache.bufOffsets); + releaseBuffer(cache.bufBlockSums); + memset(&cache, 0, sizeof(cache)); +} + +static cl_mem createOrFail(cl_mem_flags flags, size_t bytes, void *host, int *failed) +{ + cl_int err = CL_SUCCESS; + cl_mem buffer = clCreateBuffer(oclContext, flags, bytes, host, &err); + if (err != CL_SUCCESS) + *failed = 1; + return buffer; +} + +static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) +{ + if (cache.valid && cache.w == w && cache.h == h && cache.s == s) + return 1; + releaseCache(); + + const size_t imageBytes = (size_t)s * (size_t)h; + const size_t tileBytes = (size_t)tw * (size_t)th; + const size_t pixelCount = (size_t)w * (size_t)h; + int failed = 0; + cache.bufIm = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, imageBytes, NULL, &failed); + cache.bufOut = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, imageBytes, NULL, &failed); + cache.bufMaxRaw = createOrFail(CL_MEM_READ_WRITE, tileBytes, NULL, &failed); + cache.bufMinRaw = createOrFail(CL_MEM_READ_WRITE, tileBytes, NULL, &failed); + cache.bufMaxBlur = createOrFail(CL_MEM_READ_WRITE, tileBytes, NULL, &failed); + cache.bufMinBlur = createOrFail(CL_MEM_READ_WRITE, tileBytes, NULL, &failed); + cache.bufLabels = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); + cache.bufSizes = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); + cache.bufBigMap = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); + cache.bufCounter = createOrFail(CL_MEM_READ_WRITE, 4, NULL, &failed); + cache.bufRecords = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); + cache.bufPartitioned = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); + cache.bufHist = createOrFail(CL_MEM_READ_WRITE, OCL_BIN_COUNT * 4, NULL, &failed); + cache.bufOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_BIN_COUNT * 4, NULL, &failed); + cache.bufBlockSums = createOrFail(CL_MEM_READ_WRITE, 256 * 4, NULL, &failed); + if (failed) { + releaseCache(); + return 0; + } + + // Stride padding bytes are never written by the threshold kernels; zero + // the output buffer once so padding matches the CPU path's calloc'd image. + const cl_uchar zero = 0; + if (clEnqueueFillBuffer(oclQueue, cache.bufOut, &zero, 1, 0, imageBytes, 0, NULL, NULL) != CL_SUCCESS) { + releaseCache(); + return 0; + } + + cache.valid = 1; + cache.w = w; + cache.h = h; + cache.s = s; + return 1; +} + +static size_t roundUp(size_t value, size_t multiple) +{ + return ((value + multiple - 1) / multiple) * multiple; +} + +static cl_int setThresholdArgs(cl_mem input, cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th, cl_int minWhiteBlackDiff) +{ + cl_int err = CL_SUCCESS; + err |= clSetKernelArg(oclKernelTileMinmax, 0, sizeof(cl_mem), &input); + err |= clSetKernelArg(oclKernelTileMinmax, 1, sizeof(cl_int), &s); + err |= clSetKernelArg(oclKernelTileMinmax, 2, sizeof(cl_int), &tw); + err |= clSetKernelArg(oclKernelTileMinmax, 3, sizeof(cl_mem), &cache.bufMaxRaw); + err |= clSetKernelArg(oclKernelTileMinmax, 4, sizeof(cl_mem), &cache.bufMinRaw); + err |= clSetKernelArg(oclKernelTileBlur, 0, sizeof(cl_mem), &cache.bufMaxRaw); + err |= clSetKernelArg(oclKernelTileBlur, 1, sizeof(cl_mem), &cache.bufMinRaw); + err |= clSetKernelArg(oclKernelTileBlur, 2, sizeof(cl_int), &tw); + err |= clSetKernelArg(oclKernelTileBlur, 3, sizeof(cl_int), &th); + err |= clSetKernelArg(oclKernelTileBlur, 4, sizeof(cl_mem), &cache.bufMaxBlur); + err |= clSetKernelArg(oclKernelTileBlur, 5, sizeof(cl_mem), &cache.bufMinBlur); + err |= clSetKernelArg(oclKernelClassify, 0, sizeof(cl_mem), &input); + err |= clSetKernelArg(oclKernelClassify, 1, sizeof(cl_int), &s); + err |= clSetKernelArg(oclKernelClassify, 2, sizeof(cl_int), &w); + err |= clSetKernelArg(oclKernelClassify, 3, sizeof(cl_int), &h); + err |= clSetKernelArg(oclKernelClassify, 4, sizeof(cl_int), &tw); + err |= clSetKernelArg(oclKernelClassify, 5, sizeof(cl_int), &th); + err |= clSetKernelArg(oclKernelClassify, 6, sizeof(cl_mem), &cache.bufMaxBlur); + err |= clSetKernelArg(oclKernelClassify, 7, sizeof(cl_mem), &cache.bufMinBlur); + err |= clSetKernelArg(oclKernelClassify, 8, sizeof(cl_int), &minWhiteBlackDiff); + err |= clSetKernelArg(oclKernelClassify, 9, sizeof(cl_mem), &cache.bufOut); + err |= clSetKernelArg(oclKernelClassify, 10, sizeof(cl_mem), &cache.bufLabels); + return err; +} + +image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im) +{ + if (getenv("APRILTAG_OPENCL") == NULL) + return NULL; + if (td->qtp.deglitch != 0) + return NULL; + + pthread_once(&oclInitOnce, oclInit); + if (oclReady == 0) + return NULL; + + const int tilesz = 4; + const cl_int w = im->width, h = im->height, s = im->stride; + const cl_int tw = w / tilesz, th = h / tilesz; + if (tw < 1 || th < 1) + return NULL; + const size_t imageBytes = (size_t)s * (size_t)h; + + image_u8_t *threshim = NULL; + int ok = 0; + + pthread_mutex_lock(&oclMutex); + if (!ensureCache(w, h, s, tw, th)) + goto done; + + cl_int err = CL_SUCCESS; + void *stagingIn = clEnqueueMapBuffer(oclQueue, cache.bufIm, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, imageBytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto done; + memcpy(stagingIn, im->buf, imageBytes); + err = clEnqueueUnmapMemObject(oclQueue, cache.bufIm, stagingIn, 0, NULL, NULL); + if (err != CL_SUCCESS) + goto done; + + err = setThresholdArgs(cache.bufIm, w, h, s, tw, th, td->qtp.min_white_black_diff); + if (err != CL_SUCCESS) + goto done; + + const size_t tileGlobal[2] = { (size_t)tw, (size_t)th }; + const size_t pixelGlobal[2] = { roundUp((size_t)w, 16), roundUp((size_t)h, 16) }; + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelTileMinmax, 2, NULL, tileGlobal, NULL, 0, NULL, NULL); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelTileBlur, 2, NULL, tileGlobal, NULL, 0, NULL, NULL); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelClassify, 2, NULL, pixelGlobal, NULL, 0, NULL, NULL); + if (err != CL_SUCCESS) + goto done; + + void *stagingOut = clEnqueueMapBuffer(oclQueue, cache.bufOut, CL_TRUE, CL_MAP_READ, 0, imageBytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto done; + threshim = image_u8_create_alignment(w, h, s); + if (threshim->stride == s) { + memcpy(threshim->buf, stagingOut, imageBytes); + cache.thresholdOutputFor = threshim->buf; + ok = 1; + } + clEnqueueUnmapMemObject(oclQueue, cache.bufOut, stagingOut, 0, NULL, NULL); + +done: + pthread_mutex_unlock(&oclMutex); + if (ok == 0) { + oclDebugLog("GPU threshold failed, falling back to CPU"); + if (threshim != NULL) + image_u8_destroy(threshim); + return NULL; + } + return threshim; +} + +static void appendPt(zarray_t *cluster, uint64_t packed) +{ + if (cluster->size == cluster->alloc) + zarray_ensure_capacity(cluster, cluster->size == 0 ? 16 : cluster->size * 2); + OclPt *dst = (OclPt *)(cluster->data + (size_t)cluster->size * cluster->el_sz); + dst->x = (uint16_t)(packed >> 48); + dst->y = (uint16_t)(packed >> 32); + dst->gx = (int16_t)(uint16_t)(packed >> 16); + dst->gy = (int16_t)(uint16_t)packed; + dst->slope = 0.0f; + cluster->size++; +} + +typedef struct { + uint64_t key; + zarray_t *cluster; +} PartnerSlot; + +// Canonical within-cluster ordering: reconstruct the CPU emitter's raster +// order (y, then x, then connectivity-check index) from the point fields. +// This makes GPU output deterministic run-to-run regardless of atomic emit +// order, and aligns marginal quad fits with the CPU implementation. +static uint64_t ptOrderKey(const OclPt *p) +{ + int conn; + if (p->gy == 0) + conn = 0; + else if (p->gx == 0) + conn = 1; + else if (p->gx == -p->gy) + conn = 2; + else + conn = 3; + int dx = (conn == 0 || conn == 3) ? 1 : (conn == 2 ? -1 : 0); + int dy = (conn == 0) ? 0 : 1; + uint64_t y = ((uint64_t)p->y - (uint64_t)dy) / 2; + uint64_t x = ((uint64_t)(p->x - dx)) / 2; + return (y << 18) | (x << 2) | (uint64_t)conn; +} + +static int comparePtOrder(const void *a, const void *b) +{ + uint64_t ka = ptOrderKey((const OclPt *)a); + uint64_t kb = ptOrderKey((const OclPt *)b); + return (ka > kb) - (ka < kb); +} + +typedef struct { + const uint64_t *records; + uint32_t binStart, binEnd; + zarray_t *clusters; +} BuildTask; + +static void doBuildTask(void *p) +{ + BuildTask *task = (BuildTask *)p; + const uint64_t *records = task->records; + int partnersCap = 256; + PartnerSlot *partners = malloc(sizeof(PartnerSlot) * partnersCap); + + for (uint32_t bin = task->binStart; bin < task->binEnd; bin++) { + uint32_t n = histHost[bin]; + if (n == 0) + continue; + uint32_t base = offsetsHost[bin]; + int partnerCount = 0; + for (uint32_t i = 0; i < n; i++) { + uint64_t key = records[2 * (base + i)]; + uint64_t payload = records[2 * (base + i) + 1]; + int slot = -1; + for (int j = partnerCount - 1; j >= 0; j--) { + if (partners[j].key == key) { + slot = j; + break; + } + } + if (slot < 0) { + if (partnerCount == partnersCap) { + partnersCap *= 2; + partners = realloc(partners, sizeof(PartnerSlot) * partnersCap); + } + partners[partnerCount].key = key; + partners[partnerCount].cluster = zarray_create(sizeof(OclPt)); + slot = partnerCount++; + } + appendPt(partners[slot].cluster, payload); + } + for (int j = 0; j < partnerCount; j++) { + zarray_t *cluster = partners[j].cluster; + if (oclExactOrder) + qsort(cluster->data, cluster->size, cluster->el_sz, comparePtOrder); + zarray_add(task->clusters, &cluster); + } + } + free(partners); +} + +static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount) +{ + int taskCount = (td->wp != NULL && td->nthreads > 1) ? td->nthreads : 1; + if (taskCount > 16) + taskCount = 16; + BuildTask tasks[16]; + + // Split bins into ranges balanced by record count so workers finish together. + uint32_t targetPerTask = recordCount / (uint32_t)taskCount + 1; + uint32_t bin = 0; + for (int t = 0; t < taskCount; t++) { + tasks[t].records = records; + tasks[t].binStart = bin; + tasks[t].clusters = zarray_create(sizeof(zarray_t *)); + uint32_t taken = 0; + while (bin < OCL_BIN_COUNT && (taken < targetPerTask || t == taskCount - 1)) { + taken += histHost[bin]; + bin++; + } + tasks[t].binEnd = bin; + } + tasks[taskCount - 1].binEnd = OCL_BIN_COUNT; + + if (taskCount == 1) { + doBuildTask(&tasks[0]); + } else { + for (int t = 0; t < taskCount; t++) + workerpool_add_task(td->wp, doBuildTask, &tasks[t]); + workerpool_run(td->wp); + } + + zarray_t *clusters = zarray_create(sizeof(zarray_t *)); + for (int t = 0; t < taskCount; t++) { + for (int i = 0; i < zarray_size(tasks[t].clusters); i++) { + zarray_t *cluster; + zarray_get(tasks[t].clusters, i, &cluster); + zarray_add(clusters, &cluster); + } + zarray_destroy(tasks[t].clusters); + } + return clusters; +} + +// Runs CCL + sizes + extraction + partition over the threshold image already +// in inputBuffer, then builds the cluster arrays on the CPU. Caller holds +// oclMutex and has a valid cache. labelsReady indicates the classify kernel +// already seeded the labels buffer. +static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl_int cw, cl_int ch, cl_int cs, int labelsReady) +{ + oclExactOrder = getenv("APRILTAG_OPENCL_EXACT") != NULL; + const cl_uint minCluster = (cl_uint)td->qtp.min_cluster_pixels; + const cl_uint capacity = OCL_RECORD_CAPACITY; + zarray_t *clusters = NULL; + cl_int err = CL_SUCCESS; + const cl_uint zero = 0; + const size_t pixelCount = (size_t)cw * (size_t)ch; + + err |= clEnqueueFillBuffer(oclQueue, cache.bufSizes, &zero, 4, 0, pixelCount * 4, 0, NULL, profSlot("fillSizes")); + err |= clEnqueueFillBuffer(oclQueue, cache.bufCounter, &zero, 4, 0, 4, 0, NULL, NULL); + err |= clEnqueueFillBuffer(oclQueue, cache.bufHist, &zero, 4, 0, OCL_BIN_COUNT * 4, 0, NULL, NULL); + if (err != CL_SUCCESS) + goto done; + + cl_kernel layoutKernels[4] = { oclKernelInitLabels, oclKernelMergeEdges, oclKernelCompressAndCount, oclKernelBuildBigMap }; + for (int k = 0; k < 4; k++) { + err |= clSetKernelArg(layoutKernels[k], 0, sizeof(cl_mem), &inputBuffer); + err |= clSetKernelArg(layoutKernels[k], 1, sizeof(cl_int), &cs); + err |= clSetKernelArg(layoutKernels[k], 2, sizeof(cl_int), &cw); + err |= clSetKernelArg(layoutKernels[k], 3, sizeof(cl_int), &ch); + err |= clSetKernelArg(layoutKernels[k], 4, sizeof(cl_mem), &cache.bufLabels); + } + err |= clSetKernelArg(oclKernelCompressAndCount, 5, sizeof(cl_mem), &cache.bufSizes); + err |= clSetKernelArg(oclKernelBuildBigMap, 5, sizeof(cl_mem), &cache.bufSizes); + err |= clSetKernelArg(oclKernelBuildBigMap, 6, sizeof(cl_uint), &minCluster); + err |= clSetKernelArg(oclKernelBuildBigMap, 7, sizeof(cl_mem), &cache.bufBigMap); + err |= clSetKernelArg(oclKernelExtractPairs, 0, sizeof(cl_mem), &inputBuffer); + err |= clSetKernelArg(oclKernelExtractPairs, 1, sizeof(cl_int), &cs); + err |= clSetKernelArg(oclKernelExtractPairs, 2, sizeof(cl_int), &cw); + err |= clSetKernelArg(oclKernelExtractPairs, 3, sizeof(cl_int), &ch); + err |= clSetKernelArg(oclKernelExtractPairs, 4, sizeof(cl_mem), &cache.bufLabels); + err |= clSetKernelArg(oclKernelExtractPairs, 5, sizeof(cl_mem), &cache.bufBigMap); + err |= clSetKernelArg(oclKernelExtractPairs, 6, sizeof(cl_mem), &cache.bufCounter); + err |= clSetKernelArg(oclKernelExtractPairs, 7, sizeof(cl_mem), &cache.bufRecords); + err |= clSetKernelArg(oclKernelExtractPairs, 8, sizeof(cl_uint), &capacity); + if (err != CL_SUCCESS) + goto done; + + const size_t global[2] = { roundUp((size_t)cw, 16), roundUp((size_t)ch, 16) }; + const size_t countGlobal[2] = { roundUp((size_t)cw, 128), (size_t)ch }; + const size_t countLocal[2] = { 128, 1 }; + if (!labelsReady) { + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelInitLabels, 2, NULL, global, NULL, 0, NULL, profSlot("initLabels")); + } + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelMergeEdges, 2, NULL, global, NULL, 0, NULL, profSlot("mergeEdges")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCompressAndCount, 2, NULL, countGlobal, countLocal, 0, NULL, profSlot("compressCount")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelBuildBigMap, 2, NULL, global, NULL, 0, NULL, profSlot("buildBigMap")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelExtractPairs, 2, NULL, global, NULL, 0, NULL, profSlot("extractPairs")); + if (err != CL_SUCCESS) + goto done; + + err |= clSetKernelArg(oclKernelHistKeys, 0, sizeof(cl_mem), &cache.bufRecords); + err |= clSetKernelArg(oclKernelHistKeys, 1, sizeof(cl_mem), &cache.bufCounter); + err |= clSetKernelArg(oclKernelHistKeys, 2, sizeof(cl_uint), &capacity); + err |= clSetKernelArg(oclKernelHistKeys, 3, sizeof(cl_mem), &cache.bufHist); + err |= clSetKernelArg(oclKernelScanLocal, 0, sizeof(cl_mem), &cache.bufHist); + err |= clSetKernelArg(oclKernelScanLocal, 1, sizeof(cl_mem), &cache.bufOffsets); + err |= clSetKernelArg(oclKernelScanLocal, 2, sizeof(cl_mem), &cache.bufBlockSums); + err |= clSetKernelArg(oclKernelScanBlocks, 0, sizeof(cl_mem), &cache.bufBlockSums); + err |= clSetKernelArg(oclKernelAddBlockOffsets, 0, sizeof(cl_mem), &cache.bufOffsets); + err |= clSetKernelArg(oclKernelAddBlockOffsets, 1, sizeof(cl_mem), &cache.bufBlockSums); + err |= clSetKernelArg(oclKernelScatterRecords, 0, sizeof(cl_mem), &cache.bufRecords); + err |= clSetKernelArg(oclKernelScatterRecords, 1, sizeof(cl_mem), &cache.bufCounter); + err |= clSetKernelArg(oclKernelScatterRecords, 2, sizeof(cl_uint), &capacity); + err |= clSetKernelArg(oclKernelScatterRecords, 3, sizeof(cl_mem), &cache.bufOffsets); + err |= clSetKernelArg(oclKernelScatterRecords, 4, sizeof(cl_mem), &cache.bufPartitioned); + if (err != CL_SUCCESS) + goto done; + + const size_t capacityGlobal[1] = { (size_t)capacity }; + const size_t scanGlobal[1] = { OCL_BIN_COUNT }; + const size_t scanLocalSize[1] = { 256 }; + const size_t singleItem[1] = { 1 }; + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelHistKeys, 1, NULL, capacityGlobal, NULL, 0, NULL, profSlot("histKeys")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanLocal, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("scanLocal")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanBlocks, 1, NULL, singleItem, NULL, 0, NULL, profSlot("scanBlocks")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelAddBlockOffsets, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("addBlockOffs")); + // Read pre-scatter offsets, counts, and the record counter for the host + // build walk; the in-order queue places these before the scatter mutates + // the offsets, and none of them stall the host. + cl_event counterEvent = NULL; + err |= clEnqueueReadBuffer(oclQueue, cache.bufCounter, CL_FALSE, 0, 4, &counterHost, 0, NULL, &counterEvent); + err |= clEnqueueReadBuffer(oclQueue, cache.bufOffsets, CL_FALSE, 0, OCL_BIN_COUNT * 4, offsetsHost, 0, NULL, profSlot("readOffsets")); + err |= clEnqueueReadBuffer(oclQueue, cache.bufHist, CL_FALSE, 0, OCL_BIN_COUNT * 4, histHost, 0, NULL, profSlot("readHist")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScatterRecords, 1, NULL, capacityGlobal, NULL, 0, NULL, profSlot("scatter")); + if (err != CL_SUCCESS) { + if (counterEvent != NULL) + clReleaseEvent(counterEvent); + goto done; + } + + // The counter read completes mid-chain; waiting on it costs nothing + // extra (the map below blocks on the whole chain anyway) and lets us map + // only the live records instead of the full capacity buffer. + clWaitForEvents(1, &counterEvent); + clReleaseEvent(counterEvent); + if (counterHost > capacity) { + oclDebugLog("record capacity exceeded"); + goto done; + } + if (counterHost == 0) { + clusters = zarray_create(sizeof(zarray_t *)); + goto done; + } + + void *mapped = clEnqueueMapBuffer(oclQueue, cache.bufPartitioned, CL_TRUE, CL_MAP_READ, 0, + (size_t)counterHost * 16, 0, NULL, profSlot("mapRecords"), &err); + if (err != CL_SUCCESS) + goto done; + clusters = buildClusters(td, (const uint64_t *)mapped, counterHost); + clEnqueueUnmapMemObject(oclQueue, cache.bufPartitioned, mapped, 0, NULL, NULL); + +done: + return clusters; +} + +zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts) +{ + if (getenv("APRILTAG_OPENCL") == NULL) + return NULL; + if (td->debug != 0) + return NULL; + + pthread_once(&oclInitOnce, oclInit); + if (oclReady == 0) + return NULL; + if ((int64_t)w * (int64_t)h >= ((int64_t)1 << 23)) + return NULL; + if (w / 4 < 1 || h / 4 < 1) + return NULL; + + const size_t imageBytes = (size_t)ts * (size_t)h; + zarray_t *clusters = NULL; + + pthread_mutex_lock(&oclMutex); + profReset(); + if (!ensureCache(w, h, ts, w / 4, h / 4)) + goto done; + + // When the GPU threshold just produced this exact image, its device copy + // is still in bufOut — consume the tag and skip the re-upload. + { + cl_mem inputBuffer = cache.bufIm; + int labelsReady = 0; + if (cache.thresholdOutputFor == (const void *)threshim->buf) { + inputBuffer = cache.bufOut; + labelsReady = 1; + } else { + cl_int err = CL_SUCCESS; + void *stagingIn = clEnqueueMapBuffer(oclQueue, cache.bufIm, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, imageBytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto done; + memcpy(stagingIn, threshim->buf, imageBytes); + if (clEnqueueUnmapMemObject(oclQueue, cache.bufIm, stagingIn, 0, NULL, NULL) != CL_SUCCESS) + goto done; + } + cache.thresholdOutputFor = NULL; + clusters = runClusterChain(td, inputBuffer, w, h, ts, labelsReady); + } + +done: + profPrint(); + pthread_mutex_unlock(&oclMutex); + if (clusters == NULL) + oclDebugLog("GPU clusters failed, falling back to CPU"); + return clusters; +} + +zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im) +{ + if (getenv("APRILTAG_OPENCL") == NULL) + return NULL; + if (td->debug != 0) + return NULL; + if (td->qtp.deglitch != 0) + return NULL; + + pthread_once(&oclInitOnce, oclInit); + if (oclReady == 0) + return NULL; + + const int tilesz = 4; + const cl_int w = im->width, h = im->height, s = im->stride; + const cl_int tw = w / tilesz, th = h / tilesz; + if (tw < 1 || th < 1) + return NULL; + if ((int64_t)w * (int64_t)h >= ((int64_t)1 << 23)) + return NULL; + const size_t imageBytes = (size_t)s * (size_t)h; + + zarray_t *clusters = NULL; + pthread_mutex_lock(&oclMutex); + profReset(); + if (!ensureCache(w, h, s, tw, th)) + goto done; + cache.thresholdOutputFor = NULL; + + cl_int err = CL_SUCCESS; + void *stagingIn = clEnqueueMapBuffer(oclQueue, cache.bufIm, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, imageBytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto done; + memcpy(stagingIn, im->buf, imageBytes); + err = clEnqueueUnmapMemObject(oclQueue, cache.bufIm, stagingIn, 0, NULL, NULL); + if (err != CL_SUCCESS) + goto done; + + err = setThresholdArgs(cache.bufIm, w, h, s, tw, th, td->qtp.min_white_black_diff); + if (err != CL_SUCCESS) + goto done; + + const size_t tileGlobal[2] = { (size_t)tw, (size_t)th }; + const size_t pixelGlobal[2] = { roundUp((size_t)w, 16), roundUp((size_t)h, 16) }; + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelTileMinmax, 2, NULL, tileGlobal, NULL, 0, NULL, profSlot("tileMinmax")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelTileBlur, 2, NULL, tileGlobal, NULL, 0, NULL, profSlot("tileBlur")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelClassify, 2, NULL, pixelGlobal, NULL, 0, NULL, profSlot("classify")); + if (err != CL_SUCCESS) + goto done; + + clusters = runClusterChain(td, cache.bufOut, w, h, s, 1); + +done: + profPrint(); + pthread_mutex_unlock(&oclMutex); + if (clusters == NULL) + oclDebugLog("GPU frontend failed, falling back to CPU"); + return clusters; +} diff --git a/ocl_threshold.h b/ocl_threshold.h new file mode 100644 index 00000000..2be7f139 --- /dev/null +++ b/ocl_threshold.h @@ -0,0 +1,24 @@ +#pragma once + +#include "apriltag.h" +#include "common/image_u8.h" +#include "common/zarray.h" + +// GPU implementation of the adaptive tile threshold stage. Returns NULL +// whenever the GPU path is unavailable or disabled (APRILTAG_OPENCL unset), +// in which case the caller must run the CPU implementation. A non-NULL +// result is byte-identical to the CPU implementation's output. +image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im); + +// GPU implementation of connected components + gradient clustering over a +// threshold image. Returns a zarray of zarray-of-struct-pt clusters whose +// content matches the CPU implementation (cluster and point order may +// differ), or NULL when the GPU path is unavailable, in which case the +// caller must run connected_components + gradient_clusters on the CPU. +zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts); + +// Full GPU frontend over the (already decimated/blurred) input image: +// threshold, connected components, and gradient clustering in one device +// chain — the threshold image never materializes on the host. Same return +// contract as oclClusters. +zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im); From cbabba69a9fc43b707a1b23026901722ea0930ab Mon Sep 17 00:00:00 2001 From: James McVay Date: Wed, 10 Jun 2026 23:29:07 +0200 Subject: [PATCH 02/18] Emit boundary records in raster order: bit-exact output by default Replace the atomic extract + histogram/scatter partition with a two-pass per-row-segment emission (count, scan, sequential emit), so boundary records land in the CPU emitter's exact raster order. The CPU build walk becomes ordered hashmap grouping, reproducing the CPU path's cluster content and within-cluster point order without the validation-only sort the previous design needed. Detector output is now bit-identical to the CPU implementation and deterministic run-to-run by default (corpus 108/108 cases, max corner delta 0.0 px); APRILTAG_OPENCL_EXACT is gone. Also deletes the 128 MB partition buffer and two kernels. Co-Authored-By: Claude Fable 5 --- ocl_threshold.c | 485 ++++++++++++++++++++++++++---------------------- ocl_threshold.h | 9 +- 2 files changed, 265 insertions(+), 229 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index b17e42c5..a74334da 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -12,9 +12,11 @@ // GPU implementation of the detector frontend: adaptive tile threshold, // connected components, component sizes, and boundary-pair extraction, // replicating the CPU implementations' exact semantics (see -// apriltag_quad_thresh.c). Cluster grouping uses a single GPU partition pass -// over the high bits of the component-pair key; final grouping happens on -// the CPU during the cluster build it must perform anyway. +// apriltag_quad_thresh.c). Boundary records are emitted in the CPU +// emitter's raster order (per 256-pixel row segment: count, scan, then +// sequential emit), so the CPU-side grouping walk reproduces the CPU +// path's cluster content and within-cluster point order exactly — output +// is bit-identical to the CPU implementation. // // All entry points return NULL when the GPU path is disabled (APRILTAG_OPENCL // unset) or unavailable, in which case callers run the CPU implementation. @@ -150,7 +152,7 @@ static const char *sourceCompress = " while ((uint)lid + run < lsz && roots[lid + run] == root) run++;\n" " atomic_add(&sizes[root], run);\n" "}\n" - // One coalesced byte per pixel replaces the extract kernel's repeated + // One coalesced byte per pixel replaces the extract logic's repeated // scattered label+size lookups. "__kernel void buildBigMap(__global const uchar *im, int s, int w, int h,\n" " __global const uint *labels, __global const uint *sizes,\n" @@ -173,9 +175,20 @@ static const char *sourceExtract = " if ((int)v0 + (int)v1 != 255) return 0;\n" " return bigMap[(y + dy) * w + x + dx] != 0;\n" "}\n" - "inline void emitPair(__global const uchar *im, __global const uint *labels,\n" - " int s, int w, int x, int y, int dx, int dy,\n" - " __global volatile uint *counter, __global ulong2 *records, uint capacity) {\n" + // Mask bits follow the CPU's DO_CONN emit order: (1,0), (0,1), (-1,1), (1,1). + "inline int emitMask(__global const uchar *im, __global const uchar *bigMap,\n" + " int s, int w, int x, int y) {\n" + " if (bigMap[y * w + x] == 0) return 0;\n" + " int mask = 0;\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 1, 0)) mask |= 1;\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 0, 1)) mask |= 2;\n" + " int prevEmitted = (x > 1) && wouldEmit(im, bigMap, s, w, x - 1, y, 1, 1);\n" + " if (!prevEmitted && wouldEmit(im, bigMap, s, w, x, y, -1, 1)) mask |= 4;\n" + " if (wouldEmit(im, bigMap, s, w, x, y, 1, 1)) mask |= 8;\n" + " return mask;\n" + "}\n" + "inline ulong2 makeRecord(__global const uchar *im, __global const uint *labels,\n" + " int s, int w, int x, int y, int dx, int dy) {\n" " uchar v0 = im[y * s + x];\n" " uchar v1 = im[(y + dy) * s + x + dx];\n" " uint rep0 = labels[y * w + x];\n" @@ -185,49 +198,46 @@ static const char *sourceExtract = " ushort px = (ushort)(2 * x + dx), py = (ushort)(2 * y + dy);\n" " ushort pgx = (ushort)(short)(dx * grad), pgy = (ushort)(short)(dy * grad);\n" " ulong packed = ((ulong)px << 48) | ((ulong)py << 32) | ((ulong)pgx << 16) | (ulong)pgy;\n" - " uint slot = atomic_inc(counter);\n" - " if (slot < capacity) records[slot] = (ulong2)(key, packed);\n" - "}\n" - "__kernel void extractPairs(__global const uchar *im, int s, int w, int h,\n" - " __global const uint *labels, __global const uchar *bigMap,\n" - " __global volatile uint *counter,\n" - " __global ulong2 *records, uint capacity) {\n" - " int x = get_global_id(0), y = get_global_id(1);\n" - " if (x < 1 || x >= w - 1 || y < 1 || y >= h - 1) return;\n" - " if (bigMap[y * w + x] == 0) return;\n" - " if (wouldEmit(im, bigMap, s, w, x, y, 1, 0))\n" - " emitPair(im, labels, s, w, x, y, 1, 0, counter, records, capacity);\n" - " if (wouldEmit(im, bigMap, s, w, x, y, 0, 1))\n" - " emitPair(im, labels, s, w, x, y, 0, 1, counter, records, capacity);\n" - " int prevEmitted = (x > 1) && wouldEmit(im, bigMap, s, w, x - 1, y, 1, 1);\n" - " if (!prevEmitted && wouldEmit(im, bigMap, s, w, x, y, -1, 1))\n" - " emitPair(im, labels, s, w, x, y, -1, 1, counter, records, capacity);\n" - " if (wouldEmit(im, bigMap, s, w, x, y, 1, 1))\n" - " emitPair(im, labels, s, w, x, y, 1, 1, counter, records, capacity);\n" + " return (ulong2)(key, packed);\n" "}\n"; -static const char *sourcePartition = - // Both kernels read the live record count from the device so the host - // never has to stall mid-chain; launched over the full capacity. - "__kernel void histKeys(__global const ulong2 *records, __global const uint *counter,\n" - " uint capacity, __global volatile uint *hist) {\n" - " __local uint localCount;\n" - " if (get_local_id(0) == 0) localCount = min(counter[0], capacity);\n" - " barrier(CLK_LOCAL_MEM_FENCE);\n" - " uint i = get_global_id(0);\n" - " if (i >= localCount) return;\n" - " atomic_inc(&hist[(uint)((records[i].x >> 39) & 0xFFFFul)]);\n" +static const char *sourceEmit = + "__kernel void countSegments(__global const uchar *im, __global const uchar *bigMap,\n" + " int s, int w, int h, int segsPerRow, int nSegs,\n" + " __global uint *segCounts) {\n" + " int seg = get_global_id(0);\n" + " if (seg >= nSegs) return;\n" + " int y = seg / segsPerRow;\n" + " int x0 = (seg % segsPerRow) * 256;\n" + " int x1 = min(x0 + 256, w - 1);\n" + " if (x0 < 1) x0 = 1;\n" + " uint count = 0;\n" + " if (y >= 1 && y < h - 1) {\n" + " for (int x = x0; x < x1; x++)\n" + " count += (uint)popcount(emitMask(im, bigMap, s, w, x, y));\n" + " }\n" + " segCounts[seg] = count;\n" "}\n" - "__kernel void scatterRecords(__global const ulong2 *records, __global const uint *counter,\n" - " uint capacity, __global volatile uint *offsets,\n" - " __global ulong2 *out) {\n" - " __local uint localCount;\n" - " if (get_local_id(0) == 0) localCount = min(counter[0], capacity);\n" - " barrier(CLK_LOCAL_MEM_FENCE);\n" - " uint i = get_global_id(0);\n" - " if (i >= localCount) return;\n" - " ulong2 r = records[i];\n" - " out[atomic_inc(&offsets[(uint)((r.x >> 39) & 0xFFFFul)])] = r;\n" + "__kernel void emitSegments(__global const uchar *im, __global const uchar *bigMap,\n" + " __global const uint *labels, int s, int w, int h,\n" + " int segsPerRow, int nSegs, __global const uint *segOffsets,\n" + " uint capacity, __global ulong2 *records) {\n" + " int seg = get_global_id(0);\n" + " if (seg >= nSegs) return;\n" + " int y = seg / segsPerRow;\n" + " if (y < 1 || y >= h - 1) return;\n" + " int x0 = (seg % segsPerRow) * 256;\n" + " int x1 = min(x0 + 256, w - 1);\n" + " if (x0 < 1) x0 = 1;\n" + " uint slot = segOffsets[seg];\n" + " for (int x = x0; x < x1; x++) {\n" + " int mask = emitMask(im, bigMap, s, w, x, y);\n" + " if (mask == 0) continue;\n" + " if (mask & 1) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 0); slot++; }\n" + " if (mask & 2) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 0, 1); slot++; }\n" + " if (mask & 4) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, -1, 1); slot++; }\n" + " if (mask & 8) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 1); slot++; }\n" + " }\n" "}\n"; static const char *sourceScan = @@ -260,7 +270,8 @@ static const char *sourceScan = "}\n"; #define OCL_RECORD_CAPACITY (8u * 1024u * 1024u) -#define OCL_BIN_COUNT 65536u +#define OCL_SEG_COUNT 65536u +#define OCL_SEG_WIDTH 256 typedef struct { uint16_t x, y; @@ -280,9 +291,8 @@ static cl_kernel oclKernelInitLabels; static cl_kernel oclKernelMergeEdges; static cl_kernel oclKernelCompressAndCount; static cl_kernel oclKernelBuildBigMap; -static cl_kernel oclKernelExtractPairs; -static cl_kernel oclKernelHistKeys; -static cl_kernel oclKernelScatterRecords; +static cl_kernel oclKernelCountSegments; +static cl_kernel oclKernelEmitSegments; static cl_kernel oclKernelScanLocal; static cl_kernel oclKernelScanBlocks; static cl_kernel oclKernelAddBlockOffsets; @@ -300,25 +310,15 @@ typedef struct { cl_mem bufLabels; cl_mem bufSizes; cl_mem bufBigMap; - cl_mem bufCounter; cl_mem bufRecords; - cl_mem bufPartitioned; - cl_mem bufHist; - cl_mem bufOffsets; + cl_mem bufSegCounts; + cl_mem bufSegOffsets; cl_mem bufBlockSums; } OclBufferCache; static OclBufferCache cache; -static uint32_t histHost[OCL_BIN_COUNT]; -static uint32_t offsetsHost[OCL_BIN_COUNT]; -static uint32_t counterHost; - -// APRILTAG_OPENCL_EXACT=1 sorts cluster points into the CPU emitter's exact -// order, making output bit-identical to the CPU path (validation mode). It -// currently costs more CPU than it saves; production runs without it, where -// output is content-equivalent and corners may differ at the 0.02 px level. -// TODO: emit records row-ordered on the GPU to get exactness for free. -static int oclExactOrder; +static uint32_t segCountsHost[OCL_SEG_COUNT]; +static uint32_t segOffsetsHost[OCL_SEG_COUNT]; static cl_event profEventsArr[32]; static const char *profNamesArr[32]; @@ -395,7 +395,7 @@ static void oclInit(void) if (err != CL_SUCCESS) return; - const char *sources[6] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourcePartition, sourceScan }; + const char *sources[6] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourceEmit, sourceScan }; cl_program program = clCreateProgramWithSource(oclContext, 6, sources, NULL, &err); if (err != CL_SUCCESS) return; @@ -416,9 +416,8 @@ static void oclInit(void) { &oclKernelMergeEdges, "mergeEdges" }, { &oclKernelCompressAndCount, "compressAndCount" }, { &oclKernelBuildBigMap, "buildBigMap" }, - { &oclKernelExtractPairs, "extractPairs" }, - { &oclKernelHistKeys, "histKeys" }, - { &oclKernelScatterRecords, "scatterRecords" }, + { &oclKernelCountSegments, "countSegments" }, + { &oclKernelEmitSegments, "emitSegments" }, { &oclKernelScanLocal, "scanLocal" }, { &oclKernelScanBlocks, "scanBlocks" }, { &oclKernelAddBlockOffsets, "addBlockOffsets" }, @@ -453,11 +452,9 @@ static void releaseCache(void) releaseBuffer(cache.bufLabels); releaseBuffer(cache.bufSizes); releaseBuffer(cache.bufBigMap); - releaseBuffer(cache.bufCounter); releaseBuffer(cache.bufRecords); - releaseBuffer(cache.bufPartitioned); - releaseBuffer(cache.bufHist); - releaseBuffer(cache.bufOffsets); + releaseBuffer(cache.bufSegCounts); + releaseBuffer(cache.bufSegOffsets); releaseBuffer(cache.bufBlockSums); memset(&cache, 0, sizeof(cache)); } @@ -490,11 +487,9 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) cache.bufLabels = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); cache.bufSizes = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); cache.bufBigMap = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); - cache.bufCounter = createOrFail(CL_MEM_READ_WRITE, 4, NULL, &failed); - cache.bufRecords = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); - cache.bufPartitioned = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); - cache.bufHist = createOrFail(CL_MEM_READ_WRITE, OCL_BIN_COUNT * 4, NULL, &failed); - cache.bufOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_BIN_COUNT * 4, NULL, &failed); + cache.bufRecords = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); + cache.bufSegCounts = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); + cache.bufSegOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufBlockSums = createOrFail(CL_MEM_READ_WRITE, 256 * 4, NULL, &failed); if (failed) { releaseCache(); @@ -631,112 +626,152 @@ static void appendPt(zarray_t *cluster, uint64_t packed) cluster->size++; } +// Open-addressing map from cluster key to cluster index. Keys are never 0 +// (the high half is always the larger of two distinct roots), so 0 marks an +// empty slot. Records arrive in raster order, so appending in encounter +// order reproduces the CPU emitter's within-cluster point order. +#define OCL_HASH_BITS 16 +#define OCL_HASH_SIZE (1u << OCL_HASH_BITS) + typedef struct { uint64_t key; - zarray_t *cluster; -} PartnerSlot; - -// Canonical within-cluster ordering: reconstruct the CPU emitter's raster -// order (y, then x, then connectivity-check index) from the point fields. -// This makes GPU output deterministic run-to-run regardless of atomic emit -// order, and aligns marginal quad fits with the CPU implementation. -static uint64_t ptOrderKey(const OclPt *p) -{ - int conn; - if (p->gy == 0) - conn = 0; - else if (p->gx == 0) - conn = 1; - else if (p->gx == -p->gy) - conn = 2; - else - conn = 3; - int dx = (conn == 0 || conn == 3) ? 1 : (conn == 2 ? -1 : 0); - int dy = (conn == 0) ? 0 : 1; - uint64_t y = ((uint64_t)p->y - (uint64_t)dy) / 2; - uint64_t x = ((uint64_t)(p->x - dx)) / 2; - return (y << 18) | (x << 2) | (uint64_t)conn; -} - -static int comparePtOrder(const void *a, const void *b) -{ - uint64_t ka = ptOrderKey((const OclPt *)a); - uint64_t kb = ptOrderKey((const OclPt *)b); - return (ka > kb) - (ka < kb); -} + uint32_t clusterIdx; +} HashEntry; typedef struct { const uint64_t *records; - uint32_t binStart, binEnd; + uint32_t recStart, recEnd; zarray_t *clusters; + uint64_t *clusterKeys; + int clusterCap; + int failed; } BuildTask; +static uint32_t hashSlot(uint64_t key) +{ + return (uint32_t)((key * 0x9E3779B97F4A7C15ull) >> (64 - OCL_HASH_BITS)); +} + static void doBuildTask(void *p) { BuildTask *task = (BuildTask *)p; - const uint64_t *records = task->records; - int partnersCap = 256; - PartnerSlot *partners = malloc(sizeof(PartnerSlot) * partnersCap); - - for (uint32_t bin = task->binStart; bin < task->binEnd; bin++) { - uint32_t n = histHost[bin]; - if (n == 0) - continue; - uint32_t base = offsetsHost[bin]; - int partnerCount = 0; - for (uint32_t i = 0; i < n; i++) { - uint64_t key = records[2 * (base + i)]; - uint64_t payload = records[2 * (base + i) + 1]; - int slot = -1; - for (int j = partnerCount - 1; j >= 0; j--) { - if (partners[j].key == key) { - slot = j; - break; - } + HashEntry *table = calloc(OCL_HASH_SIZE, sizeof(HashEntry)); + if (table == NULL) { + task->failed = 1; + return; + } + int clusterCount = 0; + + for (uint32_t i = task->recStart; i < task->recEnd; i++) { + uint64_t key = task->records[2 * i]; + uint64_t payload = task->records[2 * i + 1]; + uint32_t slot = hashSlot(key); + while (table[slot].key != 0 && table[slot].key != key) + slot = (slot + 1) & (OCL_HASH_SIZE - 1); + if (table[slot].key == 0) { + if (clusterCount >= (int)(OCL_HASH_SIZE / 2)) { + task->failed = 1; + break; } - if (slot < 0) { - if (partnerCount == partnersCap) { - partnersCap *= 2; - partners = realloc(partners, sizeof(PartnerSlot) * partnersCap); - } - partners[partnerCount].key = key; - partners[partnerCount].cluster = zarray_create(sizeof(OclPt)); - slot = partnerCount++; + if (clusterCount == task->clusterCap) { + task->clusterCap *= 2; + task->clusterKeys = realloc(task->clusterKeys, sizeof(uint64_t) * task->clusterCap); } - appendPt(partners[slot].cluster, payload); - } - for (int j = 0; j < partnerCount; j++) { - zarray_t *cluster = partners[j].cluster; - if (oclExactOrder) - qsort(cluster->data, cluster->size, cluster->el_sz, comparePtOrder); + table[slot].key = key; + table[slot].clusterIdx = (uint32_t)clusterCount; + zarray_t *cluster = zarray_create(sizeof(OclPt)); zarray_add(task->clusters, &cluster); + task->clusterKeys[clusterCount] = key; + clusterCount++; } + zarray_t *cluster; + zarray_get(task->clusters, (int)table[slot].clusterIdx, &cluster); + appendPt(cluster, payload); } - free(partners); + free(table); } -static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount) +static void destroyTaskClusters(BuildTask *task) +{ + for (int i = 0; i < zarray_size(task->clusters); i++) { + zarray_t *cluster; + zarray_get(task->clusters, i, &cluster); + zarray_destroy(cluster); + } + zarray_destroy(task->clusters); + free(task->clusterKeys); +} + +// Merge per-task clusters in task order: tasks cover ascending row ranges, +// so concatenation preserves raster point order within each cluster. +static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount) +{ + zarray_t *clusters = zarray_create(sizeof(zarray_t *)); + HashEntry *table = calloc(OCL_HASH_SIZE, sizeof(HashEntry)); + if (table == NULL) { + for (int t = 0; t < taskCount; t++) + destroyTaskClusters(&tasks[t]); + return clusters; + } + + for (int t = 0; t < taskCount; t++) { + for (int i = 0; i < zarray_size(tasks[t].clusters); i++) { + zarray_t *cluster; + zarray_get(tasks[t].clusters, i, &cluster); + uint64_t key = tasks[t].clusterKeys[i]; + uint32_t slot = hashSlot(key); + while (table[slot].key != 0 && table[slot].key != key) + slot = (slot + 1) & (OCL_HASH_SIZE - 1); + if (table[slot].key == 0) { + table[slot].key = key; + table[slot].clusterIdx = (uint32_t)zarray_size(clusters); + zarray_add(clusters, &cluster); + } else { + zarray_t *dst; + zarray_get(clusters, (int)table[slot].clusterIdx, &dst); + zarray_ensure_capacity(dst, dst->size + cluster->size); + memcpy(dst->data + (size_t)dst->size * dst->el_sz, cluster->data, + (size_t)cluster->size * cluster->el_sz); + dst->size += cluster->size; + zarray_destroy(cluster); + } + } + zarray_destroy(tasks[t].clusters); + free(tasks[t].clusterKeys); + } + free(table); + return clusters; +} + +static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount, + int segsPerRow, cl_int h) { int taskCount = (td->wp != NULL && td->nthreads > 1) ? td->nthreads : 1; if (taskCount > 16) taskCount = 16; BuildTask tasks[16]; - // Split bins into ranges balanced by record count so workers finish together. + // Split rows into contiguous ranges balanced by record count; row r's + // records start at segOffsetsHost[r * segsPerRow]. uint32_t targetPerTask = recordCount / (uint32_t)taskCount + 1; - uint32_t bin = 0; + cl_int row = 0; for (int t = 0; t < taskCount; t++) { + uint32_t recStart = (row < h) ? segOffsetsHost[(size_t)row * segsPerRow] : recordCount; tasks[t].records = records; - tasks[t].binStart = bin; + tasks[t].recStart = recStart; tasks[t].clusters = zarray_create(sizeof(zarray_t *)); - uint32_t taken = 0; - while (bin < OCL_BIN_COUNT && (taken < targetPerTask || t == taskCount - 1)) { - taken += histHost[bin]; - bin++; + tasks[t].clusterCap = 256; + tasks[t].clusterKeys = malloc(sizeof(uint64_t) * tasks[t].clusterCap); + tasks[t].failed = 0; + while (row < h) { + row++; + uint32_t nextStart = (row < h) ? segOffsetsHost[(size_t)row * segsPerRow] : recordCount; + if (t < taskCount - 1 && nextStart - recStart >= targetPerTask) + break; } - tasks[t].binEnd = bin; + tasks[t].recEnd = (row < h) ? segOffsetsHost[(size_t)row * segsPerRow] : recordCount; } - tasks[taskCount - 1].binEnd = OCL_BIN_COUNT; + tasks[taskCount - 1].recEnd = recordCount; if (taskCount == 1) { doBuildTask(&tasks[0]); @@ -746,35 +781,36 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, workerpool_run(td->wp); } - zarray_t *clusters = zarray_create(sizeof(zarray_t *)); for (int t = 0; t < taskCount; t++) { - for (int i = 0; i < zarray_size(tasks[t].clusters); i++) { - zarray_t *cluster; - zarray_get(tasks[t].clusters, i, &cluster); - zarray_add(clusters, &cluster); + if (tasks[t].failed) { + for (int u = 0; u < taskCount; u++) + destroyTaskClusters(&tasks[u]); + return NULL; } - zarray_destroy(tasks[t].clusters); } - return clusters; + return mergeTaskClusters(tasks, taskCount); } -// Runs CCL + sizes + extraction + partition over the threshold image already -// in inputBuffer, then builds the cluster arrays on the CPU. Caller holds +// Runs CCL + sizes + raster-ordered extraction over the threshold image in +// inputBuffer, then builds the cluster arrays on the CPU. Caller holds // oclMutex and has a valid cache. labelsReady indicates the classify kernel // already seeded the labels buffer. static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl_int cw, cl_int ch, cl_int cs, int labelsReady) { - oclExactOrder = getenv("APRILTAG_OPENCL_EXACT") != NULL; const cl_uint minCluster = (cl_uint)td->qtp.min_cluster_pixels; const cl_uint capacity = OCL_RECORD_CAPACITY; + const cl_int segsPerRow = (cw + OCL_SEG_WIDTH - 1) / OCL_SEG_WIDTH; + const cl_int nSegs = segsPerRow * ch; zarray_t *clusters = NULL; cl_int err = CL_SUCCESS; const cl_uint zero = 0; const size_t pixelCount = (size_t)cw * (size_t)ch; + if ((size_t)nSegs > OCL_SEG_COUNT) + return NULL; + err |= clEnqueueFillBuffer(oclQueue, cache.bufSizes, &zero, 4, 0, pixelCount * 4, 0, NULL, profSlot("fillSizes")); - err |= clEnqueueFillBuffer(oclQueue, cache.bufCounter, &zero, 4, 0, 4, 0, NULL, NULL); - err |= clEnqueueFillBuffer(oclQueue, cache.bufHist, &zero, 4, 0, OCL_BIN_COUNT * 4, 0, NULL, NULL); + err |= clEnqueueFillBuffer(oclQueue, cache.bufSegCounts, &zero, 4, 0, OCL_SEG_COUNT * 4, 0, NULL, NULL); if (err != CL_SUCCESS) goto done; @@ -790,91 +826,90 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl err |= clSetKernelArg(oclKernelBuildBigMap, 5, sizeof(cl_mem), &cache.bufSizes); err |= clSetKernelArg(oclKernelBuildBigMap, 6, sizeof(cl_uint), &minCluster); err |= clSetKernelArg(oclKernelBuildBigMap, 7, sizeof(cl_mem), &cache.bufBigMap); - err |= clSetKernelArg(oclKernelExtractPairs, 0, sizeof(cl_mem), &inputBuffer); - err |= clSetKernelArg(oclKernelExtractPairs, 1, sizeof(cl_int), &cs); - err |= clSetKernelArg(oclKernelExtractPairs, 2, sizeof(cl_int), &cw); - err |= clSetKernelArg(oclKernelExtractPairs, 3, sizeof(cl_int), &ch); - err |= clSetKernelArg(oclKernelExtractPairs, 4, sizeof(cl_mem), &cache.bufLabels); - err |= clSetKernelArg(oclKernelExtractPairs, 5, sizeof(cl_mem), &cache.bufBigMap); - err |= clSetKernelArg(oclKernelExtractPairs, 6, sizeof(cl_mem), &cache.bufCounter); - err |= clSetKernelArg(oclKernelExtractPairs, 7, sizeof(cl_mem), &cache.bufRecords); - err |= clSetKernelArg(oclKernelExtractPairs, 8, sizeof(cl_uint), &capacity); + + err |= clSetKernelArg(oclKernelCountSegments, 0, sizeof(cl_mem), &inputBuffer); + err |= clSetKernelArg(oclKernelCountSegments, 1, sizeof(cl_mem), &cache.bufBigMap); + err |= clSetKernelArg(oclKernelCountSegments, 2, sizeof(cl_int), &cs); + err |= clSetKernelArg(oclKernelCountSegments, 3, sizeof(cl_int), &cw); + err |= clSetKernelArg(oclKernelCountSegments, 4, sizeof(cl_int), &ch); + err |= clSetKernelArg(oclKernelCountSegments, 5, sizeof(cl_int), &segsPerRow); + err |= clSetKernelArg(oclKernelCountSegments, 6, sizeof(cl_int), &nSegs); + err |= clSetKernelArg(oclKernelCountSegments, 7, sizeof(cl_mem), &cache.bufSegCounts); + + err |= clSetKernelArg(oclKernelEmitSegments, 0, sizeof(cl_mem), &inputBuffer); + err |= clSetKernelArg(oclKernelEmitSegments, 1, sizeof(cl_mem), &cache.bufBigMap); + err |= clSetKernelArg(oclKernelEmitSegments, 2, sizeof(cl_mem), &cache.bufLabels); + err |= clSetKernelArg(oclKernelEmitSegments, 3, sizeof(cl_int), &cs); + err |= clSetKernelArg(oclKernelEmitSegments, 4, sizeof(cl_int), &cw); + err |= clSetKernelArg(oclKernelEmitSegments, 5, sizeof(cl_int), &ch); + err |= clSetKernelArg(oclKernelEmitSegments, 6, sizeof(cl_int), &segsPerRow); + err |= clSetKernelArg(oclKernelEmitSegments, 7, sizeof(cl_int), &nSegs); + err |= clSetKernelArg(oclKernelEmitSegments, 8, sizeof(cl_mem), &cache.bufSegOffsets); + err |= clSetKernelArg(oclKernelEmitSegments, 9, sizeof(cl_uint), &capacity); + err |= clSetKernelArg(oclKernelEmitSegments, 10, sizeof(cl_mem), &cache.bufRecords); + + err |= clSetKernelArg(oclKernelScanLocal, 0, sizeof(cl_mem), &cache.bufSegCounts); + err |= clSetKernelArg(oclKernelScanLocal, 1, sizeof(cl_mem), &cache.bufSegOffsets); + err |= clSetKernelArg(oclKernelScanLocal, 2, sizeof(cl_mem), &cache.bufBlockSums); + err |= clSetKernelArg(oclKernelScanBlocks, 0, sizeof(cl_mem), &cache.bufBlockSums); + err |= clSetKernelArg(oclKernelAddBlockOffsets, 0, sizeof(cl_mem), &cache.bufSegOffsets); + err |= clSetKernelArg(oclKernelAddBlockOffsets, 1, sizeof(cl_mem), &cache.bufBlockSums); if (err != CL_SUCCESS) goto done; const size_t global[2] = { roundUp((size_t)cw, 16), roundUp((size_t)ch, 16) }; const size_t countGlobal[2] = { roundUp((size_t)cw, 128), (size_t)ch }; const size_t countLocal[2] = { 128, 1 }; - if (!labelsReady) { + const size_t segGlobal[1] = { roundUp((size_t)nSegs, 64) }; + const size_t scanGlobal[1] = { OCL_SEG_COUNT }; + const size_t scanLocalSize[1] = { 256 }; + const size_t singleItem[1] = { 1 }; + + if (!labelsReady) err |= clEnqueueNDRangeKernel(oclQueue, oclKernelInitLabels, 2, NULL, global, NULL, 0, NULL, profSlot("initLabels")); - } err |= clEnqueueNDRangeKernel(oclQueue, oclKernelMergeEdges, 2, NULL, global, NULL, 0, NULL, profSlot("mergeEdges")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCompressAndCount, 2, NULL, countGlobal, countLocal, 0, NULL, profSlot("compressCount")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelBuildBigMap, 2, NULL, global, NULL, 0, NULL, profSlot("buildBigMap")); - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelExtractPairs, 2, NULL, global, NULL, 0, NULL, profSlot("extractPairs")); - if (err != CL_SUCCESS) - goto done; - - err |= clSetKernelArg(oclKernelHistKeys, 0, sizeof(cl_mem), &cache.bufRecords); - err |= clSetKernelArg(oclKernelHistKeys, 1, sizeof(cl_mem), &cache.bufCounter); - err |= clSetKernelArg(oclKernelHistKeys, 2, sizeof(cl_uint), &capacity); - err |= clSetKernelArg(oclKernelHistKeys, 3, sizeof(cl_mem), &cache.bufHist); - err |= clSetKernelArg(oclKernelScanLocal, 0, sizeof(cl_mem), &cache.bufHist); - err |= clSetKernelArg(oclKernelScanLocal, 1, sizeof(cl_mem), &cache.bufOffsets); - err |= clSetKernelArg(oclKernelScanLocal, 2, sizeof(cl_mem), &cache.bufBlockSums); - err |= clSetKernelArg(oclKernelScanBlocks, 0, sizeof(cl_mem), &cache.bufBlockSums); - err |= clSetKernelArg(oclKernelAddBlockOffsets, 0, sizeof(cl_mem), &cache.bufOffsets); - err |= clSetKernelArg(oclKernelAddBlockOffsets, 1, sizeof(cl_mem), &cache.bufBlockSums); - err |= clSetKernelArg(oclKernelScatterRecords, 0, sizeof(cl_mem), &cache.bufRecords); - err |= clSetKernelArg(oclKernelScatterRecords, 1, sizeof(cl_mem), &cache.bufCounter); - err |= clSetKernelArg(oclKernelScatterRecords, 2, sizeof(cl_uint), &capacity); - err |= clSetKernelArg(oclKernelScatterRecords, 3, sizeof(cl_mem), &cache.bufOffsets); - err |= clSetKernelArg(oclKernelScatterRecords, 4, sizeof(cl_mem), &cache.bufPartitioned); - if (err != CL_SUCCESS) - goto done; - - const size_t capacityGlobal[1] = { (size_t)capacity }; - const size_t scanGlobal[1] = { OCL_BIN_COUNT }; - const size_t scanLocalSize[1] = { 256 }; - const size_t singleItem[1] = { 1 }; - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelHistKeys, 1, NULL, capacityGlobal, NULL, 0, NULL, profSlot("histKeys")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCountSegments, 1, NULL, segGlobal, NULL, 0, NULL, profSlot("countSegments")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanLocal, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("scanLocal")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanBlocks, 1, NULL, singleItem, NULL, 0, NULL, profSlot("scanBlocks")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelAddBlockOffsets, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("addBlockOffs")); - // Read pre-scatter offsets, counts, and the record counter for the host - // build walk; the in-order queue places these before the scatter mutates - // the offsets, and none of them stall the host. - cl_event counterEvent = NULL; - err |= clEnqueueReadBuffer(oclQueue, cache.bufCounter, CL_FALSE, 0, 4, &counterHost, 0, NULL, &counterEvent); - err |= clEnqueueReadBuffer(oclQueue, cache.bufOffsets, CL_FALSE, 0, OCL_BIN_COUNT * 4, offsetsHost, 0, NULL, profSlot("readOffsets")); - err |= clEnqueueReadBuffer(oclQueue, cache.bufHist, CL_FALSE, 0, OCL_BIN_COUNT * 4, histHost, 0, NULL, profSlot("readHist")); - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScatterRecords, 1, NULL, capacityGlobal, NULL, 0, NULL, profSlot("scatter")); + if (err != CL_SUCCESS) + goto done; + + // Read counts and pre-emit offsets for the host build walk; the in-order + // queue keeps them ordered after the scan, and neither stalls the host. + cl_event offsetsEvent = NULL; + err |= clEnqueueReadBuffer(oclQueue, cache.bufSegCounts, CL_FALSE, 0, OCL_SEG_COUNT * 4, segCountsHost, 0, NULL, NULL); + err |= clEnqueueReadBuffer(oclQueue, cache.bufSegOffsets, CL_FALSE, 0, OCL_SEG_COUNT * 4, segOffsetsHost, 0, NULL, &offsetsEvent); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelEmitSegments, 1, NULL, segGlobal, NULL, 0, NULL, profSlot("emitSegments")); if (err != CL_SUCCESS) { - if (counterEvent != NULL) - clReleaseEvent(counterEvent); + if (offsetsEvent != NULL) + clReleaseEvent(offsetsEvent); goto done; } - // The counter read completes mid-chain; waiting on it costs nothing - // extra (the map below blocks on the whole chain anyway) and lets us map - // only the live records instead of the full capacity buffer. - clWaitForEvents(1, &counterEvent); - clReleaseEvent(counterEvent); - if (counterHost > capacity) { + // Totals become available mid-chain; waiting here costs nothing extra + // (the map below blocks on the emit anyway) and bounds the map size. + clWaitForEvents(1, &offsetsEvent); + clReleaseEvent(offsetsEvent); + uint32_t recordCount = segOffsetsHost[nSegs - 1] + segCountsHost[nSegs - 1]; + if (recordCount > capacity) { oclDebugLog("record capacity exceeded"); goto done; } - if (counterHost == 0) { + if (recordCount == 0) { + clFinish(oclQueue); clusters = zarray_create(sizeof(zarray_t *)); goto done; } - void *mapped = clEnqueueMapBuffer(oclQueue, cache.bufPartitioned, CL_TRUE, CL_MAP_READ, 0, - (size_t)counterHost * 16, 0, NULL, profSlot("mapRecords"), &err); + void *mapped = clEnqueueMapBuffer(oclQueue, cache.bufRecords, CL_TRUE, CL_MAP_READ, 0, + (size_t)recordCount * 16, 0, NULL, profSlot("mapRecords"), &err); if (err != CL_SUCCESS) goto done; - clusters = buildClusters(td, (const uint64_t *)mapped, counterHost); - clEnqueueUnmapMemObject(oclQueue, cache.bufPartitioned, mapped, 0, NULL, NULL); + clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch); + clEnqueueUnmapMemObject(oclQueue, cache.bufRecords, mapped, 0, NULL, NULL); done: return clusters; diff --git a/ocl_threshold.h b/ocl_threshold.h index 2be7f139..a5fa73fa 100644 --- a/ocl_threshold.h +++ b/ocl_threshold.h @@ -11,10 +11,11 @@ image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im); // GPU implementation of connected components + gradient clustering over a -// threshold image. Returns a zarray of zarray-of-struct-pt clusters whose -// content matches the CPU implementation (cluster and point order may -// differ), or NULL when the GPU path is unavailable, in which case the -// caller must run connected_components + gradient_clusters on the CPU. +// threshold image. Returns a zarray of zarray-of-struct-pt clusters with +// content and within-cluster point order identical to the CPU +// implementation (cluster order in the outer array may differ), or NULL +// when the GPU path is unavailable, in which case the caller must run +// connected_components + gradient_clusters on the CPU. zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int h, int ts); // Full GPU frontend over the (already decimated/blurred) input image: From 9ac8dbc745093fe85645f83bb2ac35a1b07d667a Mon Sep 17 00:00:00 2001 From: James McVay Date: Wed, 10 Jun 2026 23:35:39 +0200 Subject: [PATCH 03/18] Optimize extraction: workgroup compaction, mask reuse, buffer-cache reuse The per-segment serial emit loops strided adjacent SIMD lanes 256 bytes apart; replace them with one 256-thread workgroup per segment (one pixel per thread, coalesced reads, local prefix scan assigns record slots), preserving raster record order. The count pass stores per-pixel emit masks so the emit pass reads one byte instead of re-evaluating neighbour conditions. The buffer cache now reuses oversized allocations across frame-size changes (re-zeroing only the threshold output), so mixed-size consumers stop paying full reallocation per size change. countSegments 3.9 -> 1.1 ms, emitSegments 2.6 -> 1.8 ms on Arc 140T; output remains bit-identical to the CPU implementation (corpus 108/108, max corner delta 0.0 px, deterministic run-to-run). Co-Authored-By: Claude Fable 5 --- ocl_threshold.c | 107 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index a74334da..9026050a 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -202,42 +202,60 @@ static const char *sourceExtract = "}\n"; static const char *sourceEmit = + // Stream compaction: one 256-thread workgroup per row segment, one pixel + // per thread (coalesced), local prefix scan assigns each thread's record + // slots. Per-pixel records stay in DO_CONN order and threads ascend in x, + // so global record order remains the CPU emitter's raster order. The + // count pass stores each pixel's emit mask so the emit pass reads one + // byte instead of re-evaluating the neighbour conditions. "__kernel void countSegments(__global const uchar *im, __global const uchar *bigMap,\n" " int s, int w, int h, int segsPerRow, int nSegs,\n" - " __global uint *segCounts) {\n" - " int seg = get_global_id(0);\n" - " if (seg >= nSegs) return;\n" + " __global uint *segCounts, __global uchar *masks) {\n" + " int seg = get_group_id(0);\n" + " int lid = get_local_id(0);\n" " int y = seg / segsPerRow;\n" - " int x0 = (seg % segsPerRow) * 256;\n" - " int x1 = min(x0 + 256, w - 1);\n" - " if (x0 < 1) x0 = 1;\n" - " uint count = 0;\n" - " if (y >= 1 && y < h - 1) {\n" - " for (int x = x0; x < x1; x++)\n" - " count += (uint)popcount(emitMask(im, bigMap, s, w, x, y));\n" + " int x = (seg % segsPerRow) * 256 + lid;\n" + " int mask = 0;\n" + " if (y >= 1 && y < h - 1 && x >= 1 && x < w - 1)\n" + " mask = emitMask(im, bigMap, s, w, x, y);\n" + " if (x < w && y < h)\n" + " masks[y * w + x] = (uchar)mask;\n" + " __local uint counts[256];\n" + " counts[lid] = (uint)popcount(mask);\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int off = 128; off > 0; off >>= 1) {\n" + " if (lid < off) counts[lid] += counts[lid + off];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" - " segCounts[seg] = count;\n" + " if (lid == 0 && seg < nSegs) segCounts[seg] = counts[0];\n" "}\n" - "__kernel void emitSegments(__global const uchar *im, __global const uchar *bigMap,\n" + "__kernel void emitSegments(__global const uchar *im, __global const uchar *masks,\n" " __global const uint *labels, int s, int w, int h,\n" " int segsPerRow, int nSegs, __global const uint *segOffsets,\n" " uint capacity, __global ulong2 *records) {\n" - " int seg = get_global_id(0);\n" - " if (seg >= nSegs) return;\n" + " int seg = get_group_id(0);\n" + " int lid = get_local_id(0);\n" " int y = seg / segsPerRow;\n" - " if (y < 1 || y >= h - 1) return;\n" - " int x0 = (seg % segsPerRow) * 256;\n" - " int x1 = min(x0 + 256, w - 1);\n" - " if (x0 < 1) x0 = 1;\n" - " uint slot = segOffsets[seg];\n" - " for (int x = x0; x < x1; x++) {\n" - " int mask = emitMask(im, bigMap, s, w, x, y);\n" - " if (mask == 0) continue;\n" - " if (mask & 1) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 0); slot++; }\n" - " if (mask & 2) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 0, 1); slot++; }\n" - " if (mask & 4) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, -1, 1); slot++; }\n" - " if (mask & 8) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 1); slot++; }\n" + " int x = (seg % segsPerRow) * 256 + lid;\n" + " int mask = 0;\n" + " if (y >= 1 && y < h - 1 && x >= 1 && x < w - 1)\n" + " mask = masks[y * w + x];\n" + " uint mine = (uint)popcount(mask);\n" + " __local uint scanBuf[256];\n" + " scanBuf[lid] = mine;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int off = 1; off < 256; off <<= 1) {\n" + " uint v = (lid >= off) ? scanBuf[lid - off] : 0u;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " scanBuf[lid] += v;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" + " if (mask == 0 || seg >= nSegs) return;\n" + " uint slot = segOffsets[seg] + scanBuf[lid] - mine;\n" + " if (mask & 1) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 0); slot++; }\n" + " if (mask & 2) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 0, 1); slot++; }\n" + " if (mask & 4) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, -1, 1); slot++; }\n" + " if (mask & 8) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 1); slot++; }\n" "}\n"; static const char *sourceScan = @@ -300,6 +318,9 @@ static cl_kernel oclKernelAddBlockOffsets; typedef struct { int valid; cl_int w, h, s; + size_t allocImageBytes; + size_t allocTileBytes; + size_t allocPixelCount; const void *thresholdOutputFor; cl_mem bufIm; cl_mem bufOut; @@ -310,6 +331,7 @@ typedef struct { cl_mem bufLabels; cl_mem bufSizes; cl_mem bufBigMap; + cl_mem bufMasks; cl_mem bufRecords; cl_mem bufSegCounts; cl_mem bufSegOffsets; @@ -452,6 +474,7 @@ static void releaseCache(void) releaseBuffer(cache.bufLabels); releaseBuffer(cache.bufSizes); releaseBuffer(cache.bufBigMap); + releaseBuffer(cache.bufMasks); releaseBuffer(cache.bufRecords); releaseBuffer(cache.bufSegCounts); releaseBuffer(cache.bufSegOffsets); @@ -472,11 +495,27 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) { if (cache.valid && cache.w == w && cache.h == h && cache.s == s) return 1; - releaseCache(); const size_t imageBytes = (size_t)s * (size_t)h; const size_t tileBytes = (size_t)tw * (size_t)th; const size_t pixelCount = (size_t)w * (size_t)h; + + // Smaller frames reuse the existing (larger) buffers: only the threshold + // output needs re-zeroing so stride padding from a previous larger frame + // can't leak into the byte-exact output image. + if (cache.valid && imageBytes <= cache.allocImageBytes && + tileBytes <= cache.allocTileBytes && pixelCount <= cache.allocPixelCount) { + const cl_uchar zeroByte = 0; + if (clEnqueueFillBuffer(oclQueue, cache.bufOut, &zeroByte, 1, 0, imageBytes, 0, NULL, NULL) != CL_SUCCESS) + return 0; + cache.w = w; + cache.h = h; + cache.s = s; + cache.thresholdOutputFor = NULL; + return 1; + } + + releaseCache(); int failed = 0; cache.bufIm = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, imageBytes, NULL, &failed); cache.bufOut = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, imageBytes, NULL, &failed); @@ -487,6 +526,7 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) cache.bufLabels = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); cache.bufSizes = createOrFail(CL_MEM_READ_WRITE, pixelCount * 4, NULL, &failed); cache.bufBigMap = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); + cache.bufMasks = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); cache.bufRecords = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); cache.bufSegCounts = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufSegOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); @@ -495,6 +535,9 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) releaseCache(); return 0; } + cache.allocImageBytes = imageBytes; + cache.allocTileBytes = tileBytes; + cache.allocPixelCount = pixelCount; // Stride padding bytes are never written by the threshold kernels; zero // the output buffer once so padding matches the CPU path's calloc'd image. @@ -835,9 +878,10 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl err |= clSetKernelArg(oclKernelCountSegments, 5, sizeof(cl_int), &segsPerRow); err |= clSetKernelArg(oclKernelCountSegments, 6, sizeof(cl_int), &nSegs); err |= clSetKernelArg(oclKernelCountSegments, 7, sizeof(cl_mem), &cache.bufSegCounts); + err |= clSetKernelArg(oclKernelCountSegments, 8, sizeof(cl_mem), &cache.bufMasks); err |= clSetKernelArg(oclKernelEmitSegments, 0, sizeof(cl_mem), &inputBuffer); - err |= clSetKernelArg(oclKernelEmitSegments, 1, sizeof(cl_mem), &cache.bufBigMap); + err |= clSetKernelArg(oclKernelEmitSegments, 1, sizeof(cl_mem), &cache.bufMasks); err |= clSetKernelArg(oclKernelEmitSegments, 2, sizeof(cl_mem), &cache.bufLabels); err |= clSetKernelArg(oclKernelEmitSegments, 3, sizeof(cl_int), &cs); err |= clSetKernelArg(oclKernelEmitSegments, 4, sizeof(cl_int), &cw); @@ -860,7 +904,8 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl const size_t global[2] = { roundUp((size_t)cw, 16), roundUp((size_t)ch, 16) }; const size_t countGlobal[2] = { roundUp((size_t)cw, 128), (size_t)ch }; const size_t countLocal[2] = { 128, 1 }; - const size_t segGlobal[1] = { roundUp((size_t)nSegs, 64) }; + const size_t segGlobal[1] = { (size_t)nSegs * 256 }; + const size_t segLocal[1] = { 256 }; const size_t scanGlobal[1] = { OCL_SEG_COUNT }; const size_t scanLocalSize[1] = { 256 }; const size_t singleItem[1] = { 1 }; @@ -870,7 +915,7 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl err |= clEnqueueNDRangeKernel(oclQueue, oclKernelMergeEdges, 2, NULL, global, NULL, 0, NULL, profSlot("mergeEdges")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCompressAndCount, 2, NULL, countGlobal, countLocal, 0, NULL, profSlot("compressCount")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelBuildBigMap, 2, NULL, global, NULL, 0, NULL, profSlot("buildBigMap")); - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCountSegments, 1, NULL, segGlobal, NULL, 0, NULL, profSlot("countSegments")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelCountSegments, 1, NULL, segGlobal, segLocal, 0, NULL, profSlot("countSegments")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanLocal, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("scanLocal")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelScanBlocks, 1, NULL, singleItem, NULL, 0, NULL, profSlot("scanBlocks")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelAddBlockOffsets, 1, NULL, scanGlobal, scanLocalSize, 0, NULL, profSlot("addBlockOffs")); @@ -882,7 +927,7 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl cl_event offsetsEvent = NULL; err |= clEnqueueReadBuffer(oclQueue, cache.bufSegCounts, CL_FALSE, 0, OCL_SEG_COUNT * 4, segCountsHost, 0, NULL, NULL); err |= clEnqueueReadBuffer(oclQueue, cache.bufSegOffsets, CL_FALSE, 0, OCL_SEG_COUNT * 4, segOffsetsHost, 0, NULL, &offsetsEvent); - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelEmitSegments, 1, NULL, segGlobal, NULL, 0, NULL, profSlot("emitSegments")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelEmitSegments, 1, NULL, segGlobal, segLocal, 0, NULL, profSlot("emitSegments")); if (err != CL_SUCCESS) { if (offsetsEvent != NULL) clReleaseEvent(offsetsEvent); From 8c3f7c26b12f5550471148d9c534f1a8521dcb9b Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 05:24:22 +0200 Subject: [PATCH 04/18] Add fit_quads GPU port plan with measured fp64 feasibility Arc 140T exposes cl_khr_fp64; the calibrated lfps simulation probe (9000 clusters x 300 pts, sequential double moment accumulation with image sampling) runs in 2.56 ms vs ~15 ms for the CPU fit_quads stage. Plan: stable key radix sort keeps records on-device, one workgroup per cluster fits quads in double preserving CPU summation order, and only the quad array returns to the host. Co-Authored-By: Claude Fable 5 --- ocl_harness/FIT_QUADS_PLAN.md | 67 +++++++++++++++++++++++++ ocl_harness/fp64_probe.c | 94 +++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 ocl_harness/FIT_QUADS_PLAN.md create mode 100644 ocl_harness/fp64_probe.c diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md new file mode 100644 index 00000000..748aa8d4 --- /dev/null +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -0,0 +1,67 @@ +# fit_quads GPU port — design and measured feasibility + +Goal: move the last big CPU stage (~12-15 ms wall, fit_quads) onto the GPU, +taking the detect pipeline to roughly 25 ms wall with CPU usage in the tens +of core-ms. With fit on the GPU, boundary records never return to the host: +the CPU receives ~100 candidate quads (KB) instead of ~43 MB of records, and +the cluster build walk disappears entirely. + +## Measured feasibility (w3cj, Arc 140T, 2026-06-11) + +- `cl_khr_fp64` is exposed, including `__opencl_c_ext_fp64_global_atomic_add`. +- `ocl_harness/fp64_probe.c`: 9000 clusters x 300 pts of sequential + double-precision moment accumulation with one image sample per point + (the compute_lfps + fit_line shape, calibrated to the measured 2.7M + record count): **2.56 ms**. The serial-per-cluster scan pattern that + preserves CPU summation order is affordable. + +## End-state architecture + +1. emitSegments leaves records on the device (raster order, key-tagged). +2. **Stable LSD radix sort by cluster key** (8-bit digits over the 46-bit + key, per-thread-sequential workgroup ranking for stability). Stability + matters: input is raster-ordered, so a stable key sort yields clusters + whose internal point order is exactly the CPU emitter's order. +3. Cluster boundary detection (key change) -> descriptor array + (offset, count) + size/hull filters from fit_quads' caller. +4. Per-cluster fit, one workgroup per cluster: + - bbox + center reduce; per-point slope; gradient dot -> reversed_border + filter (threads parallel, local reduce). + - sort by slope: segmented bitonic on (slope, position-index) pairs. + - lfps prefix moments: ONE lane per cluster, sequential double + accumulation (preserves CPU summation order; probe says this is cheap). + Image weight samples read from the device-resident input image. + - windowed errs: parallel per point via prefix moments (fit_line is O(1)). + - maxima extraction + top-K by err (local compact + small sort). + - corner combination search (<= ~210 combos for max_nmaxima=10): + parallel across threads, local argmin reduce. + - final 4 fit_line + intersections + angle/area checks: lane 0. +5. Readback: quad array only. CPU runs decode unchanged. + +## Exactness contract + +- Doubles end-to-end; per-cluster serial accumulation preserves the CPU's + floating-point operation order, so corners should match to the last bit + except where slope TIES exist: CPU ptsort's tie order is a sorting-network + artifact; the GPU sorts by (slope, raster index), a total order. The + fit_quad center-noise constants make exact float slope ties rare; the + corpus harness quantifies any residual difference. Contract: identical + detection sets; corners < 1e-6 px except tie-affected clusters. +- Determinism run-to-run: total-order sort + fixed reduction shapes = yes. + +## Phases (each gated on the corpus harness) + +- P1: stable GPU radix sort by key + descriptors; validate cluster + set/order equivalence against the CPU build walk; CPU fit still consumes + a readback in this phase. +- P2: per-cluster slope/filter/slope-sort on GPU; validate sorted point + order against ptsort output (tie cases logged). +- P3: lfps + maxima + combos + line fits; quads-only readback; corpus + detection equivalence + timing. + +## Also still open (smaller) + +- mergeEdges tiled local-memory CCL (~2.3 ms -> est. ~1 ms). +- compressAndCount (~2.5 ms): vertical run aggregation or subgroup reduce. +- Vide soak prep is blocked on distribution: ninja mode means no public + branch; the nix overlay needs either a private remote or a local-path src. diff --git a/ocl_harness/fp64_probe.c b/ocl_harness/fp64_probe.c new file mode 100644 index 00000000..b5576327 --- /dev/null +++ b/ocl_harness/fp64_probe.c @@ -0,0 +1,94 @@ +#include +#include +#include + +#define CL_TARGET_OPENCL_VERSION 300 +#include + +// Measures whether per-cluster sequential double-precision moment +// accumulation (the compute_lfps + fit_line pattern from fit_quad) is fast +// enough on the iGPU to justify a GPU fit_quads port. Each work-item plays +// the role of one cluster: a serial scan of N points accumulating the six +// line-fit moments in double, with one image byte sampled per point. +static const char *probeSource = + "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n" + "__kernel void lfpsSim(__global const uchar *im, int imBytes, int ptsPerCluster,\n" + " __global double *out) {\n" + " int cluster = get_global_id(0);\n" + " uint seed = (uint)cluster * 2654435761u + 1u;\n" + " double Mx = 0, My = 0, Mxx = 0, Mxy = 0, Myy = 0, W = 0;\n" + " for (int i = 0; i < ptsPerCluster; i++) {\n" + " seed = seed * 1664525u + 1013904223u;\n" + " double x = (double)(seed & 0xFFFu);\n" + " double y = (double)((seed >> 12) & 0xFFFu);\n" + " double w = (double)im[seed % (uint)imBytes] + 1.0;\n" + " Mx += w * x; My += w * y;\n" + " Mxx += w * x * x; Mxy += w * x * y; Myy += w * y * y;\n" + " W += w;\n" + " }\n" + " double Ex = Mx / W, Ey = My / W;\n" + " double Cxx = Mxx / W - Ex * Ex;\n" + " double Cxy = Mxy / W - Ex * Ey;\n" + " double Cyy = Myy / W - Ey * Ey;\n" + " double eig = 0.5 * (Cxx + Cyy - sqrt((Cxx - Cyy) * (Cxx - Cyy) + 4.0 * Cxy * Cxy));\n" + " out[cluster] = eig;\n" + "}\n"; + +static double nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +int main(void) { + const int nClusters = 9000; + const int ptsPerCluster = 300; + const int imBytes = 6 * 1024 * 1024; + + cl_platform_id platform; + cl_device_id device; + clGetPlatformIDs(1, &platform, NULL); + if (clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device, NULL) != CL_SUCCESS) { + fprintf(stderr, "no GPU\n"); + return 1; + } + cl_int err; + cl_context ctx = clCreateContext(NULL, 1, &device, NULL, NULL, &err); + cl_command_queue queue = clCreateCommandQueueWithProperties(ctx, device, NULL, &err); + cl_program program = clCreateProgramWithSource(ctx, 1, &probeSource, NULL, &err); + if (clBuildProgram(program, 1, &device, "", NULL, NULL) != CL_SUCCESS) { + char log[4096] = { 0 }; + clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, sizeof(log) - 1, log, NULL); + fprintf(stderr, "build failed:\n%s\n", log); + return 1; + } + cl_kernel kernel = clCreateKernel(program, "lfpsSim", &err); + + unsigned char *imageHost = malloc(imBytes); + for (int i = 0; i < imBytes; i++) + imageHost[i] = (unsigned char)(i * 31); + cl_mem bufIm = clCreateBuffer(ctx, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, imBytes, imageHost, &err); + cl_mem bufOut = clCreateBuffer(ctx, CL_MEM_WRITE_ONLY, nClusters * 8, NULL, &err); + + const cl_int cImBytes = imBytes, cPts = ptsPerCluster; + clSetKernelArg(kernel, 0, sizeof(cl_mem), &bufIm); + clSetKernelArg(kernel, 1, sizeof(cl_int), &cImBytes); + clSetKernelArg(kernel, 2, sizeof(cl_int), &cPts); + clSetKernelArg(kernel, 3, sizeof(cl_mem), &bufOut); + + const size_t global[1] = { (size_t)nClusters }; + clEnqueueNDRangeKernel(queue, kernel, 1, NULL, global, NULL, 0, NULL, NULL); + clFinish(queue); + + double best = 1e9; + for (int iter = 0; iter < 10; iter++) { + double start = nowMs(); + clEnqueueNDRangeKernel(queue, kernel, 1, NULL, global, NULL, 0, NULL, NULL); + clFinish(queue); + double elapsed = nowMs() - start; + if (elapsed < best) + best = elapsed; + } + printf("fp64 lfps simulation: %d clusters x %d pts: best %.2f ms\n", nClusters, ptsPerCluster, best); + return 0; +} From 0aed0b88c6f1b1ffc35575ec40642d466633e526 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 05:31:39 +0200 Subject: [PATCH 05/18] Add stable GPU radix sort by cluster key (P1 validation scaffolding) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six-pass 8-bit LSD radix over the compacted 46-bit key, thread-blocked for stability so sorted clusters carry the emitter's exact raster point order. Gated behind APRILTAG_OPENCL_SORTED: output validated bit-exact (detect corner delta 0.0, corpus 108/108) but radixScatter costs ~10.9 ms/pass on Arc 140T — the plan pivots to a CPU-walk-emitted permutation + GPU gather for the fit_quads port (see FIT_QUADS_PLAN.md). Co-Authored-By: Claude Fable 5 --- ocl_harness/FIT_QUADS_PLAN.md | 16 ++- ocl_threshold.c | 214 +++++++++++++++++++++++++++++++++- 2 files changed, 223 insertions(+), 7 deletions(-) diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index 748aa8d4..5eae2bde 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -51,9 +51,19 @@ the cluster build walk disappears entirely. ## Phases (each gated on the corpus harness) -- P1: stable GPU radix sort by key + descriptors; validate cluster - set/order equivalence against the CPU build walk; CPU fit still consumes - a readback in this phase. +- P1 (DONE, measured): stable GPU radix sort by compacted key, env-gated + APRILTAG_OPENCL_SORTED. Correctness: bit-exact (detect 0.000000 px, + corpus 108/108) — stability + raster emission provably yields exact + CPU cluster content and point order. Performance: radixScatter is + ~10.9 ms/pass x 6 (scattered 16 B writes + low-occupancy ranking) — + a full global sort is the wrong tool. KEPT in tree as validation + scaffolding; do not enable in production. +- P1b (REVISED grouping design for P3): the CPU build walk already + discovers groups for ~6 core-ms — have it emit a permutation array + (record index per output slot, cluster-contiguous) plus per-cluster + descriptors; upload (~11 MB) and run one coalesced GPU gather (~1 ms) + to materialize cluster-contiguous records on-device. Replaces the sort + entirely; ordering guarantees identical (walk preserves raster order). - P2: per-cluster slope/filter/slope-sort on GPU; validate sorted point order against ptsort output (tie cases logged). - P3: lfps + maxima + combos + line fits; quads-only readback; corpus diff --git a/ocl_threshold.c b/ocl_threshold.c index 9026050a..9c1da4a5 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -193,7 +193,9 @@ static const char *sourceExtract = " uchar v1 = im[(y + dy) * s + x + dx];\n" " uint rep0 = labels[y * w + x];\n" " uint rep1 = labels[(y + dy) * w + x + dx];\n" - " ulong key = (rep0 < rep1) ? (((ulong)rep1 << 32) | rep0) : (((ulong)rep0 << 32) | rep1);\n" + // Roots are < 2^23 (enforced by the w*h guard), so the pair packs into + // 46 contiguous bits — six 8-bit radix passes cover the whole key. + " ulong key = (rep0 < rep1) ? (((ulong)rep1 << 23) | rep0) : (((ulong)rep0 << 23) | rep1);\n" " int grad = (int)v1 - (int)v0;\n" " ushort px = (ushort)(2 * x + dx), py = (ushort)(2 * y + dy);\n" " ushort pgx = (ushort)(short)(dx * grad), pgy = (ushort)(short)(dy * grad);\n" @@ -258,6 +260,56 @@ static const char *sourceEmit = " if (mask & 8) { if (slot < capacity) records[slot] = makeRecord(im, labels, s, w, x, y, 1, 1); slot++; }\n" "}\n"; +static const char *sourceSort = + // Stable LSD radix sort over the compacted 46-bit cluster key, 8-bit + // digits, 1024-record blocks (32 threads x 32 records, thread-blocked so + // within-thread order is sequential). Stability preserves the raster + // point order the emitter established. Per-(digit, block) offsets are + // scanned on the host between passes. + "__kernel void radixHist(__global const ulong2 *records, uint count, uint shift,\n" + " uint numBlocks, __global uint *hist) {\n" + " int block = get_group_id(0), lid = get_local_id(0);\n" + " __local uint h[256];\n" + " for (int i = lid; i < 256; i += 32) h[i] = 0;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " uint base = (uint)block * 1024u + (uint)lid * 32u;\n" + " for (int j = 0; j < 32; j++) {\n" + " uint i = base + (uint)j;\n" + " if (i < count) atomic_inc(&h[(uint)((records[i].x >> shift) & 0xFFul)]);\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int i = lid; i < 256; i += 32)\n" + " hist[(uint)i * numBlocks + (uint)block] = h[i];\n" + "}\n" + "__kernel void radixScatter(__global const ulong2 *in, uint count, uint shift,\n" + " uint numBlocks, __global const uint *offsets,\n" + " __global ulong2 *out) {\n" + " int block = get_group_id(0), lid = get_local_id(0);\n" + " __local uint counts[8192];\n" + " for (int i = lid; i < 8192; i += 32) counts[i] = 0;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " uint base = (uint)block * 1024u + (uint)lid * 32u;\n" + " uchar digs[32];\n" + " for (int j = 0; j < 32; j++) {\n" + " uint i = base + (uint)j;\n" + " if (i < count) {\n" + " digs[j] = (uchar)((in[i].x >> shift) & 0xFFul);\n" + " counts[(uint)digs[j] * 32u + (uint)lid]++;\n" + " }\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int j = 0; j < 32; j++) {\n" + " uint i = base + (uint)j;\n" + " if (i >= count) continue;\n" + " uint d = digs[j];\n" + " uint intra = 0;\n" + " for (int t = 0; t < lid; t++) intra += counts[d * 32u + (uint)t];\n" + " uint own = 0;\n" + " for (int k = 0; k < j; k++) own += (digs[k] == (uchar)d) ? 1u : 0u;\n" + " out[offsets[d * numBlocks + (uint)block] + intra + own] = in[i];\n" + " }\n" + "}\n"; + static const char *sourceScan = "__kernel void scanLocal(__global const uint *hist, __global uint *offsets,\n" " __global uint *blockSums) {\n" @@ -314,6 +366,8 @@ static cl_kernel oclKernelEmitSegments; static cl_kernel oclKernelScanLocal; static cl_kernel oclKernelScanBlocks; static cl_kernel oclKernelAddBlockOffsets; +static cl_kernel oclKernelRadixHist; +static cl_kernel oclKernelRadixScatter; typedef struct { int valid; @@ -333,6 +387,9 @@ typedef struct { cl_mem bufBigMap; cl_mem bufMasks; cl_mem bufRecords; + cl_mem bufRecordsAlt; + cl_mem bufSortHist; + cl_mem bufSortOffsets; cl_mem bufSegCounts; cl_mem bufSegOffsets; cl_mem bufBlockSums; @@ -417,8 +474,8 @@ static void oclInit(void) if (err != CL_SUCCESS) return; - const char *sources[6] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourceEmit, sourceScan }; - cl_program program = clCreateProgramWithSource(oclContext, 6, sources, NULL, &err); + const char *sources[7] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourceEmit, sourceSort, sourceScan }; + cl_program program = clCreateProgramWithSource(oclContext, 7, sources, NULL, &err); if (err != CL_SUCCESS) return; err = clBuildProgram(program, 1, &device, "", NULL, NULL); @@ -443,6 +500,8 @@ static void oclInit(void) { &oclKernelScanLocal, "scanLocal" }, { &oclKernelScanBlocks, "scanBlocks" }, { &oclKernelAddBlockOffsets, "addBlockOffsets" }, + { &oclKernelRadixHist, "radixHist" }, + { &oclKernelRadixScatter, "radixScatter" }, }; int failed = 0; for (size_t i = 0; i < sizeof(kernels) / sizeof(kernels[0]); i++) { @@ -476,6 +535,9 @@ static void releaseCache(void) releaseBuffer(cache.bufBigMap); releaseBuffer(cache.bufMasks); releaseBuffer(cache.bufRecords); + releaseBuffer(cache.bufRecordsAlt); + releaseBuffer(cache.bufSortHist); + releaseBuffer(cache.bufSortOffsets); releaseBuffer(cache.bufSegCounts); releaseBuffer(cache.bufSegOffsets); releaseBuffer(cache.bufBlockSums); @@ -528,6 +590,9 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) cache.bufBigMap = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); cache.bufMasks = createOrFail(CL_MEM_READ_WRITE, pixelCount, NULL, &failed); cache.bufRecords = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); + cache.bufRecordsAlt = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 16, NULL, &failed); + cache.bufSortHist = createOrFail(CL_MEM_READ_WRITE, (size_t)256 * (OCL_RECORD_CAPACITY / 1024) * 4, NULL, &failed); + cache.bufSortOffsets = createOrFail(CL_MEM_READ_WRITE, (size_t)256 * (OCL_RECORD_CAPACITY / 1024) * 4, NULL, &failed); cache.bufSegCounts = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufSegOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufBlockSums = createOrFail(CL_MEM_READ_WRITE, 256 * 4, NULL, &failed); @@ -834,6 +899,137 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, return mergeTaskClusters(tasks, taskCount); } +// Stable 6-pass LSD radix sort of the record buffer by the compacted 46-bit +// cluster key. Stability plus raster-ordered input means each cluster ends +// up contiguous with its points in the CPU emitter's exact order. The +// per-(digit, block) offset scan runs on the host between passes. Caller +// holds oclMutex. Returns 0 on failure; on success the sorted records are +// back in cache.bufRecords. +static int sortRecords(uint32_t recordCount) +{ + static uint32_t *histScratch = NULL; + const uint32_t numBlocks = (recordCount + 1023u) / 1024u; + const size_t histEntries = (size_t)256 * numBlocks; + if (histScratch == NULL) { + histScratch = malloc((size_t)256 * (OCL_RECORD_CAPACITY / 1024) * 4); + if (histScratch == NULL) + return 0; + } + + cl_mem cur = cache.bufRecords; + cl_mem alt = cache.bufRecordsAlt; + const size_t sortGlobal[1] = { (size_t)numBlocks * 32 }; + const size_t sortLocal[1] = { 32 }; + + for (int pass = 0; pass < 6; pass++) { + const cl_uint shift = (cl_uint)(pass * 8); + cl_int err = CL_SUCCESS; + err |= clSetKernelArg(oclKernelRadixHist, 0, sizeof(cl_mem), &cur); + err |= clSetKernelArg(oclKernelRadixHist, 1, sizeof(cl_uint), &recordCount); + err |= clSetKernelArg(oclKernelRadixHist, 2, sizeof(cl_uint), &shift); + err |= clSetKernelArg(oclKernelRadixHist, 3, sizeof(cl_uint), &numBlocks); + err |= clSetKernelArg(oclKernelRadixHist, 4, sizeof(cl_mem), &cache.bufSortHist); + if (err != CL_SUCCESS) + return 0; + err = clEnqueueNDRangeKernel(oclQueue, oclKernelRadixHist, 1, NULL, sortGlobal, sortLocal, 0, NULL, profSlot("radixHist")); + if (err != CL_SUCCESS) + return 0; + err = clEnqueueReadBuffer(oclQueue, cache.bufSortHist, CL_TRUE, 0, histEntries * 4, histScratch, 0, NULL, NULL); + if (err != CL_SUCCESS) + return 0; + uint32_t running = 0; + for (size_t i = 0; i < histEntries; i++) { + uint32_t v = histScratch[i]; + histScratch[i] = running; + running += v; + } + err = clEnqueueWriteBuffer(oclQueue, cache.bufSortOffsets, CL_FALSE, 0, histEntries * 4, histScratch, 0, NULL, NULL); + err |= clSetKernelArg(oclKernelRadixScatter, 0, sizeof(cl_mem), &cur); + err |= clSetKernelArg(oclKernelRadixScatter, 1, sizeof(cl_uint), &recordCount); + err |= clSetKernelArg(oclKernelRadixScatter, 2, sizeof(cl_uint), &shift); + err |= clSetKernelArg(oclKernelRadixScatter, 3, sizeof(cl_uint), &numBlocks); + err |= clSetKernelArg(oclKernelRadixScatter, 4, sizeof(cl_mem), &cache.bufSortOffsets); + err |= clSetKernelArg(oclKernelRadixScatter, 5, sizeof(cl_mem), &alt); + if (err != CL_SUCCESS) + return 0; + err = clEnqueueNDRangeKernel(oclQueue, oclKernelRadixScatter, 1, NULL, sortGlobal, sortLocal, 0, NULL, profSlot("radixScatter")); + if (err != CL_SUCCESS) + return 0; + cl_mem tmp = cur; + cur = alt; + alt = tmp; + } + // Six passes: the final output landed back in cache.bufRecords. + return cur == cache.bufRecords; +} + +typedef struct { + const uint64_t *records; + uint32_t recStart, recEnd; + zarray_t *clusters; +} SortedTask; + +static void doSortedTask(void *p) +{ + SortedTask *task = (SortedTask *)p; + zarray_t *cluster = NULL; + uint64_t currentKey = 0; + for (uint32_t i = task->recStart; i < task->recEnd; i++) { + uint64_t key = task->records[2 * i]; + if (cluster == NULL || key != currentKey) { + cluster = zarray_create(sizeof(OclPt)); + zarray_add(task->clusters, &cluster); + currentKey = key; + } + appendPt(cluster, task->records[2 * i + 1]); + } +} + +// Cluster build over key-sorted records: groups are contiguous, so this is a +// linear walk with no hashing. Task ranges are aligned to key boundaries so +// no cluster spans two tasks. +static zarray_t *buildClustersSorted(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount) +{ + int taskCount = (td->wp != NULL && td->nthreads > 1) ? td->nthreads : 1; + if (taskCount > 16) + taskCount = 16; + SortedTask tasks[16]; + + uint32_t pos = 0; + for (int t = 0; t < taskCount; t++) { + tasks[t].records = records; + tasks[t].recStart = pos; + tasks[t].clusters = zarray_create(sizeof(zarray_t *)); + uint32_t target = (uint32_t)(((uint64_t)recordCount * (t + 1)) / taskCount); + if (target < pos) + target = pos; + while (target < recordCount && target > 0 && records[2 * target] == records[2 * (target - 1)]) + target++; + tasks[t].recEnd = target; + pos = target; + } + tasks[taskCount - 1].recEnd = recordCount; + + if (taskCount == 1) { + doSortedTask(&tasks[0]); + } else { + for (int t = 0; t < taskCount; t++) + workerpool_add_task(td->wp, doSortedTask, &tasks[t]); + workerpool_run(td->wp); + } + + zarray_t *clusters = zarray_create(sizeof(zarray_t *)); + for (int t = 0; t < taskCount; t++) { + for (int i = 0; i < zarray_size(tasks[t].clusters); i++) { + zarray_t *cluster; + zarray_get(tasks[t].clusters, i, &cluster); + zarray_add(clusters, &cluster); + } + zarray_destroy(tasks[t].clusters); + } + return clusters; +} + // Runs CCL + sizes + raster-ordered extraction over the threshold image in // inputBuffer, then builds the cluster arrays on the CPU. Caller holds // oclMutex and has a valid cache. labelsReady indicates the classify kernel @@ -949,11 +1145,21 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl goto done; } + // P1 scaffolding for the GPU fit_quads port: sort records by cluster key + // on the GPU so groups are contiguous (within-group raster order is + // preserved by sort stability). Gated until the GPU fit lands. + int useSorted = getenv("APRILTAG_OPENCL_SORTED") != NULL; + if (useSorted && !sortRecords(recordCount)) + goto done; + void *mapped = clEnqueueMapBuffer(oclQueue, cache.bufRecords, CL_TRUE, CL_MAP_READ, 0, (size_t)recordCount * 16, 0, NULL, profSlot("mapRecords"), &err); if (err != CL_SUCCESS) goto done; - clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch); + if (useSorted) + clusters = buildClustersSorted(td, (const uint64_t *)mapped, recordCount); + else + clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch); clEnqueueUnmapMemObject(oclQueue, cache.bufRecords, mapped, 0, NULL, NULL); done: From ffd9853624327e2686c3fc6866b2309b35f96c97 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 08:12:13 +0200 Subject: [PATCH 06/18] Add gather permutation path: cluster-contiguous records on-device (P1b) The build walk's grouping becomes a counting-sort permutation: the walk stores each record's task-local cluster index (one flat store per record), the merge records each local cluster's final index and chunk start, and a parallel pass writes each record's output slot directly into the mapped staging buffer once final cluster offsets are known (which also fills the per-cluster descriptors). One coalesced gather kernel (~1.5 ms, async) then materializes cluster-contiguous records in bufRecordsAlt for the upcoming GPU fit stages. An earlier per-cluster index-list version cost ~13 ms/frame of host bookkeeping; this shape measures ~3 ms and end-to-end detect is indistinguishable from the no-gather baseline. Gated APRILTAG_OPENCL_GATHER=1; with APRILTAG_OPENCL_GATHER_VALIDATE=1 the gathered records are read back and checked against the CPU-built clusters (216/216 across the corpus; detect 0.000000 px, corpus 108/108). APRILTAG_OPENCL_PROFILE=1 now also prints host-side frontend stamps. Co-Authored-By: Claude Fable 5 --- ocl_harness/FIT_QUADS_PLAN.md | 40 ++-- ocl_harness/README.md | 13 +- ocl_threshold.c | 336 ++++++++++++++++++++++++++++++++-- 3 files changed, 361 insertions(+), 28 deletions(-) diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index 5eae2bde..b4d7a0d7 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -18,12 +18,14 @@ the cluster build walk disappears entirely. ## End-state architecture 1. emitSegments leaves records on the device (raster order, key-tagged). -2. **Stable LSD radix sort by cluster key** (8-bit digits over the 46-bit - key, per-thread-sequential workgroup ranking for stability). Stability - matters: input is raster-ordered, so a stable key sort yields clusters - whose internal point order is exactly the CPU emitter's order. -3. Cluster boundary detection (key change) -> descriptor array - (offset, count) + size/hull filters from fit_quads' caller. +2. **Permutation + gather** (P1b, replaces the P1 radix sort): the host + build walk already discovers the grouping, so it emits a permutation + (source record index per output slot, clusters contiguous, points in + raster order) and one coalesced gather kernel materializes + cluster-contiguous records on-device (bufRecordsAlt). Within-cluster + point order is exactly the CPU emitter's order by construction. +3. Per-cluster descriptor array (offset, count) uploaded alongside the + permutation + size/hull filters from fit_quads' caller. 4. Per-cluster fit, one workgroup per cluster: - bbox + center reduce; per-point slope; gradient dot -> reversed_border filter (threads parallel, local reduce). @@ -58,12 +60,26 @@ the cluster build walk disappears entirely. ~10.9 ms/pass x 6 (scattered 16 B writes + low-occupancy ranking) — a full global sort is the wrong tool. KEPT in tree as validation scaffolding; do not enable in production. -- P1b (REVISED grouping design for P3): the CPU build walk already - discovers groups for ~6 core-ms — have it emit a permutation array - (record index per output slot, cluster-contiguous) plus per-cluster - descriptors; upload (~11 MB) and run one coalesced GPU gather (~1 ms) - to materialize cluster-contiguous records on-device. Replaces the sort - entirely; ordering guarantees identical (walk preserves raster order). +- P1b (DONE, measured on w3cj 2026-06-11): permutation + gather, gated + APRILTAG_OPENCL_GATHER=1 until the GPU fit consumes it. The permutation + is computed as a counting sort, not per-cluster index lists — the list + version cost ~13 ms/frame of host bookkeeping (walk +5.8 ms from 2.7M + zarray appends, 2.7 ms single-thread flatten, 3.0 ms freeing 9k + zarrays) and was rewritten. Final shape: the walk stores each record's + task-local cluster index into a flat uint32 array (pass A, one + sequential store per record); the merge records each task-local + cluster's final index and chunk start within the final cluster; once + final cluster offsets are known (running sum of sizes, also fills the + descriptors), a parallel workerpool pass computes each record's output + slot and writes the permutation straight into the mapped staging + buffer (pass B). One coalesced gather kernel (~1.5 ms GPU, async with + downstream CPU fit) materializes cluster-contiguous records in + bufRecordsAlt. Measured host cost: descFill 0.13 ms, permPass 1.8 ms, + map/unmap ~1.1 ms — detect wall/cpu indistinguishable from no-gather. + Gates: detect PASS 0.000000 px; corpus 108/108 @ 0.0000; gathered + content vs CPU clusters (APRILTAG_OPENCL_GATHER_VALIDATE=1) 216/216. + P3 note: with fit on-GPU the walk's appendPt/merge copying disappears; + what remains on the host is the hash probe + pass A/B — the slim walk. - P2: per-cluster slope/filter/slope-sort on GPU; validate sorted point order against ptsort output (tie cases logged). - P3: lfps + maxima + combos + line fits; quads-only readback; corpus diff --git a/ocl_harness/README.md b/ocl_harness/README.md index 6d4e87e8..68965f86 100644 --- a/ocl_harness/README.md +++ b/ocl_harness/README.md @@ -9,8 +9,16 @@ Run with `OCL_ICD_VENDORS` pointing at the Intel OpenCL ICD and `LD_LIBRARY_PATH` at the built library. Environment flags: - `APRILTAG_OPENCL=1` enables the GPU frontend (silent CPU fallback). -- `APRILTAG_OPENCL_EXACT=1` bit-exact output vs the CPU path (validation). -- `APRILTAG_OPENCL_PROFILE=1` per-kernel GPU timings on stderr. + Output is bit-exact vs the CPU path by default. +- `APRILTAG_OPENCL_GATHER=1` fit-port groundwork (P1b): the build walk + emits a permutation and a GPU gather materializes cluster-contiguous + records on-device. +- `APRILTAG_OPENCL_GATHER_VALIDATE=1` reads the gathered records back and + verifies them against the CPU-built clusters (dev gate, slow). +- `APRILTAG_OPENCL_SORTED=1` P1 validation scaffolding: GPU radix-sort + grouping instead of the hash build walk. Not for production. +- `APRILTAG_OPENCL_PROFILE=1` per-kernel GPU timings + host-side frontend + stamps on stderr. - `APRILTAG_OPENCL_DEBUG=1` fallback diagnostics on stderr. - `detect_harness.c` — full-detect CPU/GPU benchmark + detection equivalence. @@ -19,3 +27,4 @@ Run with `OCL_ICD_VENDORS` pointing at the Intel OpenCL ICD and - `uf_harness.c` — GPU connected-components equivalence vs CPU unionfind. - `cluster_harness.c` — GPU cluster extraction equivalence vs gradient_clusters. - `profile_harness.c` — per-stage timeprofile of apriltag_detector_detect. +- `fp64_probe.c` — Arc 140T fp64 throughput probe in the lfps/fit shape. diff --git a/ocl_threshold.c b/ocl_threshold.c index 9c1da4a5..bcea9162 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "common/workerpool.h" @@ -310,6 +311,19 @@ static const char *sourceSort = " }\n" "}\n"; +static const char *sourceGather = + // Materialize cluster-contiguous records on-device for the GPU fit + // stages: the host build walk discovers the grouping anyway, so it emits + // a permutation (source record index per output slot, clusters + // contiguous, points in raster order) and one coalesced pass gathers the + // records into that order. Replaces a full GPU key sort. + "__kernel void gatherRecords(__global const ulong2 *in, __global const uint *perm,\n" + " uint count, __global ulong2 *out) {\n" + " uint i = get_global_id(0);\n" + " if (i >= count) return;\n" + " out[i] = in[perm[i]];\n" + "}\n"; + static const char *sourceScan = "__kernel void scanLocal(__global const uint *hist, __global uint *offsets,\n" " __global uint *blockSums) {\n" @@ -342,6 +356,8 @@ static const char *sourceScan = #define OCL_RECORD_CAPACITY (8u * 1024u * 1024u) #define OCL_SEG_COUNT 65536u #define OCL_SEG_WIDTH 256 +// 16 build tasks x at most OCL_HASH_SIZE/2 clusters each. +#define OCL_MAX_CLUSTERS (1u << 19) typedef struct { uint16_t x, y; @@ -368,6 +384,7 @@ static cl_kernel oclKernelScanBlocks; static cl_kernel oclKernelAddBlockOffsets; static cl_kernel oclKernelRadixHist; static cl_kernel oclKernelRadixScatter; +static cl_kernel oclKernelGatherRecords; typedef struct { int valid; @@ -393,6 +410,8 @@ typedef struct { cl_mem bufSegCounts; cl_mem bufSegOffsets; cl_mem bufBlockSums; + cl_mem bufPerm; + cl_mem bufClusterDesc; } OclBufferCache; static OclBufferCache cache; @@ -445,6 +464,19 @@ static void oclDebugLog(const char *message) fprintf(stderr, "apriltag opencl: %s\n", message); } +static double hostNowUs(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e6 + ts.tv_nsec / 1e3; +} + +static void profHost(const char *name, double startUs) +{ + if (profEnabled) + fprintf(stderr, " host %-11s %8.1f us\n", name, hostNowUs() - startUs); +} + static void oclInit(void) { cl_platform_id platforms[8]; @@ -474,8 +506,8 @@ static void oclInit(void) if (err != CL_SUCCESS) return; - const char *sources[7] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourceEmit, sourceSort, sourceScan }; - cl_program program = clCreateProgramWithSource(oclContext, 7, sources, NULL, &err); + const char *sources[8] = { sourceThreshold, sourceCcl, sourceCompress, sourceExtract, sourceEmit, sourceSort, sourceScan, sourceGather }; + cl_program program = clCreateProgramWithSource(oclContext, 8, sources, NULL, &err); if (err != CL_SUCCESS) return; err = clBuildProgram(program, 1, &device, "", NULL, NULL); @@ -502,6 +534,7 @@ static void oclInit(void) { &oclKernelAddBlockOffsets, "addBlockOffsets" }, { &oclKernelRadixHist, "radixHist" }, { &oclKernelRadixScatter, "radixScatter" }, + { &oclKernelGatherRecords, "gatherRecords" }, }; int failed = 0; for (size_t i = 0; i < sizeof(kernels) / sizeof(kernels[0]); i++) { @@ -541,6 +574,8 @@ static void releaseCache(void) releaseBuffer(cache.bufSegCounts); releaseBuffer(cache.bufSegOffsets); releaseBuffer(cache.bufBlockSums); + releaseBuffer(cache.bufPerm); + releaseBuffer(cache.bufClusterDesc); memset(&cache, 0, sizeof(cache)); } @@ -596,6 +631,8 @@ static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) cache.bufSegCounts = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufSegOffsets = createOrFail(CL_MEM_READ_WRITE, OCL_SEG_COUNT * 4, NULL, &failed); cache.bufBlockSums = createOrFail(CL_MEM_READ_WRITE, 256 * 4, NULL, &failed); + cache.bufPerm = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 4, NULL, &failed); + cache.bufClusterDesc = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_MAX_CLUSTERS * 8, NULL, &failed); if (failed) { releaseCache(); return 0; @@ -746,10 +783,34 @@ typedef struct { uint32_t clusterIdx; } HashEntry; +// Bookkeeping that turns the build walk's grouping into the gather +// permutation as a counting sort, with no per-cluster index storage: +// the walk records each record's task-local cluster index (one flat +// uint32 per record); the merge records, per task-local cluster, the +// final cluster index and the chunk's start offset within that final +// cluster (task-order concatenation); once final cluster offsets are +// known, a parallel pass computes each record's output slot directly. +typedef struct { + uint32_t recStart, recEnd; + int localCount; + uint32_t *finalIdx; + uint32_t *chunkStart; +} GatherTaskPlan; + +typedef struct { + int taskCount; + const uint32_t *recCluster; + GatherTaskPlan tasks[16]; +} GatherPlan; + typedef struct { const uint64_t *records; uint32_t recStart, recEnd; zarray_t *clusters; + // Per-record task-local cluster index slots (the whole flat array, + // indexed by absolute record index). NULL when no permutation is + // requested. + uint32_t *recCluster; uint64_t *clusterKeys; int clusterCap; int failed; @@ -792,6 +853,8 @@ static void doBuildTask(void *p) task->clusterKeys[clusterCount] = key; clusterCount++; } + if (task->recCluster != NULL) + task->recCluster[i] = table[slot].clusterIdx; zarray_t *cluster; zarray_get(task->clusters, (int)table[slot].clusterIdx, &cluster); appendPt(cluster, payload); @@ -810,20 +873,53 @@ static void destroyTaskClusters(BuildTask *task) free(task->clusterKeys); } +static void destroyGatherPlan(GatherPlan *plan) +{ + for (int t = 0; t < plan->taskCount; t++) { + free(plan->tasks[t].finalIdx); + free(plan->tasks[t].chunkStart); + } + plan->taskCount = 0; +} + +static void concatAndDestroy(zarray_t *dst, zarray_t *src) +{ + zarray_ensure_capacity(dst, dst->size + src->size); + memcpy(dst->data + (size_t)dst->size * dst->el_sz, src->data, + (size_t)src->size * src->el_sz); + dst->size += src->size; + zarray_destroy(src); +} + // Merge per-task clusters in task order: tasks cover ascending row ranges, -// so concatenation preserves raster point order within each cluster. -static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount) +// so concatenation preserves raster point order within each cluster. When +// a gather plan is requested, the merge also records where each task-local +// cluster lands: its final cluster index and its chunk's start offset +// within that final cluster. +static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount, GatherPlan *plan) { zarray_t *clusters = zarray_create(sizeof(zarray_t *)); HashEntry *table = calloc(OCL_HASH_SIZE, sizeof(HashEntry)); if (table == NULL) { for (int t = 0; t < taskCount; t++) destroyTaskClusters(&tasks[t]); + if (plan != NULL) + plan->taskCount = 0; return clusters; } for (int t = 0; t < taskCount; t++) { - for (int i = 0; i < zarray_size(tasks[t].clusters); i++) { + const int localCount = zarray_size(tasks[t].clusters); + GatherTaskPlan *taskPlan = NULL; + if (plan != NULL) { + taskPlan = &plan->tasks[t]; + taskPlan->recStart = tasks[t].recStart; + taskPlan->recEnd = tasks[t].recEnd; + taskPlan->localCount = localCount; + taskPlan->finalIdx = malloc(sizeof(uint32_t) * (size_t)(localCount > 0 ? localCount : 1)); + taskPlan->chunkStart = malloc(sizeof(uint32_t) * (size_t)(localCount > 0 ? localCount : 1)); + } + for (int i = 0; i < localCount; i++) { zarray_t *cluster; zarray_get(tasks[t].clusters, i, &cluster); uint64_t key = tasks[t].clusterKeys[i]; @@ -833,32 +929,53 @@ static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount) if (table[slot].key == 0) { table[slot].key = key; table[slot].clusterIdx = (uint32_t)zarray_size(clusters); + if (taskPlan != NULL) { + taskPlan->finalIdx[i] = table[slot].clusterIdx; + taskPlan->chunkStart[i] = 0; + } zarray_add(clusters, &cluster); } else { zarray_t *dst; zarray_get(clusters, (int)table[slot].clusterIdx, &dst); - zarray_ensure_capacity(dst, dst->size + cluster->size); - memcpy(dst->data + (size_t)dst->size * dst->el_sz, cluster->data, - (size_t)cluster->size * cluster->el_sz); - dst->size += cluster->size; - zarray_destroy(cluster); + if (taskPlan != NULL) { + taskPlan->finalIdx[i] = table[slot].clusterIdx; + taskPlan->chunkStart[i] = (uint32_t)dst->size; + } + concatAndDestroy(dst, cluster); } } zarray_destroy(tasks[t].clusters); free(tasks[t].clusterKeys); } free(table); + if (plan != NULL) + plan->taskCount = taskCount; return clusters; } +static uint32_t *recClusterScratch = NULL; +static uint32_t recClusterScratchCap = 0; + static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount, - int segsPerRow, cl_int h) + int segsPerRow, cl_int h, GatherPlan *planOut) { int taskCount = (td->wp != NULL && td->nthreads > 1) ? td->nthreads : 1; if (taskCount > 16) taskCount = 16; BuildTask tasks[16]; + uint32_t *recCluster = NULL; + if (planOut != NULL) { + planOut->taskCount = 0; + if (recClusterScratchCap < recordCount) { + free(recClusterScratch); + recClusterScratch = malloc(sizeof(uint32_t) * (size_t)recordCount); + recClusterScratchCap = (recClusterScratch != NULL) ? recordCount : 0; + } + recCluster = recClusterScratch; + planOut->recCluster = recCluster; + } + // Split rows into contiguous ranges balanced by record count; row r's // records start at segOffsetsHost[r * segsPerRow]. uint32_t targetPerTask = recordCount / (uint32_t)taskCount + 1; @@ -868,6 +985,7 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, tasks[t].records = records; tasks[t].recStart = recStart; tasks[t].clusters = zarray_create(sizeof(zarray_t *)); + tasks[t].recCluster = recCluster; tasks[t].clusterCap = 256; tasks[t].clusterKeys = malloc(sizeof(uint64_t) * tasks[t].clusterCap); tasks[t].failed = 0; @@ -896,7 +1014,7 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, return NULL; } } - return mergeTaskClusters(tasks, taskCount); + return mergeTaskClusters(tasks, taskCount, (recCluster != NULL) ? planOut : NULL); } // Stable 6-pass LSD radix sort of the record buffer by the compacted 46-bit @@ -963,6 +1081,172 @@ static int sortRecords(uint32_t recordCount) return cur == cache.bufRecords; } +typedef struct { + const GatherTaskPlan *taskPlan; + const uint32_t *recCluster; + const uint32_t *finalOffsets; + uint32_t *perm; + int failed; +} PermTask; + +// Each record's output slot: its chunk's absolute start (final cluster +// offset + chunk offset within the cluster) plus its rank within the +// chunk, which ascending record order provides for free. Tasks own +// disjoint record ranges and disjoint output chunks. +static void doPermTask(void *p) +{ + PermTask *task = (PermTask *)p; + const GatherTaskPlan *taskPlan = task->taskPlan; + const int localCount = taskPlan->localCount > 0 ? taskPlan->localCount : 1; + uint32_t *cursor = malloc(sizeof(uint32_t) * (size_t)localCount); + if (cursor == NULL) { + task->failed = 1; + return; + } + for (int c = 0; c < taskPlan->localCount; c++) + cursor[c] = task->finalOffsets[taskPlan->finalIdx[c]] + taskPlan->chunkStart[c]; + for (uint32_t i = taskPlan->recStart; i < taskPlan->recEnd; i++) + task->perm[cursor[task->recCluster[i]]++] = i; + free(cursor); +} + +// P1b of the GPU fit_quads port: turn the build walk's grouping into a +// permutation array plus per-cluster (offset, count) descriptors in the +// mapped staging buffers, then gather the records into cluster-contiguous +// order in cache.bufRecordsAlt — the layout the GPU fit stages consume. +// Caller holds oclMutex. Returns 0 on failure. +static int gatherClusterRecords(apriltag_detector_t *td, zarray_t *clusters, GatherPlan *plan, + uint32_t recordCount) +{ + const uint32_t clusterCount = (uint32_t)zarray_size(clusters); + if (clusterCount == 0 || clusterCount > OCL_MAX_CLUSTERS || plan->taskCount == 0) + return 0; + uint32_t *finalOffsets = malloc(sizeof(uint32_t) * (size_t)clusterCount); + if (finalOffsets == NULL) + return 0; + + cl_int err = CL_SUCCESS; + double t = hostNowUs(); + uint32_t *perm = clEnqueueMapBuffer(oclQueue, cache.bufPerm, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, + (size_t)recordCount * 4, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + free(finalOffsets); + return 0; + } + uint32_t *desc = clEnqueueMapBuffer(oclQueue, cache.bufClusterDesc, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, + (size_t)clusterCount * 8, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clEnqueueUnmapMemObject(oclQueue, cache.bufPerm, perm, 0, NULL, NULL); + free(finalOffsets); + return 0; + } + profHost("mapPerm", t); + + t = hostNowUs(); + uint32_t slot = 0; + for (uint32_t c = 0; c < clusterCount; c++) { + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + finalOffsets[c] = slot; + desc[2 * c] = slot; + desc[2 * c + 1] = (uint32_t)cluster->size; + slot += (uint32_t)cluster->size; + } + int complete = slot == recordCount; + profHost("descFill", t); + + t = hostNowUs(); + PermTask permTasks[16]; + if (complete) { + for (int pt = 0; pt < plan->taskCount; pt++) { + permTasks[pt].taskPlan = &plan->tasks[pt]; + permTasks[pt].recCluster = plan->recCluster; + permTasks[pt].finalOffsets = finalOffsets; + permTasks[pt].perm = perm; + permTasks[pt].failed = 0; + } + if (plan->taskCount == 1) { + doPermTask(&permTasks[0]); + } else { + for (int pt = 0; pt < plan->taskCount; pt++) + workerpool_add_task(td->wp, doPermTask, &permTasks[pt]); + workerpool_run(td->wp); + } + for (int pt = 0; pt < plan->taskCount; pt++) + complete &= permTasks[pt].failed == 0; + } + free(finalOffsets); + profHost("permPass", t); + + t = hostNowUs(); + err = clEnqueueUnmapMemObject(oclQueue, cache.bufPerm, perm, 0, NULL, NULL); + err |= clEnqueueUnmapMemObject(oclQueue, cache.bufClusterDesc, desc, 0, NULL, NULL); + if (err != CL_SUCCESS || !complete) + return 0; + profHost("unmapPerm", t); + + err |= clSetKernelArg(oclKernelGatherRecords, 0, sizeof(cl_mem), &cache.bufRecords); + err |= clSetKernelArg(oclKernelGatherRecords, 1, sizeof(cl_mem), &cache.bufPerm); + err |= clSetKernelArg(oclKernelGatherRecords, 2, sizeof(cl_uint), &recordCount); + err |= clSetKernelArg(oclKernelGatherRecords, 3, sizeof(cl_mem), &cache.bufRecordsAlt); + if (err != CL_SUCCESS) + return 0; + t = hostNowUs(); + const size_t gatherGlobal[1] = { roundUp(recordCount, 256) }; + err = clEnqueueNDRangeKernel(oclQueue, oclKernelGatherRecords, 1, NULL, gatherGlobal, NULL, + 0, NULL, profSlot("gather")); + if (err != CL_SUCCESS) + return 0; + profHost("gatherEnq", t); + if (profEnabled) + clFinish(oclQueue); + return 1; +} + +// Development gate for the gather path: read the gathered records back and +// check each cluster's range carries a uniform key and reproduces the +// cluster's points in order. Reports on stderr. +static void validateGather(zarray_t *clusters, uint32_t recordCount) +{ + cl_int err = CL_SUCCESS; + const uint64_t *gathered = clEnqueueMapBuffer(oclQueue, cache.bufRecordsAlt, CL_TRUE, CL_MAP_READ, + 0, (size_t)recordCount * 16, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + fprintf(stderr, "apriltag opencl: gather validate: map failed\n"); + return; + } + + uint64_t mismatches = 0; + uint32_t slot = 0; + for (int c = 0; c < zarray_size(clusters); c++) { + zarray_t *cluster; + zarray_get(clusters, c, &cluster); + const uint64_t clusterKey = gathered[2 * slot]; + for (int j = 0; j < zarray_size(cluster); j++, slot++) { + const OclPt *pt = (const OclPt *)(cluster->data + (size_t)j * cluster->el_sz); + const uint64_t key = gathered[2 * slot]; + const uint64_t payload = gathered[2 * slot + 1]; + const int ok = key == clusterKey && + pt->x == (uint16_t)(payload >> 48) && + pt->y == (uint16_t)(payload >> 32) && + pt->gx == (int16_t)(uint16_t)(payload >> 16) && + pt->gy == (int16_t)(uint16_t)payload; + if (!ok) + mismatches++; + } + } + clEnqueueUnmapMemObject(oclQueue, cache.bufRecordsAlt, (void *)gathered, 0, NULL, NULL); + + if (mismatches == 0 && slot == recordCount) + fprintf(stderr, "apriltag opencl: gather validate: PASS (%d clusters, %u records)\n", + zarray_size(clusters), recordCount); + else + fprintf(stderr, "apriltag opencl: gather validate: FAIL (%llu mismatches, %u/%u records)\n", + (unsigned long long)mismatches, slot, recordCount); +} + typedef struct { const uint64_t *records; uint32_t recStart, recEnd; @@ -1147,20 +1431,44 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl // P1 scaffolding for the GPU fit_quads port: sort records by cluster key // on the GPU so groups are contiguous (within-group raster order is - // preserved by sort stability). Gated until the GPU fit lands. + // preserved by sort stability). Superseded by the gather path below; + // kept as validation scaffolding. int useSorted = getenv("APRILTAG_OPENCL_SORTED") != NULL; + // P1b: the hash build walk emits a permutation so one GPU gather + // materializes cluster-contiguous records on-device for the fit stages. + // Gated until the GPU fit lands; mutually exclusive with the sorted + // path, whose cluster order differs from the walk's encounter order. + int useGather = !useSorted && getenv("APRILTAG_OPENCL_GATHER") != NULL; if (useSorted && !sortRecords(recordCount)) goto done; + double t = hostNowUs(); void *mapped = clEnqueueMapBuffer(oclQueue, cache.bufRecords, CL_TRUE, CL_MAP_READ, 0, (size_t)recordCount * 16, 0, NULL, profSlot("mapRecords"), &err); if (err != CL_SUCCESS) goto done; + profHost("mapWait", t); + t = hostNowUs(); + GatherPlan plan = { 0 }; if (useSorted) clusters = buildClustersSorted(td, (const uint64_t *)mapped, recordCount); else - clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch); + clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch, + useGather ? &plan : NULL); + profHost("buildWalk", t); + t = hostNowUs(); clEnqueueUnmapMemObject(oclQueue, cache.bufRecords, mapped, 0, NULL, NULL); + profHost("unmapRecords", t); + + if (useGather && clusters != NULL && plan.taskCount > 0) { + if (gatherClusterRecords(td, clusters, &plan, recordCount)) { + if (getenv("APRILTAG_OPENCL_GATHER_VALIDATE") != NULL) + validateGather(clusters, recordCount); + } else { + oclDebugLog("gather failed"); + } + } + destroyGatherPlan(&plan); done: return clusters; From 3d77cc3418985c7b6065df47b7d87dd1640e4505 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 10:41:59 +0200 Subject: [PATCH 07/18] Add GPU fit preparation and exact-ptsort slope sort (P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three kernels over the gathered cluster-contiguous records, gated APRILTAG_OPENCL_FIT=1: fitPrep replicates the do_quad_task/fit_quad filter cascade, bbox, double-evaluated center, per-point slopes, and the gradient dot (parallel terms, then one lane sums them in point order); fitSortSlm and fitSortBig sort each cluster's keys by slope. Slope ties turned out pervasive — 2266 of 2510 sorted clusters on the vide frame — so a total-order sort would reorder points in ~88% of clusters and break downstream bit-exactness. The sort instead replicates ptsort's exact comparison network: its recursion tree is arithmetic on the cluster size (floor-half splits terminating at the verbatim <=5 sorting networks), internal nodes run the verbatim right-biased merge, and depth parity ping-pongs between two buffers. Big clusters batch one workgroup each with scratch slices, splitting shallow merges across lanes via an exact merge-path search. Bit-exactness also needs FP_CONTRACT OFF (the CPU build has no FMA) and correctly-rounded fp32 division, both handled by a separate fp64 fit program. Validation (APRILTAG_OPENCL_FIT_VALIDATE=1) checks flags, center/dot bits, bbox, and the full sorted sequence against a host replication including a verbatim ptsort: 25/25 on the vide frame, 216/216 across the corpus, tie order matching ptsort exactly. Detect stays 0.000000 px and corpus 108/108. Kernel times: fitPrep 3.7 ms, fitSortSlm 4.8 ms, fitSortBig 4x0.6 ms (the first monolithic cut cost 30 ms; 33 KB of SLM per workgroup crushed occupancy). Co-Authored-By: Claude Fable 5 --- ocl_harness/FIT_QUADS_PLAN.md | 58 ++- ocl_harness/README.md | 6 + ocl_threshold.c | 810 +++++++++++++++++++++++++++++++++- 3 files changed, 861 insertions(+), 13 deletions(-) diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index b4d7a0d7..015514c3 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -42,14 +42,22 @@ the cluster build walk disappears entirely. ## Exactness contract -- Doubles end-to-end; per-cluster serial accumulation preserves the CPU's - floating-point operation order, so corners should match to the last bit - except where slope TIES exist: CPU ptsort's tie order is a sorting-network - artifact; the GPU sorts by (slope, raster index), a total order. The - fit_quad center-noise constants make exact float slope ties rare; the - corpus harness quantifies any residual difference. Contract: identical - detection sets; corners < 1e-6 px except tie-affected clusters. -- Determinism run-to-run: total-order sort + fixed reduction shapes = yes. +- Doubles where the CPU evaluates in double; per-cluster serial + accumulation preserves the CPU's floating-point operation order. +- Slope ties are NOT rare — the center-noise constants do not prevent + them (vide frame: 2266 of 2510 sorted clusters carry ties, 199k tied + pairs; consistent with PERF_NOTES in the faster branch). A total-order + sort therefore reorders points in ~88% of clusters and would break + bit-exactness downstream. The GPU sort instead REPLICATES ptsort's + exact comparison network (its recursion tree is pure arithmetic on the + cluster size), so tie order matches the CPU bit for bit and the + contract strengthens to: identical detections, corners bit-identical. +- Float details that matter: FP_CONTRACT OFF (the CPU build has no FMA), + -cl-fp32-correctly-rounded-divide-sqrt for the slope divide, the + center computed in double then narrowed exactly as the CPU expression, + and the gradient dot summed serially in point order from + parallel-computed terms. +- Determinism run-to-run: fixed network + fixed reduction shapes = yes. ## Phases (each gated on the corpus harness) @@ -80,10 +88,36 @@ the cluster build walk disappears entirely. content vs CPU clusters (APRILTAG_OPENCL_GATHER_VALIDATE=1) 216/216. P3 note: with fit on-GPU the walk's appendPt/merge copying disappears; what remains on the host is the hash probe + pass A/B — the slim walk. -- P2: per-cluster slope/filter/slope-sort on GPU; validate sorted point - order against ptsort output (tie cases logged). -- P3: lfps + maxima + combos + line fits; quads-only readback; corpus - detection equivalence + timing. +- P2 (DONE, measured on w3cj 2026-06-11): per-cluster preparation and + slope sort, gated APRILTAG_OPENCL_FIT=1 (implies the gather path). + Three kernels over the gathered records + descriptors: + - fitPrep: one WG per cluster, tiny SLM for occupancy — the + do_quad_task/fit_quad filter cascade, bbox reduce, double-evaluated + center, per-point slope keys (orderedSlopeBits<<32 | point index), + parallel dot terms, then one lane sums the terms in point order + (bit-exact dot) and writes flags/meta. + - fitSortSlm: ptsort-replica sort in two 4 KB SLM buffers for clusters + of at most FIT_SLM_CAP=512 points (host-prefiltered id list). + - fitSortBig: same replica in global memory for bigger clusters, + batched FIT_BATCH=256 workgroups per launch with per-WG scratch + slices; shallow depths split each merge across lanes with an exact + right-biased merge-path search. + The first cut ran the sort inside a monolithic fitPrep: 30 ms — SLM + footprint (33 KB/WG) crushed occupancy and serial global merges burned + the rest (a FIT_SLM_CAP=512 experiment halved it, proving occupancy). + The split gets fitPrep 3.7 ms + fitSortSlm 4.8 ms + fitSortBig 4x0.6 ms + (~11 ms chain, +8 ms detect wall over gather-only — it hides behind + the CPU fit stage it will replace in P3). Gates: detect 0.000000 px; + corpus 108/108; fit validation (APRILTAG_OPENCL_FIT_VALIDATE=1, host + replication incl. a verbatim ValPt ptsort) 25/25 on the vide frame and + 216/216 across the corpus — flags, center/dot bits, bbox, and the full + sorted sequence including every tie cluster match ptsort exactly. + Still open (P3 polish): merge-path for the SLM sort's top levels; + trim fitSortBig's list (host lists border-undecided candidates). +- P3: lfps + maxima + combos + line fits consuming the sorted keys + (point index in the low half indexes bufRecordsAlt); quads-only + readback; corpus detection equivalence + timing. With ptsort order + replicated, the target is bit-identical corners everywhere. ## Also still open (smaller) diff --git a/ocl_harness/README.md b/ocl_harness/README.md index 68965f86..dd59e044 100644 --- a/ocl_harness/README.md +++ b/ocl_harness/README.md @@ -15,6 +15,12 @@ Run with `OCL_ICD_VENDORS` pointing at the Intel OpenCL ICD and records on-device. - `APRILTAG_OPENCL_GATHER_VALIDATE=1` reads the gathered records back and verifies them against the CPU-built clusters (dev gate, slow). +- `APRILTAG_OPENCL_FIT=1` fit-port P2 (implies the gather path): GPU + per-cluster filter cascade, center, gradient dot, and a slope sort + replicating CPU ptsort exactly, tie order included. +- `APRILTAG_OPENCL_FIT_VALIDATE=1` checks every cluster's flags, center + and dot bits, and sorted order against a host replication of the CPU + pre-sort semantics (dev gate, slow). - `APRILTAG_OPENCL_SORTED=1` P1 validation scaffolding: GPU radix-sort grouping instead of the hash build walk. Not for production. - `APRILTAG_OPENCL_PROFILE=1` per-kernel GPU timings + host-side frontend diff --git a/ocl_threshold.c b/ocl_threshold.c index bcea9162..c434d93e 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -324,6 +324,306 @@ static const char *sourceGather = " out[i] = in[perm[i]];\n" "}\n"; +static const char *sourceFitPrep = + "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n" + "#pragma OPENCL FP_CONTRACT OFF\n" + // Per-cluster preparation replicating fit_quad's pre-sort steps with + // identical arithmetic (the CPU build uses no FMA contraction, hence + // FP_CONTRACT OFF): the do_quad_task/fit_quad filter cascade, bbox, + // center (double then float, as the CPU's mixed expression evaluates), + // per-point slopes, the gradient dot accumulated serially in point + // order, and the slope sort. Keys are (orderedSlopeBits, point index); + // comparisons use the slope half only and the sort replicates ptsort's + // exact network — slope ties are pervasive (~88% of sorted clusters on + // the vide frame), so matching ptsort's tie order is what keeps the + // downstream fit bit-exact. Flag bits mirror the FIT_* defines in + // ocl_threshold.c; FIT_SLM_CAP/FIT_BIG_CAP arrive via build options. + "#define PROCESSED (1u<<0)\n" + "#define SKIP_MINPIX (1u<<1)\n" + "#define SKIP_PERIM (1u<<2)\n" + "#define SKIP_AREA (1u<<3)\n" + "#define SKIP_BORDER (1u<<4)\n" + "#define REVERSED (1u<<5)\n" + "#define SORT_GLOBAL (1u<<6)\n" + "#define SKIP_TOOBIG (1u<<7)\n" + "#define SORTED (1u<<8)\n" + "#define SORT_SLM (1u<<9)\n" + "inline uint orderedFloatBits(float f) {\n" + " uint u = as_uint(f);\n" + " return (u & 0x80000000u) ? ~u : (u | 0x80000000u);\n" + "}\n" + "inline float cpuSlope(float fx, float fy, float cx, float cy) {\n" + " float dx = fx - cx;\n" + " float dy = fy - cy;\n" + " float quadrant = (dy > 0) ? ((dx > 0) ? 65536.0f : 131072.0f)\n" + " : ((dx > 0) ? 0.0f : -65536.0f);\n" + " if (dy < 0) { dy = -dy; dx = -dx; }\n" + " if (dx < 0) { float tmp = dx; dx = dy; dy = -tmp; }\n" + " return quadrant + dy / dx;\n" + "}\n" + "inline void writeMeta(__global uint *meta, uint c, uint flags, float cx, float cy,\n" + " float dot, uint xmin, uint xmax, uint ymin, uint ymax, uint n) {\n" + " __global uint *m = meta + 8u * c;\n" + " m[0] = flags; m[1] = as_uint(cx); m[2] = as_uint(cy); m[3] = as_uint(dot);\n" + " m[4] = xmin | (xmax << 16); m[5] = ymin | (ymax << 16); m[6] = n; m[7] = 0u;\n" + "}\n" + "inline uint reduceMin(__local uint *sred, int lid, uint v) {\n" + " sred[lid] = v;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int s = 128; s > 0; s >>= 1) {\n" + " if (lid < s) sred[lid] = min(sred[lid], sred[lid + s]);\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " uint r = sred[0];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " return r;\n" + "}\n" + "inline uint reduceMax(__local uint *sred, int lid, uint v) {\n" + " sred[lid] = v;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int s = 128; s > 0; s >>= 1) {\n" + " if (lid < s) sred[lid] = max(sred[lid], sred[lid + s]);\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " uint r = sred[0];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " return r;\n" + "}\n"; + +static const char *sourceFitSortHelpers = + // ptsort's recursion tree is pure arithmetic on the cluster size: + // floor-half/rest splits, terminating at five or fewer elements where + // the CPU runs fixed sorting networks. ptsortNode locates the node for + // a (depth, path) pair; net5 replicates the exact swap sequences and + // ptsortMerge the exact right-biased two-pointer merge (ties take the + // right run). Comparisons use only the ordered-slope half of the key. + "#define KGT(a, b) ((uint)((a) >> 32) > (uint)((b) >> 32))\n" + "#define KLT(a, b) ((uint)((a) >> 32) < (uint)((b) >> 32))\n" + "inline int ptsortNode(uint n, uint depth, uint path, uint *outOff, uint *outSize) {\n" + " uint off = 0, m = n;\n" + " for (uint b = 0; b < depth; b++) {\n" + " if (m <= 5u) return 0;\n" + " uint hsz = m / 2u;\n" + " if ((path >> (depth - 1u - b)) & 1u) { off += hsz; m -= hsz; }\n" + " else { m = hsz; }\n" + " }\n" + " *outOff = off;\n" + " *outSize = m;\n" + " return 1;\n" + "}\n" + "#define SW(i, j) if (KGT(p[i], p[j])) { t = p[i]; p[i] = p[j]; p[j] = t; }\n" + "#define NET5_BODY \\\n" + " ulong t; \\\n" + " if (m == 2u) { SW(0, 1) } \\\n" + " else if (m == 3u) { SW(0, 1) SW(1, 2) SW(0, 1) } \\\n" + " else if (m == 4u) { SW(0, 1) SW(2, 3) SW(0, 2) SW(1, 3) SW(1, 2) } \\\n" + " else if (m == 5u) { SW(0, 1) SW(3, 4) SW(1, 2) SW(0, 1) SW(0, 3) SW(2, 4) SW(1, 2) SW(2, 3) SW(1, 2) }\n" + "inline void net5L(__local ulong *p, uint m) { NET5_BODY }\n" + "inline void net5G(__global ulong *p, uint m) { NET5_BODY }\n" + "#define MERGE_BODY \\\n" + " uint roff = loff + lsz; \\\n" + " uint i = 0, j = 0, o = loff; \\\n" + " while (i < lsz && j < rsz) { \\\n" + " ulong a = src[loff + i]; \\\n" + " ulong b = src[roff + j]; \\\n" + " if (KLT(a, b)) { dst[o++] = a; i++; } \\\n" + " else { dst[o++] = b; j++; } \\\n" + " } \\\n" + " while (i < lsz) dst[o++] = src[loff + i++]; \\\n" + " while (j < rsz) dst[o++] = src[roff + j++];\n" + "inline void ptsortMergeL(__local const ulong *src, __local ulong *dst, uint loff, uint lsz, uint rsz) { MERGE_BODY }\n" + "inline void ptsortMergeG(__global const ulong *src, __global ulong *dst, uint loff, uint lsz, uint rsz) { MERGE_BODY }\n" + // Merge-path split for lane-parallel merges: returns how many elements + // of a the exact right-biased serial merge consumes among its first k + // outputs, so a lane can start mid-merge and produce an identical + // output chunk. + "inline uint mergePathSearch(__global const ulong *a, uint asz, __global const ulong *b, uint bsz, uint k) {\n" + " uint lo = (k > bsz) ? (k - bsz) : 0u;\n" + " uint hi = (k < asz) ? k : asz;\n" + " while (lo < hi) {\n" + " uint mid = (lo + hi) >> 1u;\n" + " if (KLT(a[mid], b[k - mid - 1u])) lo = mid + 1u; else hi = mid;\n" + " }\n" + " return lo;\n" + "}\n" + // One ptsort depth pass: leaves copy (when the parity differs from the + // input buffer) and run their network, internal nodes merge their + // children from the opposite-parity buffer. Depth-d results land in + // bufA when d is even, bufB when odd; the raster-order input lives in + // bufA, and a leaf's bufA region is untouched by other nodes' merges + // until its own depth is processed. + "#define DEPTH_BODY(MERGEFN, NETFN) \\\n" + " for (uint node = (uint)lid; node < (1u << d); node += 256u) { \\\n" + " uint noff, nsz; \\\n" + " if (!ptsortNode(n, (uint)d, node, &noff, &nsz)) continue; \\\n" + " int even = (d & 1) == 0; \\\n" + " if (nsz <= 5u) { \\\n" + " if (!even) for (uint q = 0; q < nsz; q++) bufB[noff + q] = bufA[noff + q]; \\\n" + " if (even) NETFN(bufA + noff, nsz); else NETFN(bufB + noff, nsz); \\\n" + " } else { \\\n" + " uint hsz = nsz / 2u; \\\n" + " if (even) MERGEFN(bufB, bufA, noff, hsz, nsz - hsz); \\\n" + " else MERGEFN(bufA, bufB, noff, hsz, nsz - hsz); \\\n" + " } \\\n" + " }\n"; + +static const char *sourceFitPrep2 = + // Preparation only — the sort runs in fitSortSlm/fitSortBig so this + // kernel keeps a tiny SLM footprint and full occupancy. The gradient + // dot's per-point terms are computed in parallel; one lane then sums + // the precomputed terms in cluster point order, which reproduces the + // CPU's float accumulation exactly. + "__kernel void fitPrep(__global const ulong2 *records, __global const uint2 *desc,\n" + " uint clusterCount, int minClusterPixels, int perimCap, int tagWidth,\n" + " int normalAllowed, int reversedAllowed,\n" + " __global ulong *keys, __global uint *meta, __global float *terms) {\n" + " uint c = get_group_id(0);\n" + " if (c >= clusterCount) return;\n" + " uint off = desc[c].x;\n" + " uint n = desc[c].y;\n" + " int lid = get_local_id(0);\n" + " __local uint sred[256];\n" + " uint flags = PROCESSED;\n" + " if ((int)n < minClusterPixels) flags |= SKIP_MINPIX;\n" + " else if ((int)n > perimCap) flags |= SKIP_PERIM;\n" + " if (flags != PROCESSED) {\n" + " if (lid == 0) writeMeta(meta, c, flags, 0.0f, 0.0f, 0.0f, 0u, 0u, 0u, 0u, n);\n" + " return;\n" + " }\n" + " uint lxmin = 65535u, lxmax = 0u, lymin = 65535u, lymax = 0u;\n" + " for (uint i = (uint)lid; i < n; i += 256u) {\n" + " ulong payload = records[off + i].y;\n" + " uint px = (uint)((payload >> 48) & 0xFFFFul);\n" + " uint py = (uint)((payload >> 32) & 0xFFFFul);\n" + " lxmin = min(lxmin, px); lxmax = max(lxmax, px);\n" + " lymin = min(lymin, py); lymax = max(lymax, py);\n" + " }\n" + " uint xmin = reduceMin(sred, lid, lxmin);\n" + " uint xmax = reduceMax(sred, lid, lxmax);\n" + " uint ymin = reduceMin(sred, lid, lymin);\n" + " uint ymax = reduceMax(sred, lid, lymax);\n" + " if ((int)(xmax - xmin) * (int)(ymax - ymin) < tagWidth) {\n" + " if (lid == 0) writeMeta(meta, c, flags | SKIP_AREA, 0.0f, 0.0f, 0.0f, xmin, xmax, ymin, ymax, n);\n" + " return;\n" + " }\n" + " float cx = (float)((xmin + xmax) * 0.5 + 0.05118);\n" + " float cy = (float)((ymin + ymax) * 0.5 - 0.028581);\n" + " for (uint i = (uint)lid; i < n; i += 256u) {\n" + " ulong payload = records[off + i].y;\n" + " float fx = (float)((payload >> 48) & 0xFFFFul);\n" + " float fy = (float)((payload >> 32) & 0xFFFFul);\n" + " float gx = (float)as_short((ushort)((payload >> 16) & 0xFFFFul));\n" + " float gy = (float)as_short((ushort)(payload & 0xFFFFul));\n" + " float slope = cpuSlope(fx, fy, cx, cy);\n" + " keys[off + i] = ((ulong)orderedFloatBits(slope) << 32) | (ulong)i;\n" + " float dx = fx - cx;\n" + " float dy = fy - cy;\n" + " terms[off + i] = dx * gx + dy * gy;\n" + " }\n" + " barrier(CLK_GLOBAL_MEM_FENCE);\n" + " if (lid == 0) {\n" + " float dot = 0.0f;\n" + " for (uint i = 0; i < n; i++) dot += terms[off + i];\n" + " int rev = dot < 0;\n" + " if (rev) flags |= REVERSED;\n" + " if (rev ? !reversedAllowed : !normalAllowed) flags |= SKIP_BORDER;\n" + " if ((flags & SKIP_BORDER) == 0) {\n" + " if (n <= (uint)FIT_SLM_CAP) flags |= SORT_SLM;\n" + " else if (n <= (uint)FIT_BIG_CAP) flags |= SORT_GLOBAL;\n" + " else flags |= SKIP_TOOBIG;\n" + " }\n" + " writeMeta(meta, c, flags, cx, cy, dot, xmin, xmax, ymin, ymax, n);\n" + " }\n" + "}\n"; + +static const char *sourceFitSortSlm = + // Slope sort for SLM-sized clusters (ids in sortList): the ptsort + // replica over two SLM buffers. + "__kernel void fitSortSlm(__global ulong *keys, __global const uint2 *desc,\n" + " __global uint *meta, __global const uint *sortList, uint count) {\n" + " uint g = get_group_id(0);\n" + " if (g >= count) return;\n" + " uint c = sortList[g];\n" + " uint flags = meta[8u * c];\n" + " if ((flags & SORT_SLM) == 0u) return;\n" + " uint off = desc[c].x;\n" + " uint n = desc[c].y;\n" + " int lid = get_local_id(0);\n" + " __local ulong skeysA[FIT_SLM_CAP];\n" + " __local ulong skeysB[FIT_SLM_CAP];\n" + " for (uint i = (uint)lid; i < n; i += 256u) skeysA[i] = keys[off + i];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " __local ulong *bufA = skeysA;\n" + " __local ulong *bufB = skeysB;\n" + " for (int d = 9; d >= 0; d--) {\n" + " DEPTH_BODY(ptsortMergeL, net5L)\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " for (uint i = (uint)lid; i < n; i += 256u) keys[off + i] = skeysA[i];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " if (lid == 0) meta[8u * c] = (flags & ~SORT_SLM) | SORTED;\n" + "}\n"; + +static const char *sourceFitSort = + // Slope sort for clusters too large for SLM: one workgroup per + // oversized cluster (ids in the big segment of sortList, one batch per + // launch), running the same ptsort replica in global memory. The + // cluster's keys segment is bufA (raster-order input and final output), + // the workgroup's scratch slice is bufB. Deep levels assign one lane + // per node; shallow levels (few, large merges) split each merge across + // lanes with mergePathSearch, which reproduces the exact serial output. + "__kernel void fitSortBig(__global ulong *keys, __global const uint2 *desc,\n" + " __global uint *meta, __global const uint *sortList,\n" + " uint baseIdx, uint count, __global ulong *scratch) {\n" + " uint g = get_group_id(0);\n" + " if (g >= count) return;\n" + " uint c = sortList[baseIdx + g];\n" + " uint flags = meta[8u * c];\n" + " if ((flags & SORT_GLOBAL) == 0u) return;\n" + " uint n = desc[c].y;\n" + " int lid = get_local_id(0);\n" + " __global ulong *bufA = keys + desc[c].x;\n" + " __global ulong *bufB = scratch + (size_t)g * (size_t)FIT_BIG_CAP;\n" + " for (int d = 13; d >= 0; d--) {\n" + " if ((1u << d) >= 256u) {\n" + " DEPTH_BODY(ptsortMergeG, net5G)\n" + " } else {\n" + " uint lanesPerNode = 256u >> d;\n" + " uint node = (uint)lid / lanesPerNode;\n" + " uint lane = (uint)lid % lanesPerNode;\n" + " uint noff, nsz;\n" + " if (ptsortNode(n, (uint)d, node, &noff, &nsz)) {\n" + " int even = (d & 1) == 0;\n" + " __global ulong *src = even ? bufB : bufA;\n" + " __global ulong *dst = even ? bufA : bufB;\n" + " if (nsz <= 5u) {\n" + " if (lane == 0) {\n" + " if (!even) for (uint q = 0; q < nsz; q++) bufB[noff + q] = bufA[noff + q];\n" + " if (even) net5G(bufA + noff, nsz); else net5G(bufB + noff, nsz);\n" + " }\n" + " } else {\n" + " uint hsz = nsz / 2u;\n" + " uint rsz = nsz - hsz;\n" + " uint chunk = (nsz + lanesPerNode - 1u) / lanesPerNode;\n" + " uint k0 = lane * chunk;\n" + " if (k0 < nsz) {\n" + " uint k1 = (k0 + chunk < nsz) ? (k0 + chunk) : nsz;\n" + " uint ai = mergePathSearch(src + noff, hsz, src + noff + hsz, rsz, k0);\n" + " uint bi = k0 - ai;\n" + " for (uint k = k0; k < k1; k++) {\n" + " int takeA = (ai < hsz) && ((bi >= rsz) || KLT(src[noff + ai], src[noff + hsz + bi]));\n" + " if (takeA) { dst[noff + k] = src[noff + ai]; ai++; }\n" + " else { dst[noff + k] = src[noff + hsz + bi]; bi++; }\n" + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + " barrier(CLK_GLOBAL_MEM_FENCE);\n" + " }\n" + " if (lid == 0) meta[8u * c] = (flags & ~SORT_GLOBAL) | SORTED;\n" + "}\n"; + static const char *sourceScan = "__kernel void scanLocal(__global const uint *hist, __global uint *offsets,\n" " __global uint *blockSums) {\n" @@ -359,6 +659,23 @@ static const char *sourceScan = // 16 build tasks x at most OCL_HASH_SIZE/2 clusters each. #define OCL_MAX_CLUSTERS (1u << 19) +// Mirrors the flag defines in sourceFitPrep/sourceFitSort — keep in sync. +#define FIT_PROCESSED (1u << 0) +#define FIT_SKIP_MINPIX (1u << 1) +#define FIT_SKIP_PERIM (1u << 2) +#define FIT_SKIP_AREA (1u << 3) +#define FIT_SKIP_BORDER (1u << 4) +#define FIT_REVERSED (1u << 5) +#define FIT_SORT_GLOBAL (1u << 6) +#define FIT_SKIP_TOOBIG (1u << 7) +#define FIT_SORTED (1u << 8) +#define FIT_SORT_SLM (1u << 9) +// Passed to the fit program as build options. +#define FIT_SLM_CAP 512 +#define FIT_BIG_CAP 32768 +// Scratch slices (and so workgroups) per fitSortBig launch. +#define FIT_BATCH 256 + typedef struct { uint16_t x, y; int16_t gx, gy; @@ -385,6 +702,10 @@ static cl_kernel oclKernelAddBlockOffsets; static cl_kernel oclKernelRadixHist; static cl_kernel oclKernelRadixScatter; static cl_kernel oclKernelGatherRecords; +static int oclFitReady = 0; +static cl_kernel oclKernelFitPrep; +static cl_kernel oclKernelFitSortSlm; +static cl_kernel oclKernelFitSortBig; typedef struct { int valid; @@ -412,6 +733,11 @@ typedef struct { cl_mem bufBlockSums; cl_mem bufPerm; cl_mem bufClusterDesc; + cl_mem bufSortKeys; + cl_mem bufFitMeta; + cl_mem bufSortScratch; + cl_mem bufSortList; + cl_mem bufDotTerms; } OclBufferCache; static OclBufferCache cache; @@ -477,6 +803,51 @@ static void profHost(const char *name, double startUs) fprintf(stderr, " host %-11s %8.1f us\n", name, hostNowUs() - startUs); } +// The fit kernels need fp64 (for fit_quad's double-evaluated center) and +// correctly-rounded fp32 division (for bit-identical slopes), so they live +// in their own program: a device without either degrades the fit path only, +// never the frontend. +static void oclInitFitProgram(cl_device_id device) +{ + cl_device_fp_config doubleConfig = 0; + clGetDeviceInfo(device, CL_DEVICE_DOUBLE_FP_CONFIG, sizeof(doubleConfig), &doubleConfig, NULL); + if (doubleConfig == 0) { + oclDebugLog("no fp64: fit kernels disabled"); + return; + } + cl_device_fp_config singleConfig = 0; + clGetDeviceInfo(device, CL_DEVICE_SINGLE_FP_CONFIG, sizeof(singleConfig), &singleConfig, NULL); + const int exactDivide = (singleConfig & CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT) != 0; + if (!exactDivide) + oclDebugLog("no correctly-rounded fp32 divide: slopes may differ in the last ulp"); + + char options[160]; + snprintf(options, sizeof(options), "%s -DFIT_SLM_CAP=%d -DFIT_BIG_CAP=%d", + exactDivide ? "-cl-fp32-correctly-rounded-divide-sqrt" : "", FIT_SLM_CAP, FIT_BIG_CAP); + + cl_int err = CL_SUCCESS; + const char *sources[5] = { sourceFitPrep, sourceFitSortHelpers, sourceFitPrep2, sourceFitSortSlm, sourceFitSort }; + cl_program program = clCreateProgramWithSource(oclContext, 5, sources, NULL, &err); + if (err != CL_SUCCESS) + return; + err = clBuildProgram(program, 1, &device, options, NULL, NULL); + if (err != CL_SUCCESS) { + char log[8192] = { 0 }; + clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, sizeof(log) - 1, log, NULL); + oclDebugLog(log); + clReleaseProgram(program); + return; + } + oclKernelFitPrep = clCreateKernel(program, "fitPrep", &err); + cl_int err2 = CL_SUCCESS; + oclKernelFitSortSlm = clCreateKernel(program, "fitSortSlm", &err2); + cl_int err3 = CL_SUCCESS; + oclKernelFitSortBig = clCreateKernel(program, "fitSortBig", &err3); + clReleaseProgram(program); + if (err == CL_SUCCESS && err2 == CL_SUCCESS && err3 == CL_SUCCESS) + oclFitReady = 1; +} + static void oclInit(void) { cl_platform_id platforms[8]; @@ -547,6 +918,7 @@ static void oclInit(void) return; oclReady = 1; + oclInitFitProgram(device); } static void releaseBuffer(cl_mem buffer) @@ -576,6 +948,11 @@ static void releaseCache(void) releaseBuffer(cache.bufBlockSums); releaseBuffer(cache.bufPerm); releaseBuffer(cache.bufClusterDesc); + releaseBuffer(cache.bufSortKeys); + releaseBuffer(cache.bufFitMeta); + releaseBuffer(cache.bufSortScratch); + releaseBuffer(cache.bufSortList); + releaseBuffer(cache.bufDotTerms); memset(&cache, 0, sizeof(cache)); } @@ -1247,6 +1624,424 @@ static void validateGather(zarray_t *clusters, uint32_t recordCount) (unsigned long long)mismatches, slot, recordCount); } +static int ensureFitBuffers(void) +{ + if (cache.bufSortKeys != NULL) + return 1; + int failed = 0; + cache.bufSortKeys = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_RECORD_CAPACITY * 8, NULL, &failed); + cache.bufFitMeta = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_MAX_CLUSTERS * 32, NULL, &failed); + cache.bufSortScratch = createOrFail(CL_MEM_READ_WRITE, (size_t)FIT_BATCH * FIT_BIG_CAP * 8, NULL, &failed); + cache.bufSortList = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_MAX_CLUSTERS * 4, NULL, &failed); + cache.bufDotTerms = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 4, NULL, &failed); + if (failed) { + releaseBuffer(cache.bufSortKeys); + releaseBuffer(cache.bufFitMeta); + releaseBuffer(cache.bufSortScratch); + releaseBuffer(cache.bufSortList); + releaseBuffer(cache.bufDotTerms); + cache.bufSortKeys = NULL; + cache.bufFitMeta = NULL; + cache.bufSortScratch = NULL; + cache.bufSortList = NULL; + cache.bufDotTerms = NULL; + return 0; + } + return 1; +} + +typedef struct { + cl_int minClusterPixels, perimCap, tagWidth, normalAllowed, reversedAllowed; +} FitParams; + +// Replicates fit_quads' per-call parameters (apriltag_quad_thresh.c) +// expression for expression, including the int-by-float decimate division. +static void computeFitParams(apriltag_detector_t *td, cl_int cw, cl_int ch, FitParams *params) +{ + int normalAllowed = 0, reversedAllowed = 0; + int minTagWidth = 1000000; + for (int i = 0; i < zarray_size(td->tag_families); i++) { + apriltag_family_t *family; + zarray_get(td->tag_families, i, &family); + if (family->width_at_border < minTagWidth) + minTagWidth = family->width_at_border; + normalAllowed |= !family->reversed_border; + reversedAllowed |= family->reversed_border; + } + if (td->quad_decimate > 1) + minTagWidth /= td->quad_decimate; + if (minTagWidth < 3) + minTagWidth = 3; + params->minClusterPixels = td->qtp.min_cluster_pixels; + params->perimCap = 2 * (2 * cw + 2 * ch); + params->tagWidth = minTagWidth; + params->normalAllowed = normalAllowed; + params->reversedAllowed = reversedAllowed; +} + +// P2 of the GPU fit_quads port: enqueues the per-cluster preparation +// (filter cascade, bbox, center, slopes, gradient dot) and the slope sort +// over the gathered cluster-contiguous records. SLM-sized clusters sort +// inside fitPrep; bigger ones get one fitSortBig launch each through the +// shared scratch buffer. Caller holds oclMutex. Returns 0 on failure. +static int fitPrepSort(apriltag_detector_t *td, zarray_t *clusters, cl_int cw, cl_int ch) +{ + const cl_uint clusterCount = (cl_uint)zarray_size(clusters); + if (oclFitReady == 0 || clusterCount == 0 || !ensureFitBuffers()) + return 0; + + FitParams params; + computeFitParams(td, cw, ch, ¶ms); + + // The sort runs over host-prefiltered id lists: SLM-sized candidates in + // sortList[0..slmCount), oversized ones after them. Only host-checkable + // size filters apply here; the sort kernels skip area- and + // border-rejected ids via the meta flags fitPrep writes. + cl_int err = CL_SUCCESS; + cl_uint slmCount = 0, bigCount = 0; + uint32_t *sortList = clEnqueueMapBuffer(oclQueue, cache.bufSortList, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, + (size_t)clusterCount * 4, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + return 0; + for (cl_uint c = 0; c < clusterCount; c++) { + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + const int clusterSize = cluster->size; + if (clusterSize >= params.minClusterPixels && clusterSize <= FIT_SLM_CAP) + sortList[slmCount++] = c; + } + for (cl_uint c = 0; c < clusterCount; c++) { + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + const int clusterSize = cluster->size; + if (clusterSize > FIT_SLM_CAP && clusterSize <= FIT_BIG_CAP && clusterSize <= params.perimCap) + sortList[slmCount + bigCount++] = c; + } + err = clEnqueueUnmapMemObject(oclQueue, cache.bufSortList, sortList, 0, NULL, NULL); + if (err != CL_SUCCESS) + return 0; + + err |= clSetKernelArg(oclKernelFitPrep, 0, sizeof(cl_mem), &cache.bufRecordsAlt); + err |= clSetKernelArg(oclKernelFitPrep, 1, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitPrep, 2, sizeof(cl_uint), &clusterCount); + err |= clSetKernelArg(oclKernelFitPrep, 3, sizeof(cl_int), ¶ms.minClusterPixels); + err |= clSetKernelArg(oclKernelFitPrep, 4, sizeof(cl_int), ¶ms.perimCap); + err |= clSetKernelArg(oclKernelFitPrep, 5, sizeof(cl_int), ¶ms.tagWidth); + err |= clSetKernelArg(oclKernelFitPrep, 6, sizeof(cl_int), ¶ms.normalAllowed); + err |= clSetKernelArg(oclKernelFitPrep, 7, sizeof(cl_int), ¶ms.reversedAllowed); + err |= clSetKernelArg(oclKernelFitPrep, 8, sizeof(cl_mem), &cache.bufSortKeys); + err |= clSetKernelArg(oclKernelFitPrep, 9, sizeof(cl_mem), &cache.bufFitMeta); + err |= clSetKernelArg(oclKernelFitPrep, 10, sizeof(cl_mem), &cache.bufDotTerms); + if (err != CL_SUCCESS) + return 0; + const size_t prepGlobal[1] = { (size_t)clusterCount * 256 }; + const size_t wgSize[1] = { 256 }; + err = clEnqueueNDRangeKernel(oclQueue, oclKernelFitPrep, 1, NULL, prepGlobal, wgSize, 0, NULL, profSlot("fitPrep")); + if (err != CL_SUCCESS) + return 0; + + if (slmCount > 0) { + err |= clSetKernelArg(oclKernelFitSortSlm, 0, sizeof(cl_mem), &cache.bufSortKeys); + err |= clSetKernelArg(oclKernelFitSortSlm, 1, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitSortSlm, 2, sizeof(cl_mem), &cache.bufFitMeta); + err |= clSetKernelArg(oclKernelFitSortSlm, 3, sizeof(cl_mem), &cache.bufSortList); + err |= clSetKernelArg(oclKernelFitSortSlm, 4, sizeof(cl_uint), &slmCount); + const size_t slmGlobal[1] = { (size_t)slmCount * 256 }; + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitSortSlm, 1, NULL, slmGlobal, wgSize, 0, NULL, profSlot("fitSortSlm")); + if (err != CL_SUCCESS) + return 0; + } + + err |= clSetKernelArg(oclKernelFitSortBig, 0, sizeof(cl_mem), &cache.bufSortKeys); + err |= clSetKernelArg(oclKernelFitSortBig, 1, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitSortBig, 2, sizeof(cl_mem), &cache.bufFitMeta); + err |= clSetKernelArg(oclKernelFitSortBig, 3, sizeof(cl_mem), &cache.bufSortList); + err |= clSetKernelArg(oclKernelFitSortBig, 6, sizeof(cl_mem), &cache.bufSortScratch); + for (cl_uint base = 0; base < bigCount && err == CL_SUCCESS; base += FIT_BATCH) { + const cl_uint batch = (bigCount - base < FIT_BATCH) ? bigCount - base : FIT_BATCH; + const cl_uint listBase = slmCount + base; + const size_t batchGlobal[1] = { (size_t)batch * 256 }; + err |= clSetKernelArg(oclKernelFitSortBig, 4, sizeof(cl_uint), &listBase); + err |= clSetKernelArg(oclKernelFitSortBig, 5, sizeof(cl_uint), &batch); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitSortBig, 1, NULL, batchGlobal, wgSize, 0, NULL, profSlot("fitSortBig")); + } + if (err != CL_SUCCESS) + return 0; + if (profEnabled) + clFinish(oclQueue); + return 1; +} + +static uint32_t hostOrderedFloatBits(float f) +{ + uint32_t u; + memcpy(&u, &f, sizeof(u)); + return (u & 0x80000000u) ? ~u : (u | 0x80000000u); +} + +// fit_quad's slope expression, operation for operation. +static float hostSlope(float fx, float fy, float cx, float cy) +{ + float dx = fx - cx; + float dy = fy - cy; + float quadrant = dy > 0 ? (dx > 0 ? 65536.0f : 131072.0f) : (dx > 0 ? 0.0f : -65536.0f); + if (dy < 0) { + dy = -dy; + dx = -dx; + } + if (dx < 0) { + float tmp = dx; + dx = dy; + dy = -tmp; + } + return quadrant + dy / dx; +} + +typedef struct { + float slope; + uint32_t idx; +} ValPt; + +// Verbatim replica of apriltag_quad_thresh.c's ptsort, including its +// tie-order behaviour (sorting networks below six elements, then a +// right-biased merge), carrying point indices through the sort. The GPU +// sort implements the same network, so its output must match exactly. +static void refPtsort(ValPt *pts, int sz) +{ +#define MAYBE_SWAP(arr, apos, bpos) \ + if (arr[apos].slope - arr[bpos].slope > 0) { \ + tmp = arr[apos]; arr[apos] = arr[bpos]; arr[bpos] = tmp; \ + }; + + if (sz <= 1) + return; + if (sz == 2) { + ValPt tmp; + MAYBE_SWAP(pts, 0, 1); + return; + } + if (sz == 3) { + ValPt tmp; + MAYBE_SWAP(pts, 0, 1); + MAYBE_SWAP(pts, 1, 2); + MAYBE_SWAP(pts, 0, 1); + return; + } + if (sz == 4) { + ValPt tmp; + MAYBE_SWAP(pts, 0, 1); + MAYBE_SWAP(pts, 2, 3); + MAYBE_SWAP(pts, 0, 2); + MAYBE_SWAP(pts, 1, 3); + MAYBE_SWAP(pts, 1, 2); + return; + } + if (sz == 5) { + ValPt tmp; + MAYBE_SWAP(pts, 0, 1); + MAYBE_SWAP(pts, 3, 4); + MAYBE_SWAP(pts, 1, 2); + MAYBE_SWAP(pts, 0, 1); + MAYBE_SWAP(pts, 0, 3); + MAYBE_SWAP(pts, 2, 4); + MAYBE_SWAP(pts, 1, 2); + MAYBE_SWAP(pts, 2, 3); + MAYBE_SWAP(pts, 1, 2); + return; + } +#undef MAYBE_SWAP + + ValPt stackBuffer[256]; + ValPt *tmp = (sz > 256) ? malloc(sizeof(ValPt) * (size_t)sz) : stackBuffer; + memcpy(tmp, pts, sizeof(ValPt) * (size_t)sz); + + int asz = sz / 2; + int bsz = sz - asz; + ValPt *as = &tmp[0]; + ValPt *bs = &tmp[asz]; + refPtsort(as, asz); + refPtsort(bs, bsz); + + int apos = 0, bpos = 0, outpos = 0; + while (apos < asz && bpos < bsz) { + if (as[apos].slope - bs[bpos].slope < 0) + pts[outpos++] = as[apos++]; + else + pts[outpos++] = bs[bpos++]; + } + if (apos < asz) + memcpy(&pts[outpos], &as[apos], (size_t)(asz - apos) * sizeof(ValPt)); + if (bpos < bsz) + memcpy(&pts[outpos], &bs[bpos], (size_t)(bsz - bpos) * sizeof(ValPt)); + if (sz > 256) + free(tmp); +} + +typedef struct { + int skipped, sortedSlm, sortedGlobal, tooBig, tieClusters; + long tiePoints; +} FitValidateStats; + +// Returns NULL when the cluster's GPU outputs replicate the CPU pre-sort +// semantics exactly, else a short description of the first mismatch. +static const char *checkFitCluster(const FitParams *params, const OclPt *pts, uint32_t n, + const uint32_t *m, const uint64_t *gpuKeys, FitValidateStats *stats) +{ + uint32_t expect = FIT_PROCESSED; + if ((int)n < params->minClusterPixels) + expect |= FIT_SKIP_MINPIX; + else if ((int)n > params->perimCap) + expect |= FIT_SKIP_PERIM; + if (expect != FIT_PROCESSED) { + stats->skipped++; + return (m[0] == expect && m[6] == n) ? NULL : "size filter flags"; + } + + uint16_t xmin = pts[0].x, xmax = pts[0].x, ymin = pts[0].y, ymax = pts[0].y; + for (uint32_t i = 1; i < n; i++) { + if (pts[i].x > xmax) xmax = pts[i].x; else if (pts[i].x < xmin) xmin = pts[i].x; + if (pts[i].y > ymax) ymax = pts[i].y; else if (pts[i].y < ymin) ymin = pts[i].y; + } + const uint32_t bboxA = (uint32_t)xmin | ((uint32_t)xmax << 16); + const uint32_t bboxB = (uint32_t)ymin | ((uint32_t)ymax << 16); + if ((xmax - xmin) * (ymax - ymin) < params->tagWidth) { + stats->skipped++; + if (m[0] != (expect | FIT_SKIP_AREA)) + return "area filter flags"; + return (m[4] == bboxA && m[5] == bboxB && m[6] == n) ? NULL : "bbox"; + } + + float cx = (xmin + xmax) * 0.5 + 0.05118; + float cy = (ymin + ymax) * 0.5 + -0.028581; + float dot = 0; + float *slopes = malloc(sizeof(float) * n); + if (slopes == NULL) + return "out of memory"; + for (uint32_t i = 0; i < n; i++) { + float dx = pts[i].x - cx; + float dy = pts[i].y - cy; + dot += dx * pts[i].gx + dy * pts[i].gy; + slopes[i] = hostSlope(pts[i].x, pts[i].y, cx, cy); + } + + const int rev = dot < 0; + uint32_t expectFlags = expect | (rev ? FIT_REVERSED : 0u); + if (rev ? !params->reversedAllowed : !params->normalAllowed) + expectFlags |= FIT_SKIP_BORDER; + else if (n <= FIT_BIG_CAP) + expectFlags |= FIT_SORTED; + else + expectFlags |= FIT_SKIP_TOOBIG; + + uint32_t cxBits, cyBits, dotBits; + memcpy(&cxBits, &cx, sizeof(cxBits)); + memcpy(&cyBits, &cy, sizeof(cyBits)); + memcpy(&dotBits, &dot, sizeof(dotBits)); + const char *fail = NULL; + if (m[0] != expectFlags) + fail = "flags"; + else if (m[1] != cxBits || m[2] != cyBits) + fail = "center bits"; + else if (m[3] != dotBits) + fail = "dot bits"; + else if (m[4] != bboxA || m[5] != bboxB || m[6] != n) + fail = "bbox"; + + if (fail == NULL && (expectFlags & FIT_SKIP_BORDER) != 0) + stats->skipped++; + if (fail == NULL && (expectFlags & FIT_SKIP_TOOBIG) != 0) + stats->tooBig++; + + if (fail == NULL && (expectFlags & FIT_SORTED) != 0) { + if (n <= FIT_SLM_CAP) + stats->sortedSlm++; + else + stats->sortedGlobal++; + ValPt *ref = malloc(sizeof(ValPt) * n); + if (ref == NULL) { + free(slopes); + return "out of memory"; + } + for (uint32_t i = 0; i < n; i++) { + ref[i].slope = slopes[i]; + ref[i].idx = i; + } + refPtsort(ref, (int)n); + long ties = 0; + for (uint32_t i = 0; i < n && fail == NULL; i++) { + const uint64_t expectKey = ((uint64_t)hostOrderedFloatBits(ref[i].slope) << 32) | ref[i].idx; + if (gpuKeys[i] != expectKey) + fail = "sorted order vs ptsort"; + if (i > 0 && ref[i].slope == ref[i - 1].slope) + ties++; + } + if (fail == NULL && ties > 0) { + stats->tieClusters++; + stats->tiePoints += ties; + } + free(ref); + } + free(slopes); + return fail; +} + +// Development gate for the fit preparation: reads the meta and sorted-key +// buffers back and checks every cluster against a host replication of the +// CPU's pre-sort semantics. Reports on stderr. +static void validateFitPrep(apriltag_detector_t *td, zarray_t *clusters, uint32_t recordCount, + cl_int cw, cl_int ch) +{ + FitParams params; + computeFitParams(td, cw, ch, ¶ms); + const uint32_t clusterCount = (uint32_t)zarray_size(clusters); + + cl_int err = CL_SUCCESS; + const uint64_t *keys = clEnqueueMapBuffer(oclQueue, cache.bufSortKeys, CL_TRUE, CL_MAP_READ, 0, + (size_t)recordCount * 8, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + fprintf(stderr, "apriltag opencl: fit validate: keys map failed\n"); + return; + } + const uint32_t *meta = clEnqueueMapBuffer(oclQueue, cache.bufFitMeta, CL_TRUE, CL_MAP_READ, 0, + (size_t)clusterCount * 32, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clEnqueueUnmapMemObject(oclQueue, cache.bufSortKeys, (void *)keys, 0, NULL, NULL); + fprintf(stderr, "apriltag opencl: fit validate: meta map failed\n"); + return; + } + + FitValidateStats stats = { 0, 0, 0, 0, 0, 0 }; + uint64_t failures = 0; + uint32_t off = 0; + for (uint32_t c = 0; c < clusterCount; c++) { + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + const uint32_t n = (uint32_t)cluster->size; + const char *fail = checkFitCluster(¶ms, (const OclPt *)cluster->data, n, + meta + 8u * c, keys + off, &stats); + off += n; + if (fail != NULL) { + if (failures < 5) + fprintf(stderr, "apriltag opencl: fit validate: cluster %u (n=%u): %s\n", c, n, fail); + failures++; + } + } + + clEnqueueUnmapMemObject(oclQueue, cache.bufSortKeys, (void *)keys, 0, NULL, NULL); + clEnqueueUnmapMemObject(oclQueue, cache.bufFitMeta, (void *)meta, 0, NULL, NULL); + + if (failures == 0) + fprintf(stderr, + "apriltag opencl: fit validate: PASS (%u clusters: %d skipped, %d slm-sorted, " + "%d global-sorted, %d too-big; ties %d clusters / %ld pts, order == ptsort)\n", + clusterCount, stats.skipped, stats.sortedSlm, stats.sortedGlobal, stats.tooBig, + stats.tieClusters, stats.tiePoints); + else + fprintf(stderr, "apriltag opencl: fit validate: FAIL (%llu clusters mismatched)\n", + (unsigned long long)failures); +} + typedef struct { const uint64_t *records; uint32_t recStart, recEnd; @@ -1434,11 +2229,14 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl // preserved by sort stability). Superseded by the gather path below; // kept as validation scaffolding. int useSorted = getenv("APRILTAG_OPENCL_SORTED") != NULL; + // P2: per-cluster fit preparation and slope sort over the gathered + // records (implies the gather path). + int useFit = !useSorted && getenv("APRILTAG_OPENCL_FIT") != NULL; // P1b: the hash build walk emits a permutation so one GPU gather // materializes cluster-contiguous records on-device for the fit stages. // Gated until the GPU fit lands; mutually exclusive with the sorted // path, whose cluster order differs from the walk's encounter order. - int useGather = !useSorted && getenv("APRILTAG_OPENCL_GATHER") != NULL; + int useGather = !useSorted && (useFit || getenv("APRILTAG_OPENCL_GATHER") != NULL); if (useSorted && !sortRecords(recordCount)) goto done; @@ -1464,6 +2262,16 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl if (gatherClusterRecords(td, clusters, &plan, recordCount)) { if (getenv("APRILTAG_OPENCL_GATHER_VALIDATE") != NULL) validateGather(clusters, recordCount); + if (useFit) { + t = hostNowUs(); + if (fitPrepSort(td, clusters, cw, ch)) { + profHost("fitEnqueue", t); + if (getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) + validateFitPrep(td, clusters, recordCount, cw, ch); + } else { + oclDebugLog("fit prep failed"); + } + } } else { oclDebugLog("gather failed"); } From c63c5159ac6d0e304a9d913705c3b1808e3bb997 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 11:23:29 +0200 Subject: [PATCH 08/18] Note verified P3 facts: maxima qsort is order-irrelevant quad_segment_maxima only reads the sorted copy at index max_nmaxima as a threshold value and filters maxima in original order with a strict comparison, so the GPU port needs top-K-by-value selection rather than a qsort replica. Also record where lfps weights come from (the original decimated grayscale, resident in bufIm on the frontend path) and the serial-scan shape for its double accumulation. Co-Authored-By: Claude Fable 5 --- ocl_harness/FIT_QUADS_PLAN.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index 015514c3..a8eec088 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -118,6 +118,14 @@ the cluster build walk disappears entirely. (point index in the low half indexes bufRecordsAlt); quads-only readback; corpus detection equivalence + timing. With ptsort order replicated, the target is bit-identical corners everywhere. + Verified ahead of time: quad_segment_maxima's qsort of maxima errors + is order-irrelevant — it only reads the value at index max_nmaxima as + a threshold, then filters maxima in original order with a strict + comparison, so ties at the boundary drop uniformly. The GPU needs a + top-K-by-value selection, not a qsort replica. Note lfps weights + sample the original (decimated) grayscale image — bufIm holds it on + the oclFrontend path. compute_lfps accumulates doubles sequentially + per cluster: one-lane-per-cluster serial scan, like the dot. ## Also still open (smaller) From 2187bcd7e14150fb80011535d260814a311fd2e5 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 12:17:48 +0200 Subject: [PATCH 09/18] Add GPU fit_quads tail: lfps, maxima, combos, corners on-device (P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three kernels after the P2 slope sort complete the fit on the GPU with quads-only readback: fitLfpsPrep/fitLfpsScan (compute_lfps over six moment planes — parallel exact terms, then per-(cluster,field) serial add chains preserving CPU summation order), fitErrs (windowed errors, host-libm filter constants baked in as exact hex floats, order- preserving maxima compaction, top-K multiset threshold replicating the order-irrelevant qsort), and fitCombos (C(m,2) forward + wraparound pair fits cached in SLM, lex-rank combo scan with first-wins tie argmin, final line fits, intersections, float-narrowed corners, area and angle checks). fit_quads() consumes the chain through oclFitQuads(): fitPrep flags decide pre-fit rejections, GPU verdicts decide sorted clusters, and a handled[] mask leaves too-big/over-cap/failed clusters to the CPU tasks. lfps weights sample the device-resident grayscale, so the handoff (pendingFit) only arms on the oclFrontend path. Gates on w3cj: detect 32/32 at 0.000000 px, deterministic run-to-run; corpus 120/120 at 0.0000 px; FIT_VALIDATE re-fits every GPU cluster with the CPU fit_quad — verdicts and corner bits match on all 2510 vide fits. Detect: ~63-71 ms wall / ~82-88 core-ms vs CPU 58-60 / 275-290 (-70% CPU; ~+12 ms wall vs gather-only — the exactness contract pins the chain to fp64, where the iGPU trails the 8-thread CPU, so APRILTAG_OPENCL_FIT=1 is the max-CPU-offload mode). --- apriltag_quad_thresh.c | 17 + ocl_harness/FIT_QUADS_PLAN.md | 75 ++- ocl_threshold.c | 841 +++++++++++++++++++++++++++++++++- ocl_threshold.h | 10 + 4 files changed, 917 insertions(+), 26 deletions(-) diff --git a/apriltag_quad_thresh.c b/apriltag_quad_thresh.c index afd77428..71ed8014 100644 --- a/apriltag_quad_thresh.c +++ b/apriltag_quad_thresh.c @@ -96,6 +96,11 @@ struct quad_task int tag_width; bool normal_border; bool reversed_border; + + // Per-cluster flags from the GPU fit: non-zero marks clusters already + // decided (quad emitted or rejected), which this task must skip. NULL + // when every cluster takes the CPU path. + const uint8_t *gpu_handled; }; @@ -1077,6 +1082,9 @@ static void do_quad_task(void *p) for (int cidx = task->cidx0; cidx < task->cidx1; cidx++) { + if (task->gpu_handled != NULL && task->gpu_handled[cidx]) + continue; + zarray_t **cluster; zarray_get_volatile(clusters, cidx, &cluster); @@ -1855,6 +1863,13 @@ zarray_t* fit_quads(apriltag_detector_t *td, int w, int h, zarray_t* clusters, i min_tag_width = 3; } + // The GPU fit decides most clusters outright (appending their quads + // here); the tasks below only fit the clusters it left over. + uint8_t *gpu_handled = NULL; +#ifdef APRILTAG_HAVE_OPENCL + gpu_handled = oclFitQuads(td, clusters, im, quads); +#endif + int sz = zarray_size(clusters); int chunksize = 1 + sz / (APRILTAG_TASKS_PER_THREAD_TARGET * td->nthreads); struct quad_task *tasks = malloc(sizeof(struct quad_task)*(sz / chunksize + 1)); @@ -1872,6 +1887,7 @@ zarray_t* fit_quads(apriltag_detector_t *td, int w, int h, zarray_t* clusters, i tasks[ntasks].tag_width = min_tag_width; tasks[ntasks].normal_border = normal_border; tasks[ntasks].reversed_border = reversed_border; + tasks[ntasks].gpu_handled = gpu_handled; workerpool_add_task(td->wp, do_quad_task, &tasks[ntasks]); ntasks++; @@ -1880,6 +1896,7 @@ zarray_t* fit_quads(apriltag_detector_t *td, int w, int h, zarray_t* clusters, i workerpool_run(td->wp); free(tasks); + free(gpu_handled); return quads; } diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index a8eec088..80c7f4c0 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -114,18 +114,69 @@ the cluster build walk disappears entirely. sorted sequence including every tie cluster match ptsort exactly. Still open (P3 polish): merge-path for the SLM sort's top levels; trim fitSortBig's list (host lists border-undecided candidates). -- P3: lfps + maxima + combos + line fits consuming the sorted keys - (point index in the low half indexes bufRecordsAlt); quads-only - readback; corpus detection equivalence + timing. With ptsort order - replicated, the target is bit-identical corners everywhere. - Verified ahead of time: quad_segment_maxima's qsort of maxima errors - is order-irrelevant — it only reads the value at index max_nmaxima as - a threshold, then filters maxima in original order with a strict - comparison, so ties at the boundary drop uniformly. The GPU needs a - top-K-by-value selection, not a qsort replica. Note lfps weights - sample the original (decimated) grayscale image — bufIm holds it on - the oclFrontend path. compute_lfps accumulates doubles sequentially - per cluster: one-lane-per-cluster serial scan, like the dot. +- P3 (DONE, measured on w3cj 2026-06-11): lfps + maxima + combos + line + fits + corner checks on the GPU, quads-only readback. Three kernels + after the P2 sort: + - fitLfpsPrep/fitLfpsScan: compute_lfps over a six-PLANE layout + (stride = total fit points). Prep (one WG per cluster) resolves the + sorted-key indirection, samples the grayscale weight, and writes + each point's six moment TERMS — the CPU's exact per-statement + products — coalesced into the planes; scan runs one lane per + (cluster, field) doing the in-place sequential add chain (the + minimal serial work the summation-order contract allows). The + first cut (serial lane inside the 256-wide WG, AoS rows) cost + 12-13 ms; the plane split runs prep 1.4 + scan 7.0 ms, now + bandwidth-bound (~100 MB of plane traffic on shared DDR). + - fitErrs: windowed fit_line errors per point (sqrtf-on-double + narrowing kept as (double)sqrt((float)x)), the fixed 7-tap low-pass + with filter constants computed by the host's libm at init and baked + in as exact hex float build defines, order-preserving maxima + compaction (per-256-chunk local scan), then the max_nmaxima cut as + a lane-0 top-(K+1) multiset selection + strict-threshold filter + (replicates the CPU's qsort-threshold exactly). ~4.4 ms. + - fitCombos: all C(m,2) forward pair fits + C(m,2) wraparound closers + computed once into SLM (the same fit_line values the CPU recomputes + in its loop nest), combo scan in CPU lex-rank order as pure table + lookups, (err, rank) argmin reduce (exact-tie -> lower rank = CPU + first-wins), then lane 0 re-fits the winning four lines with params, + intersections, float corner narrowing exactly where the CPU assigns + quad->p, and the area/angle rejections over those float corners + (which subtract in FLOAT before promoting). Per-combo fitLineC + version cost 5.7-7.2 ms; the pair-table version runs ~2.2 ms. + Host: runClusterChain leaves a pendingFit tag (oclFrontend path only — + lfps needs the grayscale resident in bufIm); fit_quads() calls + oclFitQuads(), which blocks on the P2 meta, splits clusters into + fitPrep-flag rejections / GPU fit slots / CPU fallback (too big, over + the FIT_POINT_CAP scratch cap, or chain failure -> fitOut status 0), + uploads the (clusterIdx, lfpsOffset) list, enqueues the chain, maps + the quads-only fitOut buffer (status + 8 corner floats per cluster), + and returns a handled[] mask so do_quad_task skips decided clusters. + Gates: detect 32/32 at 0.000000 px and deterministic run-to-run; + corpus 120/120 at 0.0000 px; APRILTAG_OPENCL_FIT_VALIDATE=1 re-fits + every GPU cluster with the production CPU fit_quad — verdicts and + corner BITS match on all 2510 vide fits (443 quads, 2067 rejects) + and across the corpus. + Measured (vide 3088x2064, interleaved same-session): detect with + APRILTAG_OPENCL_FIT=1 runs ~63-71 ms wall / ~82-88 core-ms vs + gather-only ~47-54 / ~174-179 vs CPU ~58-60 / ~275-290. The fit mode + trades ~+12 ms wall for another ~90 core-ms of CPU freed (-70% vs + pure CPU overall): the GPU tail (fitPrep 3.1 + sorts ~6 + chain ~15) + exceeds the ~13 ms CPU fit it replaces because the exactness contract + pins the heavy stages to fp64, where the Arc 140T is weaker than the + 8-thread CPU. Both modes stay env-selectable: APRILTAG_OPENCL=1 alone + for the fastest wall, +APRILTAG_OPENCL_FIT=1 for max CPU offload. + +## P3 follow-ups (next session) + +- The slim walk: with the fit on-GPU, the build walk's appendPt/merge + copying (most of its 16-29 ms) only feeds desc sizes, the permutation + passes, and CPU-fallback clusters. Track counts instead of + materializing zarrays; reconstruct the rare fallback cluster from a + bufRecordsAlt range readback (payloads are (x, y, gx, gy) in CPU + point order — exactly struct pt). Biggest remaining CPU+wall lever. +- fitErrs/fitLfpsScan sit at the plane-traffic bandwidth floor; further + wall wins come from the P2 sort polish (merge-path for the SLM sort's + top levels, fitPrep dot throughput) or overlap across frames. ## Also still open (smaller) diff --git a/ocl_threshold.c b/ocl_threshold.c index c434d93e..f7d64cf3 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -2,7 +2,9 @@ #define CL_TARGET_OPENCL_VERSION 300 #include +#include #include +#include #include #include #include @@ -624,6 +626,418 @@ static const char *sourceFitSort = " if (lid == 0) meta[8u * c] = (flags & ~SORT_GLOBAL) | SORTED;\n" "}\n"; +static const char *sourceFitLfps = + // compute_lfps replica over the sorted point order (P3), in two kernels + // over a PLANE layout (six per-field planes of lfStride doubles each). + // fitLfpsPrep (one WG per cluster) resolves the sorted indirection, + // samples the grayscale weight, and writes each point's six moment + // TERMS — the exact per-statement products the CPU forms (W*fx, + // (W*fx)*fx, ...) — straight into the planes, fully parallel and + // coalesced within each plane. fitLfpsScan then turns each plane + // segment into the cumulative sums in place: one lane per (cluster, + // field), a pure sequential add chain in CPU accumulation order — the + // minimal serial work the exactness contract allows. + "__kernel void fitLfpsPrep(__global const ulong2 *records, __global const ulong *keys,\n" + " __global const uint2 *desc, __global const uint2 *fitList, uint count,\n" + " __global const uchar *im, int imW, int imH, int imS,\n" + " uint lfStride, __global double *lfps) {\n" + " uint g = get_group_id(0);\n" + " if (g >= count) return;\n" + " uint c = fitList[g].x;\n" + " uint lo = fitList[g].y;\n" + " uint off = desc[c].x;\n" + " uint n = desc[c].y;\n" + " int lid = get_local_id(0);\n" + " for (uint i = (uint)lid; i < n; i += 256u) {\n" + " ulong payload = records[off + (uint)(keys[off + i] & 0xFFFFFFFFul)].y;\n" + " int px = (int)((payload >> 48) & 0xFFFFul);\n" + " int py = (int)((payload >> 32) & 0xFFFFul);\n" + " double x = px * 0.5 + 0.5;\n" + " double y = py * 0.5 + 0.5;\n" + " int ix = (int)x, iy = (int)y;\n" + " double W = 1.0;\n" + " if (ix > 0 && ix + 1 < imW && iy > 0 && iy + 1 < imH) {\n" + " int gradX = (int)im[iy * imS + ix + 1] - (int)im[iy * imS + ix - 1];\n" + " int gradY = (int)im[(iy + 1) * imS + ix] - (int)im[(iy - 1) * imS + ix];\n" + " W = sqrt((double)(gradX * gradX + gradY * gradY)) + 1.0;\n" + " }\n" + " double fx = x, fy = y;\n" + " __global double *p = lfps + lo + i;\n" + " p[0] = W * fx;\n" + " p[lfStride] = W * fy;\n" + " p[2u * lfStride] = W * fx * fx;\n" + " p[3u * lfStride] = W * fx * fy;\n" + " p[4u * lfStride] = W * fy * fy;\n" + " p[5u * lfStride] = W;\n" + " }\n" + "}\n" + "__kernel void fitLfpsScan(__global const uint2 *desc, __global const uint2 *fitList, uint count,\n" + " uint lfStride, __global double *lfps) {\n" + " uint t = get_global_id(0);\n" + " uint slot = t / 6u;\n" + " uint field = t % 6u;\n" + " if (slot >= count) return;\n" + " uint c = fitList[slot].x;\n" + " uint lo = fitList[slot].y;\n" + " uint n = desc[c].y;\n" + " __global double *plane = lfps + field * lfStride + lo;\n" + " double acc = 0;\n" + " for (uint i = 0; i < n; i++) {\n" + " acc += plane[i];\n" + " plane[i] = acc;\n" + " }\n" + "}\n"; + +static const char *sourceFitLine = + // fit_line replicas over the cumulative moments, laid out as 6 doubles + // per point [Mx My Mxx Mxy Myy W]. The CPU branches and operation order + // are kept exactly: one subtraction when i0 > 0, last-minus-prev plus i1 + // on wraparound, divides in double, and sqrtf-on-double narrowing (the + // CPU calls sqrtf on double expressions) as (double)sqrt((float)x). + "#define LFP(p, i) base[(p) * lfStride + (uint)(i)]\n" + "inline double fitErrAt(__global const double *base, uint lfStride, int sz, int i0, int i1) {\n" + " double mx, my, mxx, mxy, myy, mw;\n" + " int N;\n" + " if (i0 < i1) {\n" + " mx = LFP(0u, i1); my = LFP(1u, i1); mxx = LFP(2u, i1);\n" + " mxy = LFP(3u, i1); myy = LFP(4u, i1); mw = LFP(5u, i1);\n" + " if (i0 > 0) {\n" + " mx -= LFP(0u, i0 - 1); my -= LFP(1u, i0 - 1); mxx -= LFP(2u, i0 - 1);\n" + " mxy -= LFP(3u, i0 - 1); myy -= LFP(4u, i0 - 1); mw -= LFP(5u, i0 - 1);\n" + " }\n" + " N = i1 - i0 + 1;\n" + " } else {\n" + " mx = LFP(0u, sz - 1) - LFP(0u, i0 - 1); my = LFP(1u, sz - 1) - LFP(1u, i0 - 1);\n" + " mxx = LFP(2u, sz - 1) - LFP(2u, i0 - 1); mxy = LFP(3u, sz - 1) - LFP(3u, i0 - 1);\n" + " myy = LFP(4u, sz - 1) - LFP(4u, i0 - 1); mw = LFP(5u, sz - 1) - LFP(5u, i0 - 1);\n" + " mx += LFP(0u, i1); my += LFP(1u, i1); mxx += LFP(2u, i1);\n" + " mxy += LFP(3u, i1); myy += LFP(4u, i1); mw += LFP(5u, i1);\n" + " N = sz - i0 + i1 + 1;\n" + " }\n" + " double ex = mx / mw;\n" + " double ey = my / mw;\n" + " double cxx = mxx / mw - ex * ex;\n" + " double cxy = mxy / mw - ex * ey;\n" + " double cyy = myy / mw - ey * ey;\n" + " double eigSmall = 0.5 * (cxx + cyy - (double)sqrt((float)((cxx - cyy) * (cxx - cyy) + 4.0 * cxy * cxy)));\n" + " return N * eigSmall;\n" + "}\n" + // Cached variant for the combo search: the at/prev moment rows of the + // (at most FIT_MAX_K) maxima plus the last row live in local memory, + // indexed by maxima slot. Arithmetic identical to fitErrAt. + "inline void fitLineC(__local const double *lfA, __local const double *lfP, __local const double *lfL,\n" + " int sz, int i0, int i1, int k0, int k1,\n" + " double *lineparm, double *err, double *mse) {\n" + " double mx, my, mxx, mxy, myy, mw;\n" + " int N;\n" + " __local const double *b = lfA + 6 * k1;\n" + " __local const double *a = lfP + 6 * k0;\n" + " if (i0 < i1) {\n" + " mx = b[0]; my = b[1]; mxx = b[2]; mxy = b[3]; myy = b[4]; mw = b[5];\n" + " if (i0 > 0) {\n" + " mx -= a[0]; my -= a[1]; mxx -= a[2]; mxy -= a[3]; myy -= a[4]; mw -= a[5];\n" + " }\n" + " N = i1 - i0 + 1;\n" + " } else {\n" + " mx = lfL[0] - a[0]; my = lfL[1] - a[1]; mxx = lfL[2] - a[2];\n" + " mxy = lfL[3] - a[3]; myy = lfL[4] - a[4]; mw = lfL[5] - a[5];\n" + " mx += b[0]; my += b[1]; mxx += b[2]; mxy += b[3]; myy += b[4]; mw += b[5];\n" + " N = sz - i0 + i1 + 1;\n" + " }\n" + " double ex = mx / mw;\n" + " double ey = my / mw;\n" + " double cxx = mxx / mw - ex * ex;\n" + " double cxy = mxy / mw - ex * ey;\n" + " double cyy = myy / mw - ey * ey;\n" + " double sq = (double)sqrt((float)((cxx - cyy) * (cxx - cyy) + 4.0 * cxy * cxy));\n" + " double eigSmall = 0.5 * (cxx + cyy - sq);\n" + " if (lineparm) {\n" + " lineparm[0] = ex;\n" + " lineparm[1] = ey;\n" + " double eig = 0.5 * (cxx + cyy + sq);\n" + " double nx1 = cxx - eig, ny1 = cxy;\n" + " double m1 = nx1 * nx1 + ny1 * ny1;\n" + " double nx2 = cxy, ny2 = cyy - eig;\n" + " double m2 = nx2 * nx2 + ny2 * ny2;\n" + " double nx, ny, mm;\n" + " if (m1 > m2) { nx = nx1; ny = ny1; mm = m1; } else { nx = nx2; ny = ny2; mm = m2; }\n" + " double length = (double)sqrt((float)mm);\n" + " if (fabs(length) < 1e-12) { lineparm[2] = 0; lineparm[3] = 0; }\n" + " else { lineparm[2] = nx / length; lineparm[3] = ny / length; }\n" + " }\n" + " *err = N * eigSmall;\n" + " *mse = eigSmall;\n" + "}\n"; + +static const char *sourceFitErrs = + // quad_segment_maxima's front half (P3): windowed errors per point, the + // fixed 7-tap low-pass (FILT* baked in from the host's libm exp, the + // values the CPU computes at runtime), maxima detection, then the + // max_nmaxima cut. The cut's qsort is order-irrelevant on the CPU — only + // the value at index max_nmaxima is read as a strict threshold — so a + // top-K multiset selection reproduces it exactly. + "__kernel void fitErrs(__global const double *lfps, uint lfStride, __global const uint2 *desc,\n" + " __global const uint2 *fitList, uint count, int maxNmaxima,\n" + " __global double *errsRaw, __global double *errsSmooth,\n" + " __global uint *maximaScratch, __global uint *maximaOut,\n" + " __global uint *fitOut) {\n" + " uint g = get_group_id(0);\n" + " if (g >= count) return;\n" + " uint c = fitList[g].x;\n" + " uint lo = fitList[g].y;\n" + " uint n = desc[c].y;\n" + " int lid = get_local_id(0);\n" + " __local uint scanBuf[256];\n" + " int ksz = min(20, (int)n / 12);\n" + " if (ksz < 2) {\n" + " if (lid == 0) fitOut[g * FIT_OUT_STRIDE] = 2u;\n" + " return;\n" + " }\n" + " __global const double *lf = lfps + lo;\n" + " for (uint i = (uint)lid; i < n; i += 256u)\n" + " errsRaw[lo + i] = fitErrAt(lf, lfStride, (int)n, (int)((i + n - (uint)ksz) % n), (int)((i + (uint)ksz) % n));\n" + " barrier(CLK_GLOBAL_MEM_FENCE);\n" + " for (uint i = (uint)lid; i < n; i += 256u) {\n" + " double acc = 0;\n" + " acc += errsRaw[lo + (i + n - 3u) % n] * FILT0;\n" + " acc += errsRaw[lo + (i + n - 2u) % n] * FILT1;\n" + " acc += errsRaw[lo + (i + n - 1u) % n] * FILT2;\n" + " acc += errsRaw[lo + i] * FILT3;\n" + " acc += errsRaw[lo + (i + 1u) % n] * FILT4;\n" + " acc += errsRaw[lo + (i + 2u) % n] * FILT5;\n" + " acc += errsRaw[lo + (i + 3u) % n] * FILT6;\n" + " errsSmooth[lo + i] = acc;\n" + " }\n" + " barrier(CLK_GLOBAL_MEM_FENCE);\n" + // Order-preserving compaction of maxima indices: per-256 chunk local + // scan with a running base, so maximaScratch holds ascending indices. + " uint base = 0;\n" + " for (uint chunk = 0; chunk < n; chunk += 256u) {\n" + " uint i = chunk + (uint)lid;\n" + " int isMax = 0;\n" + " if (i < n) {\n" + " double v = errsSmooth[lo + i];\n" + " isMax = (v > errsSmooth[lo + (i + 1u) % n]) && (v > errsSmooth[lo + (i + n - 1u) % n]);\n" + " }\n" + " scanBuf[lid] = (uint)isMax;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int s = 1; s < 256; s <<= 1) {\n" + " uint v2 = (lid >= s) ? scanBuf[lid - s] : 0u;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " scanBuf[lid] += v2;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " if (isMax) maximaScratch[lo + base + scanBuf[lid] - 1u] = i;\n" + " base += scanBuf[255];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " uint nmax = base;\n" + " if (nmax < 4u) {\n" + " if (lid == 0) fitOut[g * FIT_OUT_STRIDE] = 2u;\n" + " return;\n" + " }\n" + " if (lid != 0) return;\n" + " uint m = 0;\n" + " __global uint *outIdx = maximaOut + g * FIT_MAXIMA_STRIDE + 1u;\n" + " if (nmax > (uint)maxNmaxima) {\n" + " double top[FIT_MAX_K + 1];\n" + " int kOne = maxNmaxima + 1;\n" + " int filled = 0;\n" + " for (uint j = 0; j < nmax; j++) {\n" + " double v = errsSmooth[lo + maximaScratch[lo + j]];\n" + " if (filled == kOne && !(v > top[kOne - 1])) continue;\n" + " int pos = (filled < kOne) ? filled : (kOne - 1);\n" + " while (pos > 0 && top[pos - 1] < v) { top[pos] = top[pos - 1]; pos--; }\n" + " top[pos] = v;\n" + " if (filled < kOne) filled++;\n" + " }\n" + " double thr = top[maxNmaxima];\n" + " for (uint j = 0; j < nmax; j++) {\n" + " uint idx = maximaScratch[lo + j];\n" + " if (errsSmooth[lo + idx] <= thr) continue;\n" + " outIdx[m] = idx; m++;\n" + " }\n" + " } else {\n" + " for (uint j = 0; j < nmax; j++) { outIdx[m] = maximaScratch[lo + j]; m++; }\n" + " }\n" + " maximaOut[g * FIT_MAXIMA_STRIDE] = m;\n" + "}\n"; + +static const char *sourceFitCombosA = + // Combo search + final quad (P3 tail). Combos are evaluated in the + // CPU's lexicographic (m0,m1,m2,m3) order via rank decoding; the argmin + // reduce breaks exact err ties by lower rank, replicating the CPU's + // strict first-wins scan. + "inline uint chooseN(uint n, uint k) {\n" + " if (n < k) return 0u;\n" + " if (k == 1u) return n;\n" + " if (k == 2u) return n * (n - 1u) / 2u;\n" + " return n * (n - 1u) * (n - 2u) / 6u;\n" + "}\n" + "inline void decodeCombo(uint rank, uint m, uint *a, uint *b, uint *c, uint *d) {\n" + " uint r = rank;\n" + " uint i = 0;\n" + " while (chooseN(m - 1u - i, 3u) <= r) { r -= chooseN(m - 1u - i, 3u); i++; }\n" + " *a = i; i++;\n" + " while (chooseN(m - 1u - i, 2u) <= r) { r -= chooseN(m - 1u - i, 2u); i++; }\n" + " *b = i; i++;\n" + " while (chooseN(m - 1u - i, 1u) <= r) { r -= chooseN(m - 1u - i, 1u); i++; }\n" + " *c = i; i++;\n" + " *d = i + r;\n" + "}\n" + // The post-corner checks subtract corners in FLOAT (the CPU reads float + // quad->p fields) before promoting to double — sqf keeps that order. + "inline double sqf(float d) { double v = (double)d; return v * v; }\n" + "inline uint pairIdx(uint k0, uint k1, uint m) { return k0 * (2u * m - k0 - 1u) / 2u + (k1 - k0 - 1u); }\n" + "#define FIT_PAIRS (FIT_MAX_K * (FIT_MAX_K - 1) / 2)\n" + "__kernel void fitCombos(__global const double *lfps, uint lfStride, __global const uint2 *desc,\n" + " __global const uint2 *fitList, uint count,\n" + " __global const uint *maximaOut, double maxDot, double maxMse,\n" + " int tagWidth, __global uint *fitOut) {\n" + " uint g = get_group_id(0);\n" + " if (g >= count) return;\n" + " int lid = get_local_id(0);\n" + " __local uint midx[FIT_MAX_K];\n" + " __local double lfA[FIT_MAX_K * 6];\n" + " __local double lfP[FIT_MAX_K * 6];\n" + " __local double lfL[6];\n" + " __local double pairErrF[FIT_PAIRS];\n" + " __local double pairMseF[FIT_PAIRS];\n" + " __local double pairNx[FIT_PAIRS];\n" + " __local double pairNy[FIT_PAIRS];\n" + " __local double pairErrW[FIT_PAIRS];\n" + " __local double pairMseW[FIT_PAIRS];\n" + " __local double redErr[256];\n" + " __local uint redRank[256];\n" + " __global uint *out = fitOut + g * FIT_OUT_STRIDE;\n" + " if (out[0] != 0u) return;\n" + " uint c = fitList[g].x;\n" + " uint lo = fitList[g].y;\n" + " int sz = (int)desc[c].y;\n" + " uint m = maximaOut[g * FIT_MAXIMA_STRIDE];\n" + " __global const double *lf = lfps + lo;\n" + " if (lid < (int)m) {\n" + " uint ii = maximaOut[g * FIT_MAXIMA_STRIDE + 1u + (uint)lid];\n" + " midx[lid] = ii;\n" + " for (uint q = 0; q < 6u; q++) lfA[lid * 6 + (int)q] = lf[q * lfStride + ii];\n" + " for (uint q = 0; q < 6u; q++) lfP[lid * 6 + (int)q] = (ii > 0u) ? lf[q * lfStride + (ii - 1u)] : 0.0;\n" + " }\n" + " if (lid == 0) for (uint q = 0; q < 6u; q++) lfL[q] = lf[q * lfStride + (uint)(sz - 1)];\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n"; + +static const char *sourceFitCombosA2 = + // Every line fit the combo scan can request is one of the C(m,2) + // forward segments or the C(m,2) wraparound closers (i_high -> i_low); + // fit them all once — the same fit_line(i0, i1) values the CPU + // recomputes inside its loop nest — and turn the combo scan into pure + // table lookups. + " uint nPairs = m * (m - 1u) / 2u;\n" + " int isFwd = lid < (int)nPairs;\n" + " int isWrap = lid >= 128 && lid < 128 + (int)nPairs;\n" + " if (isFwd || isWrap) {\n" + " uint p = isFwd ? (uint)lid : (uint)(lid - 128);\n" + " uint k0 = 0, rem = p;\n" + " while (rem >= m - 1u - k0) { rem -= m - 1u - k0; k0++; }\n" + " uint k1 = k0 + 1u + rem;\n" + " double prm[4], e, ms;\n" + " if (isFwd) {\n" + " fitLineC(lfA, lfP, lfL, sz, (int)midx[k0], (int)midx[k1], (int)k0, (int)k1, prm, &e, &ms);\n" + " pairErrF[p] = e; pairMseF[p] = ms; pairNx[p] = prm[2]; pairNy[p] = prm[3];\n" + " } else {\n" + " fitLineC(lfA, lfP, lfL, sz, (int)midx[k1], (int)midx[k0], (int)k1, (int)k0, 0, &e, &ms);\n" + " pairErrW[p] = e; pairMseW[p] = ms;\n" + " }\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " uint nc = (m >= 4u) ? (m * (m - 1u) * (m - 2u) * (m - 3u) / 24u) : 0u;\n" + " double bestErr = INFINITY;\n" + " uint bestRank = 0xFFFFFFFFu;\n" + " for (uint rank = (uint)lid; rank < nc; rank += 256u) {\n" + " uint a, b, c2, d;\n" + " decodeCombo(rank, m, &a, &b, &c2, &d);\n" + " uint p01 = pairIdx(a, b, m), p12 = pairIdx(b, c2, m);\n" + " uint p23 = pairIdx(c2, d, m), p30 = pairIdx(a, d, m);\n" + " if (pairMseF[p01] > maxMse) continue;\n" + " if (pairMseF[p12] > maxMse) continue;\n" + " double dotv = pairNx[p01] * pairNx[p12] + pairNy[p01] * pairNy[p12];\n" + " if (fabs(dotv) > maxDot) continue;\n" + " if (pairMseF[p23] > maxMse) continue;\n" + " if (pairMseW[p30] > maxMse) continue;\n" + " double e = pairErrF[p01] + pairErrF[p12] + pairErrF[p23] + pairErrW[p30];\n" + " if (e < bestErr) { bestErr = e; bestRank = rank; }\n" + " }\n" + " redErr[lid] = bestErr;\n" + " redRank[lid] = bestRank;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " for (int s = 128; s > 0; s >>= 1) {\n" + " if (lid < s) {\n" + " int take = (redErr[lid + s] < redErr[lid]) ||\n" + " (redErr[lid + s] == redErr[lid] && redRank[lid + s] < redRank[lid]);\n" + " if (take) { redErr[lid] = redErr[lid + s]; redRank[lid] = redRank[lid + s]; }\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " }\n" + " if (lid != 0) return;\n" + " if (!(redErr[0] / (double)sz < maxMse)) { out[0] = 2u; return; }\n"; + +static const char *sourceFitCombosB = + // Lane 0 continues: the winning combo's four line fits, intersections, + // float corner narrowing exactly where the CPU assigns quad->p, then the + // area and angle rejections over those float corners. + " uint wa, wb, wc, wd;\n" + " decodeCombo(redRank[0], m, &wa, &wb, &wc, &wd);\n" + " int idx4[4]; int slot4[4];\n" + " idx4[0] = (int)midx[wa]; slot4[0] = (int)wa;\n" + " idx4[1] = (int)midx[wb]; slot4[1] = (int)wb;\n" + " idx4[2] = (int)midx[wc]; slot4[2] = (int)wc;\n" + " idx4[3] = (int)midx[wd]; slot4[3] = (int)wd;\n" + " double lines[4][4];\n" + " for (int i = 0; i < 4; i++) {\n" + " double e, ms;\n" + " fitLineC(lfA, lfP, lfL, sz, idx4[i], idx4[(i + 1) & 3], slot4[i], slot4[(i + 1) & 3],\n" + " lines[i], &e, &ms);\n" + " if (ms > maxMse) { out[0] = 2u; return; }\n" + " }\n" + " float pf[8];\n" + " for (int i = 0; i < 4; i++) {\n" + " double A00 = lines[i][3], A01 = -lines[(i + 1) & 3][3];\n" + " double A10 = -lines[i][2], A11 = lines[(i + 1) & 3][2];\n" + " double B0 = -lines[i][0] + lines[(i + 1) & 3][0];\n" + " double B1 = -lines[i][1] + lines[(i + 1) & 3][1];\n" + " double det = A00 * A11 - A10 * A01;\n" + " if (fabs(det) < 0.001) { out[0] = 2u; return; }\n" + " double W00 = A11 / det, W01 = -A01 / det;\n" + " double L0 = W00 * B0 + W01 * B1;\n" + " pf[2 * i] = (float)(lines[i][0] + L0 * A00);\n" + " pf[2 * i + 1] = (float)(lines[i][1] + L0 * A10);\n" + " }\n" + " double area = 0;\n" + " double l0 = sqrt(sqf(pf[2] - pf[0]) + sqf(pf[3] - pf[1]));\n" + " double l1 = sqrt(sqf(pf[4] - pf[2]) + sqf(pf[5] - pf[3]));\n" + " double l2 = sqrt(sqf(pf[0] - pf[4]) + sqf(pf[1] - pf[5]));\n" + " double p = (l0 + l1 + l2) / 2;\n" + " area += sqrt(p * (p - l0) * (p - l1) * (p - l2));\n" + " l0 = sqrt(sqf(pf[6] - pf[4]) + sqf(pf[7] - pf[5]));\n" + " l1 = sqrt(sqf(pf[0] - pf[6]) + sqf(pf[1] - pf[7]));\n" + " l2 = sqrt(sqf(pf[4] - pf[0]) + sqf(pf[5] - pf[1]));\n" + " p = (l0 + l1 + l2) / 2;\n" + " area += sqrt(p * (p - l0) * (p - l1) * (p - l2));\n" + " if (area < 0.95 * tagWidth * tagWidth) { out[0] = 2u; return; }\n" + " for (int i = 0; i < 4; i++) {\n" + " int j1 = (i + 1) & 3, j2 = (i + 2) & 3;\n" + " double dx1 = (double)(pf[2 * j1] - pf[2 * i]);\n" + " double dy1 = (double)(pf[2 * j1 + 1] - pf[2 * i + 1]);\n" + " double dx2 = (double)(pf[2 * j2] - pf[2 * j1]);\n" + " double dy2 = (double)(pf[2 * j2 + 1] - pf[2 * j1 + 1]);\n" + " double denom = sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2));\n" + " if (denom == 0) { out[0] = 2u; return; }\n" + " double cosDt = (dx1 * dx2 + dy1 * dy2) / denom;\n" + " if ((cosDt > maxDot || cosDt < -maxDot) || dx1 * dy2 < dy1 * dx2) { out[0] = 2u; return; }\n" + " }\n" + " out[0] = 1u;\n" + " for (int i = 0; i < 8; i++) out[1 + i] = as_uint(pf[i]);\n" + "}\n"; + static const char *sourceScan = "__kernel void scanLocal(__global const uint *hist, __global uint *offsets,\n" " __global uint *blockSums) {\n" @@ -675,6 +1089,19 @@ static const char *sourceScan = #define FIT_BIG_CAP 32768 // Scratch slices (and so workgroups) per fitSortBig launch. #define FIT_BATCH 256 +// P3 chain limits, also passed to the fit program as build options. +// FIT_MAX_K bounds max_nmaxima; detectors configured above it fall back to +// the CPU fit. The strides are in uints. +#define FIT_MAX_K 16 +#define FIT_OUT_STRIDE 12 +#define FIT_MAXIMA_STRIDE 18 +// Upper bound on summed fit-cluster points per frame; clusters past the cap +// fall back to the CPU fit (bounds lfps/errs scratch at ~256 MB). +#define FIT_POINT_CAP (4u * 1024u * 1024u) +// fitOut status values. +#define FIT_QUAD_PENDING 0u +#define FIT_QUAD_ACCEPT 1u +#define FIT_QUAD_REJECT 2u typedef struct { uint16_t x, y; @@ -706,6 +1133,10 @@ static int oclFitReady = 0; static cl_kernel oclKernelFitPrep; static cl_kernel oclKernelFitSortSlm; static cl_kernel oclKernelFitSortBig; +static cl_kernel oclKernelFitLfpsPrep; +static cl_kernel oclKernelFitLfpsScan; +static cl_kernel oclKernelFitErrs; +static cl_kernel oclKernelFitCombos; typedef struct { int valid; @@ -738,9 +1169,36 @@ typedef struct { cl_mem bufSortScratch; cl_mem bufSortList; cl_mem bufDotTerms; + // P3 chain buffers, grow-only, sized by the frame's fit-cluster load + // rather than the frame geometry. + cl_mem bufFitList; + size_t fitListCap; + cl_mem bufLfps; + size_t lfpsCap; + cl_mem bufErrsRaw; + size_t errsRawCap; + cl_mem bufErrsSmooth; + size_t errsSmoothCap; + cl_mem bufMaxima; + size_t maximaCap; + cl_mem bufFitOut; + size_t fitOutCap; } OclBufferCache; static OclBufferCache cache; + +// One-shot handoff from runClusterChain (which enqueues the gather + fit +// preparation) to oclFitQuads (called later from fit_quads): valid only when +// the decimated grayscale this frame's lfps must sample is still resident in +// bufIm, i.e. on the oclFrontend path. Guarded by oclMutex; every other +// entry point invalidates it. +typedef struct { + int valid; + const zarray_t *clusters; + cl_int cw, ch, cs; +} PendingFit; + +static PendingFit pendingFit; static uint32_t segCountsHost[OCL_SEG_COUNT]; static uint32_t segOffsetsHost[OCL_SEG_COUNT]; @@ -821,13 +1279,38 @@ static void oclInitFitProgram(cl_device_id device) if (!exactDivide) oclDebugLog("no correctly-rounded fp32 divide: slopes may differ in the last ulp"); - char options[160]; - snprintf(options, sizeof(options), "%s -DFIT_SLM_CAP=%d -DFIT_BIG_CAP=%d", - exactDivide ? "-cl-fp32-correctly-rounded-divide-sqrt" : "", FIT_SLM_CAP, FIT_BIG_CAP); + // quad_segment_maxima's low-pass kernel, computed with the exact CPU + // expressions (sigma = 1, cutoff = 0.05) and this process's libm — the + // same values the CPU path computes at runtime — then baked into the + // program as exact hex float literals. + const double sigma = 1.0, cutoff = 0.05; + int fsz = sqrt(-log(cutoff) * 2 * sigma * sigma) + 1; + fsz = 2 * fsz + 1; + if (fsz != 7) { + oclDebugLog("unexpected smoothing kernel size: fit kernels disabled"); + return; + } + float filt[7]; + for (int i = 0; i < 7; i++) { + int j = i - fsz / 2; + filt[i] = exp(-j * j / (2 * sigma * sigma)); + } + + char options[640]; + snprintf(options, sizeof(options), + "%s -DFIT_SLM_CAP=%d -DFIT_BIG_CAP=%d -DFIT_MAX_K=%d -DFIT_OUT_STRIDE=%d" + " -DFIT_MAXIMA_STRIDE=%d -DFILT0=%af -DFILT1=%af -DFILT2=%af -DFILT3=%af" + " -DFILT4=%af -DFILT5=%af -DFILT6=%af", + exactDivide ? "-cl-fp32-correctly-rounded-divide-sqrt" : "", FIT_SLM_CAP, FIT_BIG_CAP, + FIT_MAX_K, FIT_OUT_STRIDE, FIT_MAXIMA_STRIDE, + (double)filt[0], (double)filt[1], (double)filt[2], (double)filt[3], + (double)filt[4], (double)filt[5], (double)filt[6]); cl_int err = CL_SUCCESS; - const char *sources[5] = { sourceFitPrep, sourceFitSortHelpers, sourceFitPrep2, sourceFitSortSlm, sourceFitSort }; - cl_program program = clCreateProgramWithSource(oclContext, 5, sources, NULL, &err); + const char *sources[11] = { sourceFitPrep, sourceFitSortHelpers, sourceFitPrep2, sourceFitSortSlm, + sourceFitSort, sourceFitLfps, sourceFitLine, sourceFitErrs, + sourceFitCombosA, sourceFitCombosA2, sourceFitCombosB }; + cl_program program = clCreateProgramWithSource(oclContext, 11, sources, NULL, &err); if (err != CL_SUCCESS) return; err = clBuildProgram(program, 1, &device, options, NULL, NULL); @@ -838,13 +1321,23 @@ static void oclInitFitProgram(cl_device_id device) clReleaseProgram(program); return; } - oclKernelFitPrep = clCreateKernel(program, "fitPrep", &err); - cl_int err2 = CL_SUCCESS; - oclKernelFitSortSlm = clCreateKernel(program, "fitSortSlm", &err2); - cl_int err3 = CL_SUCCESS; - oclKernelFitSortBig = clCreateKernel(program, "fitSortBig", &err3); + struct { cl_kernel *handle; const char *name; } fitKernels[] = { + { &oclKernelFitPrep, "fitPrep" }, + { &oclKernelFitSortSlm, "fitSortSlm" }, + { &oclKernelFitSortBig, "fitSortBig" }, + { &oclKernelFitLfpsPrep, "fitLfpsPrep" }, + { &oclKernelFitLfpsScan, "fitLfpsScan" }, + { &oclKernelFitErrs, "fitErrs" }, + { &oclKernelFitCombos, "fitCombos" }, + }; + int failed = 0; + for (size_t i = 0; i < sizeof(fitKernels) / sizeof(fitKernels[0]); i++) { + *fitKernels[i].handle = clCreateKernel(program, fitKernels[i].name, &err); + if (err != CL_SUCCESS) + failed = 1; + } clReleaseProgram(program); - if (err == CL_SUCCESS && err2 == CL_SUCCESS && err3 == CL_SUCCESS) + if (!failed) oclFitReady = 1; } @@ -953,7 +1446,14 @@ static void releaseCache(void) releaseBuffer(cache.bufSortScratch); releaseBuffer(cache.bufSortList); releaseBuffer(cache.bufDotTerms); + releaseBuffer(cache.bufFitList); + releaseBuffer(cache.bufLfps); + releaseBuffer(cache.bufErrsRaw); + releaseBuffer(cache.bufErrsSmooth); + releaseBuffer(cache.bufMaxima); + releaseBuffer(cache.bufFitOut); memset(&cache, 0, sizeof(cache)); + pendingFit.valid = 0; } static cl_mem createOrFail(cl_mem_flags flags, size_t bytes, void *host, int *failed) @@ -965,6 +1465,27 @@ static cl_mem createOrFail(cl_mem_flags flags, size_t bytes, void *host, int *fa return buffer; } +// Grow-only allocation for the P3 chain scratch: kept across frames, grown +// with headroom when a frame needs more. +static int ensureChainBuffer(cl_mem *buffer, size_t *capacity, cl_mem_flags flags, size_t bytes) +{ + if (*buffer != NULL && *capacity >= bytes) + return 1; + releaseBuffer(*buffer); + *buffer = NULL; + *capacity = 0; + int failed = 0; + const size_t grown = bytes + bytes / 4; + cl_mem created = createOrFail(flags, grown, NULL, &failed); + if (failed) { + releaseBuffer(created); + return 0; + } + *buffer = created; + *capacity = grown; + return 1; +} + static int ensureCache(cl_int w, cl_int h, cl_int s, cl_int tw, cl_int th) { if (cache.valid && cache.w == w && cache.h == h && cache.s == s) @@ -1088,6 +1609,7 @@ image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im) int ok = 0; pthread_mutex_lock(&oclMutex); + pendingFit.valid = 0; if (!ensureCache(w, h, s, tw, th)) goto done; @@ -2113,7 +2635,7 @@ static zarray_t *buildClustersSorted(apriltag_detector_t *td, const uint64_t *re // inputBuffer, then builds the cluster arrays on the CPU. Caller holds // oclMutex and has a valid cache. labelsReady indicates the classify kernel // already seeded the labels buffer. -static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl_int cw, cl_int ch, cl_int cs, int labelsReady) +static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl_int cw, cl_int ch, cl_int cs, int labelsReady, int grayOnDevice) { const cl_uint minCluster = (cl_uint)td->qtp.min_cluster_pixels; const cl_uint capacity = OCL_RECORD_CAPACITY; @@ -2268,6 +2790,16 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl profHost("fitEnqueue", t); if (getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) validateFitPrep(td, clusters, recordCount, cw, ch); + // The P3 chain (oclFitQuads) samples the decimated + // grayscale for lfps weights; hand off only when bufIm + // still holds it. + if (grayOnDevice) { + pendingFit.valid = 1; + pendingFit.clusters = clusters; + pendingFit.cw = cw; + pendingFit.ch = ch; + pendingFit.cs = cs; + } } else { oclDebugLog("fit prep failed"); } @@ -2301,6 +2833,7 @@ zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int zarray_t *clusters = NULL; pthread_mutex_lock(&oclMutex); + pendingFit.valid = 0; profReset(); if (!ensureCache(w, h, ts, w / 4, h / 4)) goto done; @@ -2324,7 +2857,7 @@ zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int goto done; } cache.thresholdOutputFor = NULL; - clusters = runClusterChain(td, inputBuffer, w, h, ts, labelsReady); + clusters = runClusterChain(td, inputBuffer, w, h, ts, labelsReady, 0); } done: @@ -2359,6 +2892,7 @@ zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im) zarray_t *clusters = NULL; pthread_mutex_lock(&oclMutex); + pendingFit.valid = 0; profReset(); if (!ensureCache(w, h, s, tw, th)) goto done; @@ -2386,7 +2920,7 @@ zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im) if (err != CL_SUCCESS) goto done; - clusters = runClusterChain(td, cache.bufOut, w, h, s, 1); + clusters = runClusterChain(td, cache.bufOut, w, h, s, 1, 1); done: profPrint(); @@ -2395,3 +2929,282 @@ zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im) oclDebugLog("GPU frontend failed, falling back to CPU"); return clusters; } + +// CPU reference for the P3 validation gate (apriltag_quad_thresh.c). +int fit_quad(apriltag_detector_t *td, image_u8_t *im, zarray_t *cluster, struct quad *quad, + int tag_width, bool normal_border, bool reversed_border); + +typedef struct { + uint32_t clusterIdx; + uint32_t lfpsOffset; + uint8_t reversed; +} FitSlot; + +// Development gate for the GPU quad fit: every GPU-fitted cluster is re-fit +// with the production CPU fit_quad (on a copy — fit_quad sorts its input) +// and the verdict plus the corner bits must match exactly. +static void validateFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im, + const FitSlot *slots, uint32_t fitCount, const uint32_t *outBuf) +{ + FitParams params; + computeFitParams(td, (cl_int)im->width, (cl_int)im->height, ¶ms); + + uint32_t failures = 0, accepted = 0, rejected = 0; + for (uint32_t i = 0; i < fitCount; i++) { + zarray_t *cluster; + zarray_get(clusters, (int)slots[i].clusterIdx, &cluster); + zarray_t *copy = zarray_create(cluster->el_sz); + zarray_ensure_capacity(copy, cluster->size); + memcpy(copy->data, cluster->data, (size_t)cluster->size * cluster->el_sz); + copy->size = cluster->size; + struct quad ref; + memset(&ref, 0, sizeof(ref)); + const int res = fit_quad(td, im, copy, &ref, params.tagWidth, + params.normalAllowed != 0, params.reversedAllowed != 0); + zarray_destroy(copy); + + const uint32_t *o = outBuf + (size_t)i * FIT_OUT_STRIDE; + const char *fail = NULL; + if (o[0] == FIT_QUAD_ACCEPT) { + accepted++; + if (res != 1) + fail = "GPU accepted, CPU rejected"; + else if (memcmp(ref.p, o + 1, sizeof(float) * 8) != 0) + fail = "corner bits differ"; + else if ((slots[i].reversed != 0) != ref.reversed_border) + fail = "reversed flag differs"; + } else if (o[0] == FIT_QUAD_REJECT) { + rejected++; + if (res != 0) + fail = "GPU rejected, CPU accepted"; + } else { + fail = "no GPU verdict"; + } + if (fail != NULL) { + if (failures < 5) + fprintf(stderr, "apriltag opencl: fit quads validate: cluster %u (n=%d): %s\n", + slots[i].clusterIdx, cluster->size, fail); + failures++; + } + } + if (failures == 0) + fprintf(stderr, + "apriltag opencl: fit quads validate: PASS (%u fits: %u quads, %u rejected, corner bits exact)\n", + fitCount, accepted, rejected); + else + fprintf(stderr, "apriltag opencl: fit quads validate: FAIL (%u of %u mismatched)\n", + failures, fitCount); +} + +uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im, zarray_t *quads) +{ + if (getenv("APRILTAG_OPENCL") == NULL || clusters == NULL) + return NULL; + if (td->qtp.max_nmaxima < 0 || td->qtp.max_nmaxima > FIT_MAX_K) + return NULL; + + pthread_mutex_lock(&oclMutex); + const uint32_t clusterCount = (uint32_t)zarray_size(clusters); + if (!pendingFit.valid || pendingFit.clusters != clusters || oclFitReady == 0 || + clusterCount == 0 || pendingFit.cw != im->width || pendingFit.ch != im->height || + pendingFit.cs != im->stride) { + pendingFit.valid = 0; + pthread_mutex_unlock(&oclMutex); + return NULL; + } + pendingFit.valid = 0; + const cl_int imW = pendingFit.cw, imH = pendingFit.ch, imS = pendingFit.cs; + profReset(); + + uint8_t *handled = NULL; + FitSlot *slots = NULL; + + // Block on the P2 preparation and sorts, then split the clusters into + // GPU-fit slots and the outcomes the fitPrep flags already decide: the + // pre-fit filters (size, perimeter, bbox area, border direction) reject + // exactly the clusters the CPU path would reject before/inside fit_quad. + double t = hostNowUs(); + cl_int err = CL_SUCCESS; + const uint32_t *meta = clEnqueueMapBuffer(oclQueue, cache.bufFitMeta, CL_TRUE, CL_MAP_READ, 0, + (size_t)clusterCount * 32, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + pthread_mutex_unlock(&oclMutex); + return NULL; + } + profHost("metaWait", t); + + t = hostNowUs(); + handled = calloc(clusterCount, 1); + slots = malloc(sizeof(FitSlot) * (size_t)clusterCount); + int ok = handled != NULL && slots != NULL; + uint32_t fitCount = 0; + uint32_t fitPoints = 0; + const uint32_t skipMask = FIT_SKIP_MINPIX | FIT_SKIP_PERIM | FIT_SKIP_AREA | FIT_SKIP_BORDER; + for (uint32_t c = 0; ok && c < clusterCount; c++) { + const uint32_t flags = meta[8u * c]; + if ((flags & FIT_PROCESSED) == 0) + continue; + if ((flags & skipMask) != 0) { + handled[c] = 1; + continue; + } + if ((flags & FIT_SORTED) == 0) + continue; + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + const uint32_t n = (uint32_t)cluster->size; + if (fitPoints + n > FIT_POINT_CAP) + continue; + slots[fitCount].clusterIdx = c; + slots[fitCount].lfpsOffset = fitPoints; + slots[fitCount].reversed = (flags & FIT_REVERSED) != 0; + fitCount++; + fitPoints += n; + } + err = clEnqueueUnmapMemObject(oclQueue, cache.bufFitMeta, (void *)meta, 0, NULL, NULL); + profHost("fitSplit", t); + if (!ok || err != CL_SUCCESS) + goto fail; + if (fitCount == 0) + goto finish; + + if (!ensureChainBuffer(&cache.bufFitList, &cache.fitListCap, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, (size_t)fitCount * 8) || + !ensureChainBuffer(&cache.bufLfps, &cache.lfpsCap, CL_MEM_READ_WRITE, (size_t)fitPoints * 48) || + !ensureChainBuffer(&cache.bufErrsRaw, &cache.errsRawCap, CL_MEM_READ_WRITE, (size_t)fitPoints * 8) || + !ensureChainBuffer(&cache.bufErrsSmooth, &cache.errsSmoothCap, CL_MEM_READ_WRITE, (size_t)fitPoints * 8) || + !ensureChainBuffer(&cache.bufMaxima, &cache.maximaCap, CL_MEM_READ_WRITE, + (size_t)fitCount * FIT_MAXIMA_STRIDE * 4) || + !ensureChainBuffer(&cache.bufFitOut, &cache.fitOutCap, + CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)fitCount * FIT_OUT_STRIDE * 4)) + goto fail; + + t = hostNowUs(); + { + uint32_t *list = clEnqueueMapBuffer(oclQueue, cache.bufFitList, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, + (size_t)fitCount * 8, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto fail; + for (uint32_t i = 0; i < fitCount; i++) { + list[2 * i] = slots[i].clusterIdx; + list[2 * i + 1] = slots[i].lfpsOffset; + } + err = clEnqueueUnmapMemObject(oclQueue, cache.bufFitList, list, 0, NULL, NULL); + } + const uint32_t zero = 0; + err |= clEnqueueFillBuffer(oclQueue, cache.bufFitOut, &zero, 4, 0, + (size_t)fitCount * FIT_OUT_STRIDE * 4, 0, NULL, NULL); + if (err != CL_SUCCESS) + goto fail; + profHost("fitListUpload", t); + + t = hostNowUs(); + { + FitParams params; + computeFitParams(td, imW, imH, ¶ms); + const cl_uint count = fitCount; + const cl_int maxNmaxima = td->qtp.max_nmaxima; + const cl_double maxDot = (cl_double)td->qtp.cos_critical_rad; + const cl_double maxMse = (cl_double)td->qtp.max_line_fit_mse; + + const cl_uint lfStride = fitPoints; + err |= clSetKernelArg(oclKernelFitLfpsPrep, 0, sizeof(cl_mem), &cache.bufRecordsAlt); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 1, sizeof(cl_mem), &cache.bufSortKeys); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 2, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 3, sizeof(cl_mem), &cache.bufFitList); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 4, sizeof(cl_uint), &count); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 5, sizeof(cl_mem), &cache.bufIm); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 6, sizeof(cl_int), &imW); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 7, sizeof(cl_int), &imH); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 8, sizeof(cl_int), &imS); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 9, sizeof(cl_uint), &lfStride); + err |= clSetKernelArg(oclKernelFitLfpsPrep, 10, sizeof(cl_mem), &cache.bufLfps); + + err |= clSetKernelArg(oclKernelFitLfpsScan, 0, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitLfpsScan, 1, sizeof(cl_mem), &cache.bufFitList); + err |= clSetKernelArg(oclKernelFitLfpsScan, 2, sizeof(cl_uint), &count); + err |= clSetKernelArg(oclKernelFitLfpsScan, 3, sizeof(cl_uint), &lfStride); + err |= clSetKernelArg(oclKernelFitLfpsScan, 4, sizeof(cl_mem), &cache.bufLfps); + + err |= clSetKernelArg(oclKernelFitErrs, 0, sizeof(cl_mem), &cache.bufLfps); + err |= clSetKernelArg(oclKernelFitErrs, 1, sizeof(cl_uint), &lfStride); + err |= clSetKernelArg(oclKernelFitErrs, 2, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitErrs, 3, sizeof(cl_mem), &cache.bufFitList); + err |= clSetKernelArg(oclKernelFitErrs, 4, sizeof(cl_uint), &count); + err |= clSetKernelArg(oclKernelFitErrs, 5, sizeof(cl_int), &maxNmaxima); + err |= clSetKernelArg(oclKernelFitErrs, 6, sizeof(cl_mem), &cache.bufErrsRaw); + err |= clSetKernelArg(oclKernelFitErrs, 7, sizeof(cl_mem), &cache.bufErrsSmooth); + err |= clSetKernelArg(oclKernelFitErrs, 8, sizeof(cl_mem), &cache.bufDotTerms); + err |= clSetKernelArg(oclKernelFitErrs, 9, sizeof(cl_mem), &cache.bufMaxima); + err |= clSetKernelArg(oclKernelFitErrs, 10, sizeof(cl_mem), &cache.bufFitOut); + + err |= clSetKernelArg(oclKernelFitCombos, 0, sizeof(cl_mem), &cache.bufLfps); + err |= clSetKernelArg(oclKernelFitCombos, 1, sizeof(cl_uint), &lfStride); + err |= clSetKernelArg(oclKernelFitCombos, 2, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitCombos, 3, sizeof(cl_mem), &cache.bufFitList); + err |= clSetKernelArg(oclKernelFitCombos, 4, sizeof(cl_uint), &count); + err |= clSetKernelArg(oclKernelFitCombos, 5, sizeof(cl_mem), &cache.bufMaxima); + err |= clSetKernelArg(oclKernelFitCombos, 6, sizeof(cl_double), &maxDot); + err |= clSetKernelArg(oclKernelFitCombos, 7, sizeof(cl_double), &maxMse); + err |= clSetKernelArg(oclKernelFitCombos, 8, sizeof(cl_int), ¶ms.tagWidth); + err |= clSetKernelArg(oclKernelFitCombos, 9, sizeof(cl_mem), &cache.bufFitOut); + if (err != CL_SUCCESS) + goto fail; + + const size_t fitGlobal[1] = { (size_t)fitCount * 256 }; + const size_t fitLocal[1] = { 256 }; + const size_t scanGlobal[1] = { roundUp((size_t)fitCount * 6, 192) }; + const size_t scanLocal[1] = { 192 }; + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitLfpsPrep, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitLfpsPrep")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitLfpsScan, 1, NULL, scanGlobal, scanLocal, 0, NULL, profSlot("fitLfpsScan")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitErrs, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitErrs")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitCombos, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitCombos")); + if (err != CL_SUCCESS) + goto fail; + } + profHost("fitChainEnqueue", t); + + t = hostNowUs(); + { + const uint32_t *outBuf = clEnqueueMapBuffer(oclQueue, cache.bufFitOut, CL_TRUE, CL_MAP_READ, 0, + (size_t)fitCount * FIT_OUT_STRIDE * 4, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + goto fail; + profHost("quadWait", t); + + t = hostNowUs(); + for (uint32_t i = 0; i < fitCount; i++) { + const uint32_t *o = outBuf + (size_t)i * FIT_OUT_STRIDE; + if (o[0] == FIT_QUAD_ACCEPT) { + struct quad quad; + memset(&quad, 0, sizeof(quad)); + memcpy(quad.p, o + 1, sizeof(float) * 8); + quad.reversed_border = slots[i].reversed != 0; + zarray_add(quads, &quad); + handled[slots[i].clusterIdx] = 1; + } else if (o[0] == FIT_QUAD_REJECT) { + handled[slots[i].clusterIdx] = 1; + } + // FIT_QUAD_PENDING: the chain skipped it — CPU fallback. + } + profHost("quadBuild", t); + if (getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) + validateFitQuads(td, clusters, im, slots, fitCount, outBuf); + clEnqueueUnmapMemObject(oclQueue, cache.bufFitOut, (void *)outBuf, 0, NULL, NULL); + } + +finish: + free(slots); + profPrint(); + pthread_mutex_unlock(&oclMutex); + return handled; + +fail: + oclDebugLog("GPU fit quads failed, falling back to CPU"); + free(slots); + free(handled); + profPrint(); + pthread_mutex_unlock(&oclMutex); + return NULL; +} diff --git a/ocl_threshold.h b/ocl_threshold.h index a5fa73fa..5e91bfa6 100644 --- a/ocl_threshold.h +++ b/ocl_threshold.h @@ -23,3 +23,13 @@ zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int // chain — the threshold image never materializes on the host. Same return // contract as oclClusters. zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im); + +// GPU fit_quads tail over the clusters most recently returned by oclFrontend +// (requires APRILTAG_OPENCL_FIT=1): fits quads to the device-resident sorted +// clusters and appends accepted quads — corner bits identical to the CPU's +// fit_quad — to quads. Returns a malloc'd per-cluster array where a 1 marks +// clusters fully decided on the GPU (the caller must skip those and run the +// CPU fit only for the rest), or NULL when the GPU fit is unavailable, in +// which case the caller runs the CPU path for every cluster. The caller +// frees the array. +uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im, zarray_t *quads); From 3639a0927413348f6fbcbb47875578198e97e4e0 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 12:47:56 +0200 Subject: [PATCH 10/18] Run the build walk slim when the GPU fit consumes the frame (P4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With fit_quads on the GPU, the walk's appendPt/merge point copying only fed descriptor sizes, the permutation passes, and CPU-fallback clusters. The walk now skips it whenever the GPU fit will run: the hash grouping, pass-A indices, and merge bookkeeping are unchanged, but clusters carry sizes only (data NULL). buildWalk drops 25.8 -> 7.9 ms. Shells never reach CPU code: materializeShells() rebuilds point data from the gathered records (cluster-contiguous, CPU point order, payloads are exactly struct pt's fields) for fallback clusters after the quad readback and for every shell on oclFitQuads failure paths; flushPendingFit() materializes a pending handoff that another entry point invalidates; an unarmed slim walk is destroyed and re-walked fat from the intact record buffer. oclFitQuads performs all acceptance checks under the mutex so a rejected handoff is flushed, never stranded. Slim mode is disabled when validation envs request host-side points. Gates on w3cj: detect 32/32 at 0.000000 px deterministic; corpus 120/120 at 0.0000 px with slim active; FIT_VALIDATE (fat walk) still 2510/2510 corner-bit exact; out-of-range max_nmaxima interleave falls back cleanly. Detect in fit mode: ~58 ms wall / ~50-74 core-ms vs CPU ~60 / ~290 — wall parity with the CPU baseline at -75-80% CPU. --- ocl_harness/FIT_QUADS_PLAN.md | 59 +++++++++-- ocl_threshold.c | 190 ++++++++++++++++++++++++++++++---- 2 files changed, 219 insertions(+), 30 deletions(-) diff --git a/ocl_harness/FIT_QUADS_PLAN.md b/ocl_harness/FIT_QUADS_PLAN.md index 80c7f4c0..9ebd247a 100644 --- a/ocl_harness/FIT_QUADS_PLAN.md +++ b/ocl_harness/FIT_QUADS_PLAN.md @@ -166,17 +166,54 @@ the cluster build walk disappears entirely. 8-thread CPU. Both modes stay env-selectable: APRILTAG_OPENCL=1 alone for the fastest wall, +APRILTAG_OPENCL_FIT=1 for max CPU offload. -## P3 follow-ups (next session) - -- The slim walk: with the fit on-GPU, the build walk's appendPt/merge - copying (most of its 16-29 ms) only feeds desc sizes, the permutation - passes, and CPU-fallback clusters. Track counts instead of - materializing zarrays; reconstruct the rare fallback cluster from a - bufRecordsAlt range readback (payloads are (x, y, gx, gy) in CPU - point order — exactly struct pt). Biggest remaining CPU+wall lever. -- fitErrs/fitLfpsScan sit at the plane-traffic bandwidth floor; further - wall wins come from the P2 sort polish (merge-path for the SLM sort's - top levels, fitPrep dot throughput) or overlap across frames. +## P4 (DONE, measured on w3cj 2026-06-12): the slim walk + +With the fit on-GPU, the build walk's appendPt/merge copying only fed +desc sizes, the permutation passes, and CPU-fallback clusters. The walk +now runs slim whenever the GPU fit will consume the frame (useFit + +grayOnDevice + fit program ready + max_nmaxima within FIT_MAX_K + no +validation envs): the hash grouping, clusterKeys, recCluster pass A and +the merge's finalIdx/chunkStart bookkeeping are unchanged — grouping +arithmetic identical — but clusters carry only their sizes (data NULL). +buildWalk dropped 25.8 -> 7.9 ms. + +Safety contract: a shell must never reach CPU code. materializeShells() +rebuilds point data from the gathered records (cluster-contiguous, CPU +point order, payload = exactly struct pt's x/y/gx/gy) for: CPU-fallback +clusters after the quad readback (too-big, FIT_POINT_CAP overflow, +status 0), every shell on any oclFitQuads failure path, and — via +flushPendingFit() — a pending handoff invalidated by another entry +point before its fit_quads ran (the owner's clusters are still alive by +construction). If the handoff never arms (gather/fitPrepSort failure), +runClusterChain destroys the shells and re-walks fat from the intact +record buffer. oclFitQuads now performs all acceptance checks under the +mutex so a rejected handoff is always flushed, never stranded. On +materialize OOM, remaining shells are emptied (size 0) so the CPU path +skips rather than dereferences them. + +Gates: detect 32/32 at 0.000000 px deterministic; corpus 120/120 at +0.0000 px (slim active); FIT_VALIDATE forces the fat walk and still +passes 2510/2510 with exact corner bits; an out-of-range max_nmaxima +detector interleaved between slim detects falls back cleanly. + +Measured (vide, interleaved, noisy session +-15%): fit mode now +~51-70 ms wall / ~50-74 core-ms (median ~58/~60) vs gather-only +~41-55 / ~149-193 vs CPU ~58-62 / ~277-306. The fit mode's wall is now +at parity with the CPU baseline and near the frontend-only mode, with +detect CPU at roughly an eighth... a fifth of the CPU baseline +(-75-80%). Corpus totals improved to 2261 ms / 2340 core-ms (from +2523 / 2931 with the fat walk). + +## P4 follow-ups + +- The GPU chain (quadWait ~16 ms) is now the critical-path tail: + fitLfpsScan + fitErrs sit at the plane-traffic bandwidth floor, so + further wall comes from the P2 sort polish (merge-path for the SLM + sort's top levels, fitPrep dot throughput) or cross-frame overlap. +- fitPrepSort's sortList map blocks on the in-order queue until the + gather lands (~9 ms shows up as host fitEnqueue): an event-ordered or + pre-filled list would free that host stall, though the host has no + other work to do there yet. ## Also still open (smaller) diff --git a/ocl_threshold.c b/ocl_threshold.c index f7d64cf3..83987792 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -1194,6 +1194,11 @@ static OclBufferCache cache; // entry point invalidates it. typedef struct { int valid; + // The walk ran in slim mode: the handed-off clusters are size-only + // shells (data == NULL) and any cluster the GPU fit does not decide + // must be materialized from the gathered records before the CPU can + // touch it. + int slim; const zarray_t *clusters; cl_int cw, ch, cs; } PendingFit; @@ -1587,6 +1592,95 @@ static cl_int setThresholdArgs(cl_mem input, cl_int w, cl_int h, cl_int s, cl_in return err; } +static void destroyClusterList(zarray_t *clusters) +{ + for (int i = 0; i < zarray_size(clusters); i++) { + zarray_t *cluster; + zarray_get(clusters, i, &cluster); + zarray_destroy(cluster); + } + zarray_destroy(clusters); +} + +// The slim build walk hands the GPU fit size-only cluster shells. Any +// cluster the GPU does not decide must get its point data back before the +// CPU fit may touch it: the gathered records are cluster-contiguous in the +// CPU emitter's point order and their payloads carry exactly the struct pt +// fields, so a shell rebuilds from its descriptor range. skip[c] != 0 +// keeps that cluster a shell (already decided); NULL materializes every +// shell. On failure the remaining shells are emptied (size 0) so the CPU +// path skips them instead of dereferencing NULL data. Caller holds +// oclMutex. +static int materializeShells(zarray_t *clusters, const uint8_t *skip) +{ + const uint32_t clusterCount = (uint32_t)zarray_size(clusters); + if (clusterCount == 0) + return 1; + + cl_int err = CL_SUCCESS; + int ok = 1; + const uint32_t *desc = clEnqueueMapBuffer(oclQueue, cache.bufClusterDesc, CL_TRUE, CL_MAP_READ, 0, + (size_t)clusterCount * 8, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + desc = NULL; + const uint64_t *gathered = NULL; + if (desc != NULL) { + const uint32_t total = desc[2 * (clusterCount - 1)] + desc[2 * (clusterCount - 1) + 1]; + gathered = clEnqueueMapBuffer(oclQueue, cache.bufRecordsAlt, CL_TRUE, CL_MAP_READ, 0, + (size_t)total * 16, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) + gathered = NULL; + } + + for (uint32_t c = 0; c < clusterCount; c++) { + zarray_t *cluster; + zarray_get(clusters, (int)c, &cluster); + if (cluster->data != NULL || cluster->size == 0) + continue; + if (skip != NULL && skip[c] != 0) + continue; + char *data = NULL; + if (gathered != NULL && desc[2 * c + 1] == (uint32_t)cluster->size) + data = malloc((size_t)cluster->size * cluster->el_sz); + if (data == NULL) { + cluster->size = 0; + ok = 0; + continue; + } + const uint32_t off = desc[2 * c]; + for (int j = 0; j < cluster->size; j++) { + const uint64_t payload = gathered[2 * ((size_t)off + (size_t)j) + 1]; + OclPt *pt = (OclPt *)(data + (size_t)j * cluster->el_sz); + pt->x = (uint16_t)(payload >> 48); + pt->y = (uint16_t)(payload >> 32); + pt->gx = (int16_t)(uint16_t)(payload >> 16); + pt->gy = (int16_t)(uint16_t)payload; + pt->slope = 0.0f; + } + cluster->data = data; + cluster->alloc = cluster->size; + } + + if (gathered != NULL) + clEnqueueUnmapMemObject(oclQueue, cache.bufRecordsAlt, (void *)gathered, 0, NULL, NULL); + if (desc != NULL) + clEnqueueUnmapMemObject(oclQueue, cache.bufClusterDesc, (void *)desc, 0, NULL, NULL); + if (!ok) + oclDebugLog("shell materialize failed: dropped unmaterialized clusters"); + return ok; +} + +// A pending slim handoff still owes its clusters their point data (they +// are alive — the owning detect call has not reached fit_quads yet); +// materialize before this entry point's work invalidates the device +// buffers the shells depend on. +static void flushPendingFit(void) +{ + if (pendingFit.valid && pendingFit.slim) + materializeShells((zarray_t *)pendingFit.clusters, NULL); + pendingFit.valid = 0; +} + image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im) { if (getenv("APRILTAG_OPENCL") == NULL) @@ -1609,7 +1703,7 @@ image_u8_t *oclThreshold(apriltag_detector_t *td, image_u8_t *im) int ok = 0; pthread_mutex_lock(&oclMutex); - pendingFit.valid = 0; + flushPendingFit(); if (!ensureCache(w, h, s, tw, th)) goto done; @@ -1712,6 +1806,10 @@ typedef struct { uint32_t *recCluster; uint64_t *clusterKeys; int clusterCap; + // Count-only walk for the GPU fit path: clusters keep their exact + // sizes but no point data (the GPU consumes the gathered records + // instead; stragglers are materialized from them on demand). + int slim; int failed; } BuildTask; @@ -1756,7 +1854,10 @@ static void doBuildTask(void *p) task->recCluster[i] = table[slot].clusterIdx; zarray_t *cluster; zarray_get(task->clusters, (int)table[slot].clusterIdx, &cluster); - appendPt(cluster, payload); + if (task->slim) + cluster->size++; + else + appendPt(cluster, payload); } free(table); } @@ -1795,7 +1896,7 @@ static void concatAndDestroy(zarray_t *dst, zarray_t *src) // a gather plan is requested, the merge also records where each task-local // cluster lands: its final cluster index and its chunk's start offset // within that final cluster. -static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount, GatherPlan *plan) +static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount, GatherPlan *plan, int slim) { zarray_t *clusters = zarray_create(sizeof(zarray_t *)); HashEntry *table = calloc(OCL_HASH_SIZE, sizeof(HashEntry)); @@ -1840,7 +1941,12 @@ static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount, GatherPlan * taskPlan->finalIdx[i] = table[slot].clusterIdx; taskPlan->chunkStart[i] = (uint32_t)dst->size; } - concatAndDestroy(dst, cluster); + if (slim) { + dst->size += cluster->size; + zarray_destroy(cluster); + } else { + concatAndDestroy(dst, cluster); + } } } zarray_destroy(tasks[t].clusters); @@ -1856,7 +1962,7 @@ static uint32_t *recClusterScratch = NULL; static uint32_t recClusterScratchCap = 0; static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount, - int segsPerRow, cl_int h, GatherPlan *planOut) + int segsPerRow, cl_int h, GatherPlan *planOut, int slim) { int taskCount = (td->wp != NULL && td->nthreads > 1) ? td->nthreads : 1; if (taskCount > 16) @@ -1887,6 +1993,7 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, tasks[t].recCluster = recCluster; tasks[t].clusterCap = 256; tasks[t].clusterKeys = malloc(sizeof(uint64_t) * tasks[t].clusterCap); + tasks[t].slim = slim; tasks[t].failed = 0; while (row < h) { row++; @@ -1913,7 +2020,7 @@ static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, return NULL; } } - return mergeTaskClusters(tasks, taskCount, (recCluster != NULL) ? planOut : NULL); + return mergeTaskClusters(tasks, taskCount, (recCluster != NULL) ? planOut : NULL, slim); } // Stable 6-pass LSD radix sort of the record buffer by the compacted 46-bit @@ -2759,6 +2866,16 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl // Gated until the GPU fit lands; mutually exclusive with the sorted // path, whose cluster order differs from the walk's encounter order. int useGather = !useSorted && (useFit || getenv("APRILTAG_OPENCL_GATHER") != NULL); + // P4: when the GPU fit will consume this frame (and no validation pass + // needs host-side point data), the walk runs slim — it discovers the + // grouping and counts but copies no points; the GPU fit reads the + // gathered records and any cluster left to the CPU is materialized + // from them. Conditions mirror oclFitQuads' acceptance checks so a + // slim handoff cannot be turned down for a knowable reason. + int slimWalk = useFit && grayOnDevice && oclFitReady != 0 && + td->qtp.max_nmaxima >= 0 && td->qtp.max_nmaxima <= FIT_MAX_K && + getenv("APRILTAG_OPENCL_FIT_VALIDATE") == NULL && + getenv("APRILTAG_OPENCL_GATHER_VALIDATE") == NULL; if (useSorted && !sortRecords(recordCount)) goto done; @@ -2774,12 +2891,13 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl clusters = buildClustersSorted(td, (const uint64_t *)mapped, recordCount); else clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch, - useGather ? &plan : NULL); + useGather ? &plan : NULL, slimWalk); profHost("buildWalk", t); t = hostNowUs(); clEnqueueUnmapMemObject(oclQueue, cache.bufRecords, mapped, 0, NULL, NULL); profHost("unmapRecords", t); + int fitArmed = 0; if (useGather && clusters != NULL && plan.taskCount > 0) { if (gatherClusterRecords(td, clusters, &plan, recordCount)) { if (getenv("APRILTAG_OPENCL_GATHER_VALIDATE") != NULL) @@ -2795,10 +2913,12 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl // still holds it. if (grayOnDevice) { pendingFit.valid = 1; + pendingFit.slim = slimWalk; pendingFit.clusters = clusters; pendingFit.cw = cw; pendingFit.ch = ch; pendingFit.cs = cs; + fitArmed = 1; } } else { oclDebugLog("fit prep failed"); @@ -2810,6 +2930,22 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl } destroyGatherPlan(&plan); + // A slim walk whose fit handoff did not arm leaves shells nothing can + // fill: rebuild the clusters in full from the still-intact record + // buffer (failure path; never taken in steady state). + if (slimWalk && !fitArmed && clusters != NULL) { + oclDebugLog("slim walk discarded: rebuilding full clusters"); + destroyClusterList(clusters); + clusters = NULL; + mapped = clEnqueueMapBuffer(oclQueue, cache.bufRecords, CL_TRUE, CL_MAP_READ, 0, + (size_t)recordCount * 16, 0, NULL, NULL, &err); + if (err == CL_SUCCESS) { + clusters = buildClusters(td, (const uint64_t *)mapped, recordCount, segsPerRow, ch, + NULL, 0); + clEnqueueUnmapMemObject(oclQueue, cache.bufRecords, mapped, 0, NULL, NULL); + } + } + done: return clusters; } @@ -2833,7 +2969,7 @@ zarray_t *oclClusters(apriltag_detector_t *td, image_u8_t *threshim, int w, int zarray_t *clusters = NULL; pthread_mutex_lock(&oclMutex); - pendingFit.valid = 0; + flushPendingFit(); profReset(); if (!ensureCache(w, h, ts, w / 4, h / 4)) goto done; @@ -2892,7 +3028,7 @@ zarray_t *oclFrontend(apriltag_detector_t *td, image_u8_t *im) zarray_t *clusters = NULL; pthread_mutex_lock(&oclMutex); - pendingFit.valid = 0; + flushPendingFit(); profReset(); if (!ensureCache(w, h, s, tw, th)) goto done; @@ -2998,21 +3134,25 @@ static void validateFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im, zarray_t *quads) { - if (getenv("APRILTAG_OPENCL") == NULL || clusters == NULL) - return NULL; - if (td->qtp.max_nmaxima < 0 || td->qtp.max_nmaxima > FIT_MAX_K) + if (clusters == NULL) return NULL; + // Every check happens under the lock so a rejected (or mismatched) + // handoff is flushed rather than left pending — a slim handoff owes + // its shells point data before anyone returns to the CPU path. pthread_mutex_lock(&oclMutex); const uint32_t clusterCount = (uint32_t)zarray_size(clusters); if (!pendingFit.valid || pendingFit.clusters != clusters || oclFitReady == 0 || - clusterCount == 0 || pendingFit.cw != im->width || pendingFit.ch != im->height || + clusterCount == 0 || getenv("APRILTAG_OPENCL") == NULL || + td->qtp.max_nmaxima < 0 || td->qtp.max_nmaxima > FIT_MAX_K || + pendingFit.cw != im->width || pendingFit.ch != im->height || pendingFit.cs != im->stride) { - pendingFit.valid = 0; + flushPendingFit(); pthread_mutex_unlock(&oclMutex); return NULL; } pendingFit.valid = 0; + const int slim = pendingFit.slim; const cl_int imW = pendingFit.cw, imH = pendingFit.ch, imS = pendingFit.cs; profReset(); @@ -3027,10 +3167,8 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im cl_int err = CL_SUCCESS; const uint32_t *meta = clEnqueueMapBuffer(oclQueue, cache.bufFitMeta, CL_TRUE, CL_MAP_READ, 0, (size_t)clusterCount * 32, 0, NULL, NULL, &err); - if (err != CL_SUCCESS) { - pthread_mutex_unlock(&oclMutex); - return NULL; - } + if (err != CL_SUCCESS) + goto fail; profHost("metaWait", t); t = hostNowUs(); @@ -3189,12 +3327,24 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im // FIT_QUAD_PENDING: the chain skipped it — CPU fallback. } profHost("quadBuild", t); - if (getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) + if (!slim && getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) validateFitQuads(td, clusters, im, slots, fitCount, outBuf); clEnqueueUnmapMemObject(oclQueue, cache.bufFitOut, (void *)outBuf, 0, NULL, NULL); } finish: + // Slim shells the GPU did not decide are about to meet the CPU fit: + // give them their points back from the gathered records. + if (slim) { + uint32_t fallbacks = 0; + for (uint32_t c = 0; c < clusterCount; c++) + fallbacks += handled[c] == 0; + if (fallbacks > 0) { + t = hostNowUs(); + materializeShells(clusters, handled); + profHost("materialize", t); + } + } free(slots); profPrint(); pthread_mutex_unlock(&oclMutex); @@ -3202,6 +3352,8 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im fail: oclDebugLog("GPU fit quads failed, falling back to CPU"); + if (slim) + materializeShells(clusters, NULL); free(slots); free(handled); profPrint(); From 1b9aaa1f28e1083545b86273bb4101c87eab3db8 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 13:09:24 +0200 Subject: [PATCH 11/18] Upload the fit sort list with a non-blocking write The blocking map of bufSortList stalled the host behind the gather kernel on the in-order queue. Staging the id list in a persistent host scratch and enqueuing a non-blocking write removes the stall (fitEnqueue stamp 8.9 ms -> 0.4 ms; the old stamp also included the profiling-only clFinish, which now runs outside the stamp window). --- ocl_threshold.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index 83987792..a42640a8 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -1960,6 +1960,8 @@ static zarray_t *mergeTaskClusters(BuildTask *tasks, int taskCount, GatherPlan * static uint32_t *recClusterScratch = NULL; static uint32_t recClusterScratchCap = 0; +static uint32_t *sortListScratch = NULL; +static uint32_t sortListScratchCap = 0; static zarray_t *buildClusters(apriltag_detector_t *td, const uint64_t *records, uint32_t recordCount, int segsPerRow, cl_int h, GatherPlan *planOut, int slim) @@ -2325,14 +2327,21 @@ static int fitPrepSort(apriltag_detector_t *td, zarray_t *clusters, cl_int cw, c // The sort runs over host-prefiltered id lists: SLM-sized candidates in // sortList[0..slmCount), oversized ones after them. Only host-checkable // size filters apply here; the sort kernels skip area- and - // border-rejected ids via the meta flags fitPrep writes. - cl_int err = CL_SUCCESS; + // border-rejected ids via the meta flags fitPrep writes. The list is + // staged in a persistent host scratch and uploaded with a non-blocking + // write — a blocking map here would stall the host behind the gather + // kernel on the in-order queue. The scratch stays untouched until the + // next fitPrepSort call, by which time every entry point has issued a + // blocking map that drained this write. + if (sortListScratchCap < clusterCount) { + free(sortListScratch); + sortListScratch = malloc(sizeof(uint32_t) * (size_t)clusterCount); + sortListScratchCap = (sortListScratch != NULL) ? clusterCount : 0; + if (sortListScratch == NULL) + return 0; + } + uint32_t *sortList = sortListScratch; cl_uint slmCount = 0, bigCount = 0; - uint32_t *sortList = clEnqueueMapBuffer(oclQueue, cache.bufSortList, CL_TRUE, - CL_MAP_WRITE_INVALIDATE_REGION, 0, - (size_t)clusterCount * 4, 0, NULL, NULL, &err); - if (err != CL_SUCCESS) - return 0; for (cl_uint c = 0; c < clusterCount; c++) { zarray_t *cluster; zarray_get(clusters, (int)c, &cluster); @@ -2347,7 +2356,10 @@ static int fitPrepSort(apriltag_detector_t *td, zarray_t *clusters, cl_int cw, c if (clusterSize > FIT_SLM_CAP && clusterSize <= FIT_BIG_CAP && clusterSize <= params.perimCap) sortList[slmCount + bigCount++] = c; } - err = clEnqueueUnmapMemObject(oclQueue, cache.bufSortList, sortList, 0, NULL, NULL); + cl_int err = CL_SUCCESS; + if (slmCount + bigCount > 0) + err = clEnqueueWriteBuffer(oclQueue, cache.bufSortList, CL_FALSE, 0, + (size_t)(slmCount + bigCount) * 4, sortList, 0, NULL, NULL); if (err != CL_SUCCESS) return 0; @@ -2395,11 +2407,7 @@ static int fitPrepSort(apriltag_detector_t *td, zarray_t *clusters, cl_int cw, c err |= clSetKernelArg(oclKernelFitSortBig, 5, sizeof(cl_uint), &batch); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitSortBig, 1, NULL, batchGlobal, wgSize, 0, NULL, profSlot("fitSortBig")); } - if (err != CL_SUCCESS) - return 0; - if (profEnabled) - clFinish(oclQueue); - return 1; + return err == CL_SUCCESS; } static uint32_t hostOrderedFloatBits(float f) @@ -2906,6 +2914,11 @@ static zarray_t *runClusterChain(apriltag_detector_t *td, cl_mem inputBuffer, cl t = hostNowUs(); if (fitPrepSort(td, clusters, cw, ch)) { profHost("fitEnqueue", t); + // The frontend profPrint needs the prep/sort events + // complete; outside the fitEnqueue stamp so the stamp + // reflects the real (non-profiled) enqueue cost. + if (profEnabled) + clFinish(oclQueue); if (getenv("APRILTAG_OPENCL_FIT_VALIDATE") != NULL) validateFitPrep(td, clusters, recordCount, cw, ch); // The P3 chain (oclFitQuads) samples the decimated From 5c596a4ab4d6fcbce59b75cb8129df1660be9d02 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 13:13:27 +0200 Subject: [PATCH 12/18] Accumulate the fitPrep gradient dot from local memory chunks The per-point dot terms went through a global scratch buffer (21 MB written then re-read serially by one lane). Staging each 256-point chunk in SLM and letting lane 0 sum it in cluster point order keeps the CPU's exact float accumulation while dropping the round trip; the four bbox reductions also share one tree (max fields complemented) instead of four. fitPrep 2.8-3.1 -> 2.7 ms; dot bits still validate exactly (2510/2510 fits bit-exact). --- ocl_threshold.c | 97 +++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index a42640a8..7c772e40 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -369,27 +369,23 @@ static const char *sourceFitPrep = " m[0] = flags; m[1] = as_uint(cx); m[2] = as_uint(cy); m[3] = as_uint(dot);\n" " m[4] = xmin | (xmax << 16); m[5] = ymin | (ymax << 16); m[6] = n; m[7] = 0u;\n" "}\n" - "inline uint reduceMin(__local uint *sred, int lid, uint v) {\n" - " sred[lid] = v;\n" + // All four bbox reductions share one tree (and its 8 barriers): the + // max fields are stored complemented so a single min reduces them all. + "inline void reduceBbox(__local uint *s4, int lid, uint xmin, uint xmax, uint ymin, uint ymax,\n" + " uint *oXmin, uint *oXmax, uint *oYmin, uint *oYmax) {\n" + " s4[lid] = xmin; s4[256 + lid] = ~xmax; s4[512 + lid] = ymin; s4[768 + lid] = ~ymax;\n" " barrier(CLK_LOCAL_MEM_FENCE);\n" " for (int s = 128; s > 0; s >>= 1) {\n" - " if (lid < s) sred[lid] = min(sred[lid], sred[lid + s]);\n" - " barrier(CLK_LOCAL_MEM_FENCE);\n" - " }\n" - " uint r = sred[0];\n" - " barrier(CLK_LOCAL_MEM_FENCE);\n" - " return r;\n" - "}\n" - "inline uint reduceMax(__local uint *sred, int lid, uint v) {\n" - " sred[lid] = v;\n" - " barrier(CLK_LOCAL_MEM_FENCE);\n" - " for (int s = 128; s > 0; s >>= 1) {\n" - " if (lid < s) sred[lid] = max(sred[lid], sred[lid + s]);\n" + " if (lid < s) {\n" + " s4[lid] = min(s4[lid], s4[lid + s]);\n" + " s4[256 + lid] = min(s4[256 + lid], s4[256 + lid + s]);\n" + " s4[512 + lid] = min(s4[512 + lid], s4[512 + lid + s]);\n" + " s4[768 + lid] = min(s4[768 + lid], s4[768 + lid + s]);\n" + " }\n" " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" - " uint r = sred[0];\n" + " *oXmin = s4[0]; *oXmax = ~s4[256]; *oYmin = s4[512]; *oYmax = ~s4[768];\n" " barrier(CLK_LOCAL_MEM_FENCE);\n" - " return r;\n" "}\n"; static const char *sourceFitSortHelpers = @@ -471,20 +467,23 @@ static const char *sourceFitSortHelpers = static const char *sourceFitPrep2 = // Preparation only — the sort runs in fitSortSlm/fitSortBig so this - // kernel keeps a tiny SLM footprint and full occupancy. The gradient - // dot's per-point terms are computed in parallel; one lane then sums - // the precomputed terms in cluster point order, which reproduces the - // CPU's float accumulation exactly. + // kernel keeps a small SLM footprint and full occupancy. The gradient + // dot's per-point terms are computed chunk by chunk into local memory + // (the chunked walk has the same coalesced access pattern as a strided + // one); one lane sums each chunk in cluster point order, which + // reproduces the CPU's float accumulation exactly without a global + // round trip for the terms. "__kernel void fitPrep(__global const ulong2 *records, __global const uint2 *desc,\n" " uint clusterCount, int minClusterPixels, int perimCap, int tagWidth,\n" " int normalAllowed, int reversedAllowed,\n" - " __global ulong *keys, __global uint *meta, __global float *terms) {\n" + " __global ulong *keys, __global uint *meta) {\n" " uint c = get_group_id(0);\n" " if (c >= clusterCount) return;\n" " uint off = desc[c].x;\n" " uint n = desc[c].y;\n" " int lid = get_local_id(0);\n" - " __local uint sred[256];\n" + " __local uint s4[1024];\n" + " __local float sterms[256];\n" " uint flags = PROCESSED;\n" " if ((int)n < minClusterPixels) flags |= SKIP_MINPIX;\n" " else if ((int)n > perimCap) flags |= SKIP_PERIM;\n" @@ -500,32 +499,37 @@ static const char *sourceFitPrep2 = " lxmin = min(lxmin, px); lxmax = max(lxmax, px);\n" " lymin = min(lymin, py); lymax = max(lymax, py);\n" " }\n" - " uint xmin = reduceMin(sred, lid, lxmin);\n" - " uint xmax = reduceMax(sred, lid, lxmax);\n" - " uint ymin = reduceMin(sred, lid, lymin);\n" - " uint ymax = reduceMax(sred, lid, lymax);\n" + " uint xmin, xmax, ymin, ymax;\n" + " reduceBbox(s4, lid, lxmin, lxmax, lymin, lymax, &xmin, &xmax, &ymin, &ymax);\n" " if ((int)(xmax - xmin) * (int)(ymax - ymin) < tagWidth) {\n" " if (lid == 0) writeMeta(meta, c, flags | SKIP_AREA, 0.0f, 0.0f, 0.0f, xmin, xmax, ymin, ymax, n);\n" " return;\n" " }\n" " float cx = (float)((xmin + xmax) * 0.5 + 0.05118);\n" " float cy = (float)((ymin + ymax) * 0.5 - 0.028581);\n" - " for (uint i = (uint)lid; i < n; i += 256u) {\n" - " ulong payload = records[off + i].y;\n" - " float fx = (float)((payload >> 48) & 0xFFFFul);\n" - " float fy = (float)((payload >> 32) & 0xFFFFul);\n" - " float gx = (float)as_short((ushort)((payload >> 16) & 0xFFFFul));\n" - " float gy = (float)as_short((ushort)(payload & 0xFFFFul));\n" - " float slope = cpuSlope(fx, fy, cx, cy);\n" - " keys[off + i] = ((ulong)orderedFloatBits(slope) << 32) | (ulong)i;\n" - " float dx = fx - cx;\n" - " float dy = fy - cy;\n" - " terms[off + i] = dx * gx + dy * gy;\n" + " float dot = 0.0f;\n" + " for (uint chunk = 0; chunk < n; chunk += 256u) {\n" + " uint i = chunk + (uint)lid;\n" + " if (i < n) {\n" + " ulong payload = records[off + i].y;\n" + " float fx = (float)((payload >> 48) & 0xFFFFul);\n" + " float fy = (float)((payload >> 32) & 0xFFFFul);\n" + " float gx = (float)as_short((ushort)((payload >> 16) & 0xFFFFul));\n" + " float gy = (float)as_short((ushort)(payload & 0xFFFFul));\n" + " float slope = cpuSlope(fx, fy, cx, cy);\n" + " keys[off + i] = ((ulong)orderedFloatBits(slope) << 32) | (ulong)i;\n" + " float dx = fx - cx;\n" + " float dy = fy - cy;\n" + " sterms[lid] = dx * gx + dy * gy;\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " if (lid == 0) {\n" + " uint m = min(n - chunk, 256u);\n" + " for (uint j = 0; j < m; j++) dot += sterms[j];\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" - " barrier(CLK_GLOBAL_MEM_FENCE);\n" " if (lid == 0) {\n" - " float dot = 0.0f;\n" - " for (uint i = 0; i < n; i++) dot += terms[off + i];\n" " int rev = dot < 0;\n" " if (rev) flags |= REVERSED;\n" " if (rev ? !reversedAllowed : !normalAllowed) flags |= SKIP_BORDER;\n" @@ -1168,7 +1172,7 @@ typedef struct { cl_mem bufFitMeta; cl_mem bufSortScratch; cl_mem bufSortList; - cl_mem bufDotTerms; + cl_mem bufMaximaScratch; // P3 chain buffers, grow-only, sized by the frame's fit-cluster load // rather than the frame geometry. cl_mem bufFitList; @@ -1450,7 +1454,7 @@ static void releaseCache(void) releaseBuffer(cache.bufFitMeta); releaseBuffer(cache.bufSortScratch); releaseBuffer(cache.bufSortList); - releaseBuffer(cache.bufDotTerms); + releaseBuffer(cache.bufMaximaScratch); releaseBuffer(cache.bufFitList); releaseBuffer(cache.bufLfps); releaseBuffer(cache.bufErrsRaw); @@ -2264,18 +2268,18 @@ static int ensureFitBuffers(void) cache.bufFitMeta = createOrFail(CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_MAX_CLUSTERS * 32, NULL, &failed); cache.bufSortScratch = createOrFail(CL_MEM_READ_WRITE, (size_t)FIT_BATCH * FIT_BIG_CAP * 8, NULL, &failed); cache.bufSortList = createOrFail(CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, (size_t)OCL_MAX_CLUSTERS * 4, NULL, &failed); - cache.bufDotTerms = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 4, NULL, &failed); + cache.bufMaximaScratch = createOrFail(CL_MEM_READ_WRITE, (size_t)OCL_RECORD_CAPACITY * 4, NULL, &failed); if (failed) { releaseBuffer(cache.bufSortKeys); releaseBuffer(cache.bufFitMeta); releaseBuffer(cache.bufSortScratch); releaseBuffer(cache.bufSortList); - releaseBuffer(cache.bufDotTerms); + releaseBuffer(cache.bufMaximaScratch); cache.bufSortKeys = NULL; cache.bufFitMeta = NULL; cache.bufSortScratch = NULL; cache.bufSortList = NULL; - cache.bufDotTerms = NULL; + cache.bufMaximaScratch = NULL; return 0; } return 1; @@ -2373,7 +2377,6 @@ static int fitPrepSort(apriltag_detector_t *td, zarray_t *clusters, cl_int cw, c err |= clSetKernelArg(oclKernelFitPrep, 7, sizeof(cl_int), ¶ms.reversedAllowed); err |= clSetKernelArg(oclKernelFitPrep, 8, sizeof(cl_mem), &cache.bufSortKeys); err |= clSetKernelArg(oclKernelFitPrep, 9, sizeof(cl_mem), &cache.bufFitMeta); - err |= clSetKernelArg(oclKernelFitPrep, 10, sizeof(cl_mem), &cache.bufDotTerms); if (err != CL_SUCCESS) return 0; const size_t prepGlobal[1] = { (size_t)clusterCount * 256 }; @@ -3286,7 +3289,7 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im err |= clSetKernelArg(oclKernelFitErrs, 5, sizeof(cl_int), &maxNmaxima); err |= clSetKernelArg(oclKernelFitErrs, 6, sizeof(cl_mem), &cache.bufErrsRaw); err |= clSetKernelArg(oclKernelFitErrs, 7, sizeof(cl_mem), &cache.bufErrsSmooth); - err |= clSetKernelArg(oclKernelFitErrs, 8, sizeof(cl_mem), &cache.bufDotTerms); + err |= clSetKernelArg(oclKernelFitErrs, 8, sizeof(cl_mem), &cache.bufMaximaScratch); err |= clSetKernelArg(oclKernelFitErrs, 9, sizeof(cl_mem), &cache.bufMaxima); err |= clSetKernelArg(oclKernelFitErrs, 10, sizeof(cl_mem), &cache.bufFitOut); From 2e4c963861386f16f2637f87a359722d15c9f7f3 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 13:18:16 +0200 Subject: [PATCH 13/18] Split shallow ptsort merges across lanes in the SLM sort fitSortSlm assigned one lane per node at every depth, so the top levels ran almost serial (the depth-0 merge was one lane over the whole cluster). The shallow-depth lane-group split from fitSortBig is now shared (SHALLOW_BODY, parameterized on address space) with a __local merge-path search; deep levels are unchanged. fitSortSlm 4.2 -> 0.8 ms; sorted order still equals ptsort on all tie clusters and fit corners stay bit-exact. The helpers string outgrew the 4095 literal limit and is split in two. --- ocl_threshold.c | 103 +++++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index 7c772e40..2d4755f7 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -430,20 +430,23 @@ static const char *sourceFitSortHelpers = " while (i < lsz) dst[o++] = src[loff + i++]; \\\n" " while (j < rsz) dst[o++] = src[roff + j++];\n" "inline void ptsortMergeL(__local const ulong *src, __local ulong *dst, uint loff, uint lsz, uint rsz) { MERGE_BODY }\n" - "inline void ptsortMergeG(__global const ulong *src, __global ulong *dst, uint loff, uint lsz, uint rsz) { MERGE_BODY }\n" + "inline void ptsortMergeG(__global const ulong *src, __global ulong *dst, uint loff, uint lsz, uint rsz) { MERGE_BODY }\n"; + +static const char *sourceFitSortHelpers2 = // Merge-path split for lane-parallel merges: returns how many elements // of a the exact right-biased serial merge consumes among its first k // outputs, so a lane can start mid-merge and produce an identical // output chunk. - "inline uint mergePathSearch(__global const ulong *a, uint asz, __global const ulong *b, uint bsz, uint k) {\n" - " uint lo = (k > bsz) ? (k - bsz) : 0u;\n" - " uint hi = (k < asz) ? k : asz;\n" - " while (lo < hi) {\n" - " uint mid = (lo + hi) >> 1u;\n" - " if (KLT(a[mid], b[k - mid - 1u])) lo = mid + 1u; else hi = mid;\n" - " }\n" + "#define MPS_BODY \\\n" + " uint lo = (k > bsz) ? (k - bsz) : 0u; \\\n" + " uint hi = (k < asz) ? k : asz; \\\n" + " while (lo < hi) { \\\n" + " uint mid = (lo + hi) >> 1u; \\\n" + " if (KLT(a[mid], b[k - mid - 1u])) lo = mid + 1u; else hi = mid; \\\n" + " } \\\n" " return lo;\n" - "}\n" + "inline uint mergePathSearchG(__global const ulong *a, uint asz, __global const ulong *b, uint bsz, uint k) { MPS_BODY }\n" + "inline uint mergePathSearchL(__local const ulong *a, uint asz, __local const ulong *b, uint bsz, uint k) { MPS_BODY }\n" // One ptsort depth pass: leaves copy (when the parity differs from the // input buffer) and run their network, internal nodes merge their // children from the opposite-parity buffer. Depth-d results land in @@ -463,6 +466,41 @@ static const char *sourceFitSortHelpers = " if (even) MERGEFN(bufB, bufA, noff, hsz, nsz - hsz); \\\n" " else MERGEFN(bufA, bufB, noff, hsz, nsz - hsz); \\\n" " } \\\n" + " }\n" + // Shallow-depth pass (fewer nodes than lanes): each node's merge is + // split across its lane group with the merge-path search, which lets a + // lane start mid-merge and still produce the exact serial output. AS is + // the buffers' address space. + "#define SHALLOW_BODY(AS, MPSFN, NETFN) \\\n" + " uint lanesPerNode = 256u >> d; \\\n" + " uint node = (uint)lid / lanesPerNode; \\\n" + " uint lane = (uint)lid % lanesPerNode; \\\n" + " uint noff, nsz; \\\n" + " if (ptsortNode(n, (uint)d, node, &noff, &nsz)) { \\\n" + " int even = (d & 1) == 0; \\\n" + " AS ulong *src = even ? bufB : bufA; \\\n" + " AS ulong *dst = even ? bufA : bufB; \\\n" + " if (nsz <= 5u) { \\\n" + " if (lane == 0) { \\\n" + " if (!even) for (uint q = 0; q < nsz; q++) bufB[noff + q] = bufA[noff + q]; \\\n" + " if (even) NETFN(bufA + noff, nsz); else NETFN(bufB + noff, nsz); \\\n" + " } \\\n" + " } else { \\\n" + " uint hsz = nsz / 2u; \\\n" + " uint rsz = nsz - hsz; \\\n" + " uint chunk = (nsz + lanesPerNode - 1u) / lanesPerNode; \\\n" + " uint k0 = lane * chunk; \\\n" + " if (k0 < nsz) { \\\n" + " uint k1 = (k0 + chunk < nsz) ? (k0 + chunk) : nsz; \\\n" + " uint ai = MPSFN(src + noff, hsz, src + noff + hsz, rsz, k0); \\\n" + " uint bi = k0 - ai; \\\n" + " for (uint k = k0; k < k1; k++) { \\\n" + " int takeA = (ai < hsz) && ((bi >= rsz) || KLT(src[noff + ai], src[noff + hsz + bi])); \\\n" + " if (takeA) { dst[noff + k] = src[noff + ai]; ai++; } \\\n" + " else { dst[noff + k] = src[noff + hsz + bi]; bi++; } \\\n" + " } \\\n" + " } \\\n" + " } \\\n" " }\n"; static const char *sourceFitPrep2 = @@ -544,7 +582,9 @@ static const char *sourceFitPrep2 = static const char *sourceFitSortSlm = // Slope sort for SLM-sized clusters (ids in sortList): the ptsort - // replica over two SLM buffers. + // replica over two SLM buffers. Deep levels assign one lane per node; + // shallow levels (few, large merges) split each merge across the + // node's lane group with the merge-path search. "__kernel void fitSortSlm(__global ulong *keys, __global const uint2 *desc,\n" " __global uint *meta, __global const uint *sortList, uint count) {\n" " uint g = get_group_id(0);\n" @@ -562,7 +602,11 @@ static const char *sourceFitSortSlm = " __local ulong *bufA = skeysA;\n" " __local ulong *bufB = skeysB;\n" " for (int d = 9; d >= 0; d--) {\n" - " DEPTH_BODY(ptsortMergeL, net5L)\n" + " if ((1u << d) >= 256u) {\n" + " DEPTH_BODY(ptsortMergeL, net5L)\n" + " } else {\n" + " SHALLOW_BODY(__local, mergePathSearchL, net5L)\n" + " }\n" " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" " for (uint i = (uint)lid; i < n; i += 256u) keys[off + i] = skeysA[i];\n" @@ -594,36 +638,7 @@ static const char *sourceFitSort = " if ((1u << d) >= 256u) {\n" " DEPTH_BODY(ptsortMergeG, net5G)\n" " } else {\n" - " uint lanesPerNode = 256u >> d;\n" - " uint node = (uint)lid / lanesPerNode;\n" - " uint lane = (uint)lid % lanesPerNode;\n" - " uint noff, nsz;\n" - " if (ptsortNode(n, (uint)d, node, &noff, &nsz)) {\n" - " int even = (d & 1) == 0;\n" - " __global ulong *src = even ? bufB : bufA;\n" - " __global ulong *dst = even ? bufA : bufB;\n" - " if (nsz <= 5u) {\n" - " if (lane == 0) {\n" - " if (!even) for (uint q = 0; q < nsz; q++) bufB[noff + q] = bufA[noff + q];\n" - " if (even) net5G(bufA + noff, nsz); else net5G(bufB + noff, nsz);\n" - " }\n" - " } else {\n" - " uint hsz = nsz / 2u;\n" - " uint rsz = nsz - hsz;\n" - " uint chunk = (nsz + lanesPerNode - 1u) / lanesPerNode;\n" - " uint k0 = lane * chunk;\n" - " if (k0 < nsz) {\n" - " uint k1 = (k0 + chunk < nsz) ? (k0 + chunk) : nsz;\n" - " uint ai = mergePathSearch(src + noff, hsz, src + noff + hsz, rsz, k0);\n" - " uint bi = k0 - ai;\n" - " for (uint k = k0; k < k1; k++) {\n" - " int takeA = (ai < hsz) && ((bi >= rsz) || KLT(src[noff + ai], src[noff + hsz + bi]));\n" - " if (takeA) { dst[noff + k] = src[noff + ai]; ai++; }\n" - " else { dst[noff + k] = src[noff + hsz + bi]; bi++; }\n" - " }\n" - " }\n" - " }\n" - " }\n" + " SHALLOW_BODY(__global, mergePathSearchG, net5G)\n" " }\n" " barrier(CLK_GLOBAL_MEM_FENCE);\n" " }\n" @@ -1316,10 +1331,10 @@ static void oclInitFitProgram(cl_device_id device) (double)filt[4], (double)filt[5], (double)filt[6]); cl_int err = CL_SUCCESS; - const char *sources[11] = { sourceFitPrep, sourceFitSortHelpers, sourceFitPrep2, sourceFitSortSlm, - sourceFitSort, sourceFitLfps, sourceFitLine, sourceFitErrs, + const char *sources[12] = { sourceFitPrep, sourceFitSortHelpers, sourceFitSortHelpers2, sourceFitPrep2, + sourceFitSortSlm, sourceFitSort, sourceFitLfps, sourceFitLine, sourceFitErrs, sourceFitCombosA, sourceFitCombosA2, sourceFitCombosB }; - cl_program program = clCreateProgramWithSource(oclContext, 11, sources, NULL, &err); + cl_program program = clCreateProgramWithSource(oclContext, 12, sources, NULL, &err); if (err != CL_SUCCESS) return; err = clBuildProgram(program, 1, &device, options, NULL, NULL); From 15b0f3045d3cf4956a6f000b5643abec30abe4db Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 13:44:47 +0200 Subject: [PATCH 14/18] Fuse the lfps moment prep and scan into one per-cluster kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split pair wrote all six raw term planes to global memory and read them straight back (~2/3 of the chain's traffic). The fused kernel computes each 256-point chunk of terms into local memory and lets lanes 0-5 — one per field, SIMD lockstep — extend the six cumulative sums in CPU accumulation order, so each cluster flows through prep and scan independently. Output planes are unchanged and downstream kernels untouched. fitLfpsPrep+fitLfpsScan 8.4 -> fitLfps 4.4 ms; fit-chain span 15.6 -> 11.9 ms; corners stay bit-exact (2510/2510). --- ocl_threshold.c | 137 ++++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 74 deletions(-) diff --git a/ocl_threshold.c b/ocl_threshold.c index 2d4755f7..3f57e138 100644 --- a/ocl_threshold.c +++ b/ocl_threshold.c @@ -646,20 +646,21 @@ static const char *sourceFitSort = "}\n"; static const char *sourceFitLfps = - // compute_lfps replica over the sorted point order (P3), in two kernels - // over a PLANE layout (six per-field planes of lfStride doubles each). - // fitLfpsPrep (one WG per cluster) resolves the sorted indirection, - // samples the grayscale weight, and writes each point's six moment + // compute_lfps replica over the sorted point order (P3), fused: each + // cluster's workgroup resolves the sorted indirection, samples the + // grayscale weight, and computes one 256-point chunk of the six moment // TERMS — the exact per-statement products the CPU forms (W*fx, - // (W*fx)*fx, ...) — straight into the planes, fully parallel and - // coalesced within each plane. fitLfpsScan then turns each plane - // segment into the cumulative sums in place: one lane per (cluster, - // field), a pure sequential add chain in CPU accumulation order — the - // minimal serial work the exactness contract allows. - "__kernel void fitLfpsPrep(__global const ulong2 *records, __global const ulong *keys,\n" - " __global const uint2 *desc, __global const uint2 *fitList, uint count,\n" - " __global const uchar *im, int imW, int imH, int imS,\n" - " uint lfStride, __global double *lfps) {\n" + // (W*fx)*fx, ...) — into local memory; lanes 0-5 (one per field, SIMD + // lockstep) then extend the six cumulative sums in CPU accumulation + // order, the minimal serial work the exactness contract allows. The + // raw terms never travel through global memory (the split prep/scan + // pair moved them out and back — two thirds of the chain's traffic) + // and each cluster flows through prep and scan independently. Output + // planes (six per-field planes of lfStride doubles) are unchanged. + "__kernel void fitLfps(__global const ulong2 *records, __global const ulong *keys,\n" + " __global const uint2 *desc, __global const uint2 *fitList, uint count,\n" + " __global const uchar *im, int imW, int imH, int imS,\n" + " uint lfStride, __global double *lfps) {\n" " uint g = get_group_id(0);\n" " if (g >= count) return;\n" " uint c = fitList[g].x;\n" @@ -667,43 +668,42 @@ static const char *sourceFitLfps = " uint off = desc[c].x;\n" " uint n = desc[c].y;\n" " int lid = get_local_id(0);\n" - " for (uint i = (uint)lid; i < n; i += 256u) {\n" - " ulong payload = records[off + (uint)(keys[off + i] & 0xFFFFFFFFul)].y;\n" - " int px = (int)((payload >> 48) & 0xFFFFul);\n" - " int py = (int)((payload >> 32) & 0xFFFFul);\n" - " double x = px * 0.5 + 0.5;\n" - " double y = py * 0.5 + 0.5;\n" - " int ix = (int)x, iy = (int)y;\n" - " double W = 1.0;\n" - " if (ix > 0 && ix + 1 < imW && iy > 0 && iy + 1 < imH) {\n" - " int gradX = (int)im[iy * imS + ix + 1] - (int)im[iy * imS + ix - 1];\n" - " int gradY = (int)im[(iy + 1) * imS + ix] - (int)im[(iy - 1) * imS + ix];\n" - " W = sqrt((double)(gradX * gradX + gradY * gradY)) + 1.0;\n" + " __local double terms[256 * 6];\n" + " double acc = 0.0;\n" + " for (uint chunk = 0; chunk < n; chunk += 256u) {\n" + " uint i = chunk + (uint)lid;\n" + " if (i < n) {\n" + " ulong payload = records[off + (uint)(keys[off + i] & 0xFFFFFFFFul)].y;\n" + " int px = (int)((payload >> 48) & 0xFFFFul);\n" + " int py = (int)((payload >> 32) & 0xFFFFul);\n" + " double x = px * 0.5 + 0.5;\n" + " double y = py * 0.5 + 0.5;\n" + " int ix = (int)x, iy = (int)y;\n" + " double W = 1.0;\n" + " if (ix > 0 && ix + 1 < imW && iy > 0 && iy + 1 < imH) {\n" + " int gradX = (int)im[iy * imS + ix + 1] - (int)im[iy * imS + ix - 1];\n" + " int gradY = (int)im[(iy + 1) * imS + ix] - (int)im[(iy - 1) * imS + ix];\n" + " W = sqrt((double)(gradX * gradX + gradY * gradY)) + 1.0;\n" + " }\n" + " double fx = x, fy = y;\n" + " __local double *t = terms + 6 * lid;\n" + " t[0] = W * fx;\n" + " t[1] = W * fy;\n" + " t[2] = W * fx * fx;\n" + " t[3] = W * fx * fy;\n" + " t[4] = W * fy * fy;\n" + " t[5] = W;\n" " }\n" - " double fx = x, fy = y;\n" - " __global double *p = lfps + lo + i;\n" - " p[0] = W * fx;\n" - " p[lfStride] = W * fy;\n" - " p[2u * lfStride] = W * fx * fx;\n" - " p[3u * lfStride] = W * fx * fy;\n" - " p[4u * lfStride] = W * fy * fy;\n" - " p[5u * lfStride] = W;\n" - " }\n" - "}\n" - "__kernel void fitLfpsScan(__global const uint2 *desc, __global const uint2 *fitList, uint count,\n" - " uint lfStride, __global double *lfps) {\n" - " uint t = get_global_id(0);\n" - " uint slot = t / 6u;\n" - " uint field = t % 6u;\n" - " if (slot >= count) return;\n" - " uint c = fitList[slot].x;\n" - " uint lo = fitList[slot].y;\n" - " uint n = desc[c].y;\n" - " __global double *plane = lfps + field * lfStride + lo;\n" - " double acc = 0;\n" - " for (uint i = 0; i < n; i++) {\n" - " acc += plane[i];\n" - " plane[i] = acc;\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" + " if (lid < 6) {\n" + " uint m = min(n - chunk, 256u);\n" + " __global double *plane = lfps + (uint)lid * lfStride + lo + chunk;\n" + " for (uint j = 0; j < m; j++) {\n" + " acc += terms[6u * j + (uint)lid];\n" + " plane[j] = acc;\n" + " }\n" + " }\n" + " barrier(CLK_LOCAL_MEM_FENCE);\n" " }\n" "}\n"; @@ -1152,8 +1152,7 @@ static int oclFitReady = 0; static cl_kernel oclKernelFitPrep; static cl_kernel oclKernelFitSortSlm; static cl_kernel oclKernelFitSortBig; -static cl_kernel oclKernelFitLfpsPrep; -static cl_kernel oclKernelFitLfpsScan; +static cl_kernel oclKernelFitLfps; static cl_kernel oclKernelFitErrs; static cl_kernel oclKernelFitCombos; @@ -1349,8 +1348,7 @@ static void oclInitFitProgram(cl_device_id device) { &oclKernelFitPrep, "fitPrep" }, { &oclKernelFitSortSlm, "fitSortSlm" }, { &oclKernelFitSortBig, "fitSortBig" }, - { &oclKernelFitLfpsPrep, "fitLfpsPrep" }, - { &oclKernelFitLfpsScan, "fitLfpsScan" }, + { &oclKernelFitLfps, "fitLfps" }, { &oclKernelFitErrs, "fitErrs" }, { &oclKernelFitCombos, "fitCombos" }, }; @@ -3278,23 +3276,17 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im const cl_double maxMse = (cl_double)td->qtp.max_line_fit_mse; const cl_uint lfStride = fitPoints; - err |= clSetKernelArg(oclKernelFitLfpsPrep, 0, sizeof(cl_mem), &cache.bufRecordsAlt); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 1, sizeof(cl_mem), &cache.bufSortKeys); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 2, sizeof(cl_mem), &cache.bufClusterDesc); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 3, sizeof(cl_mem), &cache.bufFitList); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 4, sizeof(cl_uint), &count); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 5, sizeof(cl_mem), &cache.bufIm); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 6, sizeof(cl_int), &imW); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 7, sizeof(cl_int), &imH); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 8, sizeof(cl_int), &imS); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 9, sizeof(cl_uint), &lfStride); - err |= clSetKernelArg(oclKernelFitLfpsPrep, 10, sizeof(cl_mem), &cache.bufLfps); - - err |= clSetKernelArg(oclKernelFitLfpsScan, 0, sizeof(cl_mem), &cache.bufClusterDesc); - err |= clSetKernelArg(oclKernelFitLfpsScan, 1, sizeof(cl_mem), &cache.bufFitList); - err |= clSetKernelArg(oclKernelFitLfpsScan, 2, sizeof(cl_uint), &count); - err |= clSetKernelArg(oclKernelFitLfpsScan, 3, sizeof(cl_uint), &lfStride); - err |= clSetKernelArg(oclKernelFitLfpsScan, 4, sizeof(cl_mem), &cache.bufLfps); + err |= clSetKernelArg(oclKernelFitLfps, 0, sizeof(cl_mem), &cache.bufRecordsAlt); + err |= clSetKernelArg(oclKernelFitLfps, 1, sizeof(cl_mem), &cache.bufSortKeys); + err |= clSetKernelArg(oclKernelFitLfps, 2, sizeof(cl_mem), &cache.bufClusterDesc); + err |= clSetKernelArg(oclKernelFitLfps, 3, sizeof(cl_mem), &cache.bufFitList); + err |= clSetKernelArg(oclKernelFitLfps, 4, sizeof(cl_uint), &count); + err |= clSetKernelArg(oclKernelFitLfps, 5, sizeof(cl_mem), &cache.bufIm); + err |= clSetKernelArg(oclKernelFitLfps, 6, sizeof(cl_int), &imW); + err |= clSetKernelArg(oclKernelFitLfps, 7, sizeof(cl_int), &imH); + err |= clSetKernelArg(oclKernelFitLfps, 8, sizeof(cl_int), &imS); + err |= clSetKernelArg(oclKernelFitLfps, 9, sizeof(cl_uint), &lfStride); + err |= clSetKernelArg(oclKernelFitLfps, 10, sizeof(cl_mem), &cache.bufLfps); err |= clSetKernelArg(oclKernelFitErrs, 0, sizeof(cl_mem), &cache.bufLfps); err |= clSetKernelArg(oclKernelFitErrs, 1, sizeof(cl_uint), &lfStride); @@ -3323,10 +3315,7 @@ uint8_t *oclFitQuads(apriltag_detector_t *td, zarray_t *clusters, image_u8_t *im const size_t fitGlobal[1] = { (size_t)fitCount * 256 }; const size_t fitLocal[1] = { 256 }; - const size_t scanGlobal[1] = { roundUp((size_t)fitCount * 6, 192) }; - const size_t scanLocal[1] = { 192 }; - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitLfpsPrep, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitLfpsPrep")); - err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitLfpsScan, 1, NULL, scanGlobal, scanLocal, 0, NULL, profSlot("fitLfpsScan")); + err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitLfps, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitLfps")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitErrs, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitErrs")); err |= clEnqueueNDRangeKernel(oclQueue, oclKernelFitCombos, 1, NULL, fitGlobal, fitLocal, 0, NULL, profSlot("fitCombos")); if (err != CL_SUCCESS) From 02a469a178e0875ba2073e69d5fda3086c621efb Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 16:05:59 +0200 Subject: [PATCH 15/18] Add the vide live-test harness and robot setup notes vide_rehearsal.py A/Bs this build against the stock library through the production dt_apriltags entry point; vide_markers.json arms a robot's vide detector loop over gRPC. The README documents the robot-local deployment used on w3cj (staged .so + bind-mount drop-in, immutable-/etc caveat, YAML camera config, rollback). --- ocl_harness/README.md | 30 ++++++++++++++++++ ocl_harness/vide_markers.json | 19 +++++++++++ ocl_harness/vide_rehearsal.py | 59 +++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 ocl_harness/vide_markers.json create mode 100644 ocl_harness/vide_rehearsal.py diff --git a/ocl_harness/README.md b/ocl_harness/README.md index dd59e044..c690b9be 100644 --- a/ocl_harness/README.md +++ b/ocl_harness/README.md @@ -34,3 +34,33 @@ Run with `OCL_ICD_VENDORS` pointing at the Intel OpenCL ICD and - `cluster_harness.c` — GPU cluster extraction equivalence vs gradient_clusters. - `profile_harness.c` — per-stage timeprofile of apriltag_detector_detect. - `fp64_probe.c` — Arc 140T fp64 throughput probe in the lfps/fit shape. +- `vide_rehearsal.py` — production-entry-point A/B: loads the stock library + and this build side by side through dt_apriltags (vision-prod python, + prod settings) and compares corners bitwise plus wall/CPU per detect. +- `vide_markers.json` — `SetWorldMarkersRequest` payload (deprecated + single-scene form) that arms a robot's vide detector loop with four + virtual tagStandard52h13 markers (ids 40000+ so nothing physical can + match): `grpcurl -plaintext -d @ :5001 + terraform.vide.VideService/SetWorldMarkers < vide_markers.json`. + +## Live test on a robot (vide) + +Verified on w3cj 2026-06-11 (stock 3.54 -> GPU fit 1.29 cores of vide +service CPU). All robot-local, no closure rebuild: + +1. Stage `libapriltag.so` at `/var/lib/vide-gputest/` (persistent; pin its + nix-store deps with gcroot symlinks against GC). +2. If the robot has no VideConfig in rotunda, add a YAML config + (camera serial + `arcade_udp: 127.0.0.1:4040 source_port 15000`) and an + ExecStart drop-in appending `--config`. +3. GPU drop-in: `BindReadOnlyPaths=` our .so over the closure's + `dt_apriltags/libapriltag.so` (the bind follows the store symlink to the + real file) plus the APRILTAG_OPENCL/OCL_ICD_VENDORS environment. + NOTE: `/etc` is immutable on NixOS 25.11 images — drop-ins go in + `/run/systemd/system/vide.service.d/` and must be re-applied after a + reboot. +4. Arm the detector with `vide_markers.json` (markers persist across vide + restarts via its state cache). +5. Rollback: remove the GPU drop-in, `daemon-reload`, restart — stock + binary guaranteed; in-lib failures already fall back to the bit-exact + CPU path silently. diff --git a/ocl_harness/vide_markers.json b/ocl_harness/vide_markers.json new file mode 100644 index 00000000..8af264c3 --- /dev/null +++ b/ocl_harness/vide_markers.json @@ -0,0 +1,19 @@ +{ + "name": "gputest-virtual", + "markers": { + "markers": [ + {"id": 40000, "family": "tagStandard52h13", "corners": { + "topLeft": {"x": 0, "y": 2000, "z": 1500}, "topRight": {"x": 100, "y": 2000, "z": 1500}, + "bottomRight": {"x": 100, "y": 2000, "z": 1400}, "bottomLeft": {"x": 0, "y": 2000, "z": 1400}}}, + {"id": 40001, "family": "tagStandard52h13", "corners": { + "topLeft": {"x": 1000, "y": 2000, "z": 1500}, "topRight": {"x": 1100, "y": 2000, "z": 1500}, + "bottomRight": {"x": 1100, "y": 2000, "z": 1400}, "bottomLeft": {"x": 1000, "y": 2000, "z": 1400}}}, + {"id": 40002, "family": "tagStandard52h13", "corners": { + "topLeft": {"x": 0, "y": 2000, "z": 500}, "topRight": {"x": 100, "y": 2000, "z": 500}, + "bottomRight": {"x": 100, "y": 2000, "z": 400}, "bottomLeft": {"x": 0, "y": 2000, "z": 400}}}, + {"id": 40003, "family": "tagStandard52h13", "corners": { + "topLeft": {"x": 1000, "y": 2000, "z": 500}, "topRight": {"x": 1100, "y": 2000, "z": 500}, + "bottomRight": {"x": 1100, "y": 2000, "z": 400}, "bottomLeft": {"x": 1000, "y": 2000, "z": 400}}} + ] + } +} diff --git a/ocl_harness/vide_rehearsal.py b/ocl_harness/vide_rehearsal.py new file mode 100644 index 00000000..c1305de9 --- /dev/null +++ b/ocl_harness/vide_rehearsal.py @@ -0,0 +1,59 @@ +import os +import resource +import statistics +import time + +import cv2 +import numpy as np +from dt_apriltags import Detector + +os.environ["APRILTAG_OPENCL"] = "1" +os.environ["APRILTAG_OPENCL_FIT"] = "1" + +image = cv2.imread("/tmp/raw_color.pgm", cv2.IMREAD_GRAYSCALE) +print(f"fixture: {image.shape[1]}x{image.shape[0]}") + + +def makeDetector(searchpath): + kwargs = dict(families="tagStandard52h13", nthreads=4, quad_decimate=1.0, + quad_sigma=0.0, refine_edges=1, decode_sharpening=0.0) + if searchpath is not None: + kwargs["searchpath"] = searchpath + try: + return Detector(min_cluster_pixels=100, **kwargs) + except TypeError: + return Detector(**kwargs) + + +cpuDet = makeDetector(None) +gpuDet = makeDetector(["/tmp/atspike/build"]) + +cpuRes = sorted(cpuDet.detect(image), key=lambda d: d.tag_id) +gpuRes = sorted(gpuDet.detect(image), key=lambda d: d.tag_id) +print(f"tags: stock {len(cpuRes)} gpu {len(gpuRes)}") +ids = [d.tag_id for d in cpuRes] == [d.tag_id for d in gpuRes] +maxDelta = max(float(np.abs(c.corners - g.corners).max()) for c, g in zip(cpuRes, gpuRes)) +print(f"id match: {ids} max corner delta: {maxDelta:.9f} px") + + +def benchLoop(det, iters=10): + times = [] + r0 = resource.getrusage(resource.RUSAGE_SELF) + c0 = r0.ru_utime + r0.ru_stime + for _ in range(iters): + start = time.perf_counter() + det.detect(image) + times.append((time.perf_counter() - start) * 1000) + r1 = resource.getrusage(resource.RUSAGE_SELF) + coreMs = (r1.ru_utime + r1.ru_stime - c0) * 1000 / iters + return statistics.median(times), min(times), coreMs + + +for det in (cpuDet, gpuDet): + benchLoop(det, 3) + +print(f"{'round':>5} {'lib':>5} {'median_ms':>10} {'min_ms':>8} {'core_ms':>8}") +for rnd in range(3): + for name, det in (("stock", cpuDet), ("gpu", gpuDet)): + med, mn, core = benchLoop(det) + print(f"{rnd:>5} {name:>5} {med:>10.1f} {mn:>8.1f} {core:>8.1f}") From ed55efb36ab3a81b91b85747d42398f00a0200f2 Mon Sep 17 00:00:00 2001 From: James McVay Date: Thu, 11 Jun 2026 23:54:04 +0200 Subject: [PATCH 16/18] Add the fleet experiment toolkit from the live-test campaign Python harnesses that ran on the robots via the vision closure's dt_apriltags: tier_probe (stock/frontend/fit A/B on one frame), tier_burn (extended reversed-order suite with thermal/clock trajectory), power_probe (synchronized phase frequency/RAPL sampling that settled the GPU power-coupling question), clock_spinner (pinned work-rate clock proxy), galaxy_burst (raw camera capture while vide is stopped), frame_density/frame_structure (threshold-level scene analysis), and mock_vide (launcher attempting to shim the broken upstream --mock camera path). --- ocl_harness/clock_spinner.py | 14 +++++ ocl_harness/frame_density.py | 23 +++++++ ocl_harness/frame_structure.py | 26 ++++++++ ocl_harness/galaxy_burst.py | 20 ++++++ ocl_harness/mock_vide.py | 20 ++++++ ocl_harness/power_probe.py | 109 +++++++++++++++++++++++++++++++++ ocl_harness/tier_burn.py | 80 ++++++++++++++++++++++++ ocl_harness/tier_probe.py | 61 ++++++++++++++++++ 8 files changed, 353 insertions(+) create mode 100644 ocl_harness/clock_spinner.py create mode 100644 ocl_harness/frame_density.py create mode 100644 ocl_harness/frame_structure.py create mode 100644 ocl_harness/galaxy_burst.py create mode 100644 ocl_harness/mock_vide.py create mode 100644 ocl_harness/power_probe.py create mode 100644 ocl_harness/tier_burn.py create mode 100644 ocl_harness/tier_probe.py diff --git a/ocl_harness/clock_spinner.py b/ocl_harness/clock_spinner.py new file mode 100644 index 00000000..a9de5280 --- /dev/null +++ b/ocl_harness/clock_spinner.py @@ -0,0 +1,14 @@ +import time + +windows = [] +end = time.time() + 50 +while time.time() < end: + t0 = time.perf_counter() + n = 0 + x = 1.0 + while time.perf_counter() - t0 < 1.0: + for _ in range(10000): + x = x * 1.0000001 + 0.1 + n += 10000 + windows.append(n / 1e6) + print(f"{time.strftime('%H:%M:%S')} {windows[-1]:.2f} Miter/s", flush=True) diff --git a/ocl_harness/frame_density.py b/ocl_harness/frame_density.py new file mode 100644 index 00000000..dc209aa0 --- /dev/null +++ b/ocl_harness/frame_density.py @@ -0,0 +1,23 @@ +import sys + +import cv2 +import numpy as np + +for path in sys.argv[1:]: + im = cv2.imread(path, cv2.IMREAD_GRAYSCALE) + h, w = im.shape + tw, th = w // 4, h // 4 + crop = im[:th * 4, :tw * 4] + tiles = crop.reshape(th, 4, tw, 4) + tmin, tmax = tiles.min(axis=(1, 3)), tiles.max(axis=(1, 3)) + k = np.ones((3, 3), np.uint8) + bmin, bmax = cv2.erode(tmin, k), cv2.dilate(tmax, k) + mn = np.repeat(np.repeat(bmin, 4, 0), 4, 1).astype(int) + mx = np.repeat(np.repeat(bmax, 4, 0), 4, 1).astype(int) + grey = (mx - mn) < 5 + t = np.where(grey, 127, np.where(crop > mn + (mx - mn) // 2, 255, 0)) + pairs = (((t[:, :-1] + t[:, 1:]) == 255).sum() + + ((t[:-1, :] + t[1:, :]) == 255).sum()) + active = (t != 127).mean() + name = path.split("/")[3] + print(f"{name}: boundary pairs {pairs/1e6:.2f}M, active px {active*100:.0f}%") diff --git a/ocl_harness/frame_structure.py b/ocl_harness/frame_structure.py new file mode 100644 index 00000000..e91fd2ec --- /dev/null +++ b/ocl_harness/frame_structure.py @@ -0,0 +1,26 @@ +import sys + +import cv2 +import numpy as np + +for path in sys.argv[1:]: + im = cv2.imread(path, cv2.IMREAD_GRAYSCALE) + h, w = im.shape + tw, th = w // 4, h // 4 + crop = im[:th * 4, :tw * 4] + tiles = crop.reshape(th, 4, tw, 4) + tmin, tmax = tiles.min(axis=(1, 3)), tiles.max(axis=(1, 3)) + k = np.ones((3, 3), np.uint8) + mn = np.repeat(np.repeat(cv2.erode(tmin, k), 4, 0), 4, 1).astype(int) + mx = np.repeat(np.repeat(cv2.dilate(tmax, k), 4, 0), 4, 1).astype(int) + grey = (mx - mn) < 5 + t = np.where(grey, 127, np.where(crop > mn + (mx - mn) // 2, 255, 0)).astype(np.uint8) + name = path.split("/")[3] if "realframes" in path else path + for val, label in ((255, "white"), (0, "black")): + mask = (t == val).astype(np.uint8) + n, _, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8) + areas = np.sort(stats[1:, cv2.CC_STAT_AREA])[::-1] + total = areas.sum() if len(areas) else 1 + print(f"{name} {label}: {n-1} comps, largest {areas[0]/1e6:.2f}Mpx " + f"({100*areas[0]/total:.0f}% of {label}), top5 {100*areas[:5].sum()/total:.0f}%, " + f">=100px comps {(areas>=100).sum()}") diff --git a/ocl_harness/galaxy_burst.py b/ocl_harness/galaxy_burst.py new file mode 100644 index 00000000..7ab53112 --- /dev/null +++ b/ocl_harness/galaxy_burst.py @@ -0,0 +1,20 @@ +import sys +import time +from pathlib import Path + +from terra.devices.galaxy import open_camera + +serial, out = sys.argv[1], Path(sys.argv[2]) +out.mkdir(parents=True, exist_ok=True) +cam = open_camera(is_gige=False, serial_number=serial) +for _ in range(10): + cam.get_view() +views = [] +t0 = time.time() +while time.time() - t0 < 10 and len(views) < 200: + views.append(cam.get_view()) +dt = time.time() - t0 +print(f"captured {len(views)} frames in {dt:.1f}s ({len(views)/dt:.1f} fps), saving...") +for i, v in enumerate(views): + v.save(out / f"burst_{i:04d}") +print("saved") diff --git a/ocl_harness/mock_vide.py b/ocl_harness/mock_vide.py new file mode 100644 index 00000000..18f94e7f --- /dev/null +++ b/ocl_harness/mock_vide.py @@ -0,0 +1,20 @@ +import sys + +import terra.devices.mock_camera as mockCamera +from terra.devices.base_camera import AutoExposureOptions +from terra.devices.continuous_reader import ContinuousCameraReader + +for cls in (mockCamera.FilesystemCameraReader, ContinuousCameraReader): + cls.set_auto_exposure_options = lambda *a, **k: None + cls.set_auto_exposure = lambda *a, **k: None + cls.set_auto_exposure_roi = lambda *a, **k: None + cls.get_auto_exposure_options = lambda self: AutoExposureOptions() + cls.get_auto_exposure = lambda self: False + cls.get_auto_exposure_enabled = lambda self: False + cls.get_exposure_range = lambda self: (1.0, 1000000.0) + +sys.argv = ["vide", "--machine", "W3cj", "--config", "/var/lib/vide-gputest/vide.yaml", + "--mock", "/tmp/realframes/w3y"] +from apps.run_vide import main + +main() diff --git a/ocl_harness/power_probe.py b/ocl_harness/power_probe.py new file mode 100644 index 00000000..eb668c17 --- /dev/null +++ b/ocl_harness/power_probe.py @@ -0,0 +1,109 @@ +import glob +import os +import statistics +import subprocess +import sys +import threading +import time + +import cv2 + +os.environ["OCL_ICD_VENDORS"] = "/run/opengl-driver/etc/OpenCL/vendors" +from dt_apriltags import Detector + +PHASE_SECONDS = 15 +cpuDirs = sorted(glob.glob("/sys/devices/system/cpu/cpu[0-9]*"), key=lambda p: int(p.split("cpu")[-1])) +maxFreqs = [int(open(c + "/cpufreq/cpuinfo_max_freq").read()) for c in cpuDirs] +pMax = max(maxFreqs) +pCores = [c for c, m in zip(cpuDirs, maxFreqs) if m == pMax] +raplPath = next(iter(glob.glob("/sys/class/powercap/intel-rapl:0/energy_uj")), None) +gpuPath = next(iter(glob.glob("/sys/class/drm/card0/gt_cur_freq_mhz")), None) + +samples = [] +phase = ["startup"] +stop = [False] + + +def sampler(): + lastE = int(open(raplPath).read()) if raplPath else 0 + lastT = time.time() + while not stop[0]: + time.sleep(0.25) + now = time.time() + e = int(open(raplPath).read()) if raplPath else 0 + watts = (e - lastE) / 1e6 / (now - lastT) if raplPath and e >= lastE else None + lastE, lastT = e, now + samples.append({ + "phase": phase[0], + "spinCore": int(open(pCores[0] + "/cpufreq/scaling_cur_freq").read()) // 1000, + "pCores": [int(open(c + "/cpufreq/scaling_cur_freq").read()) // 1000 for c in pCores], + "gpu": int(open(gpuPath).read()) if gpuPath else None, + "watts": watts, + }) + + +img = cv2.imread("/tmp/probeframe.png", cv2.IMREAD_GRAYSCALE) +det = Detector(families="tagStandard52h13", nthreads=4, quad_decimate=1.0, + searchpath=["/var/lib/vide-gputest"]) +os.environ.pop("APRILTAG_OPENCL", None) +det.detect(img) + +spinSrc = "import time\nx = 1.0\nwhile True:\n for _ in range(100000): x = x * 1.0000001 + 0.1\n" + +t = threading.Thread(target=sampler, daemon=True) +t.start() + +detectTimes = {} + + +def runPhase(name, spin, detectMode): + spinner = None + if spin: + spinner = subprocess.Popen(["taskset", "-c", pCores[0].split("cpu")[-1], + sys.executable, "-c", spinSrc], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1) + phase[0] = name + end = time.time() + PHASE_SECONDS + times = [] + if detectMode == "cpu": + os.environ.pop("APRILTAG_OPENCL", None) + os.environ.pop("APRILTAG_OPENCL_FIT", None) + elif detectMode == "gpu": + os.environ["APRILTAG_OPENCL"] = "1" + os.environ["APRILTAG_OPENCL_FIT"] = "1" + if detectMode: + det.detect(img) + while time.time() < end: + if detectMode: + s = time.perf_counter() + det.detect(img) + times.append((time.perf_counter() - s) * 1000) + else: + time.sleep(0.2) + phase[0] = "between" + if times: + detectTimes[name] = statistics.median(times) + if spinner: + spinner.terminate() + time.sleep(1) + + +runPhase("idle", spin=False, detectMode=None) +runPhase("spinAlone", spin=True, detectMode=None) +runPhase("spinPlusCpuDetect", spin=True, detectMode="cpu") +runPhase("spinPlusGpuDetect", spin=True, detectMode="gpu") +stop[0] = True +t.join(timeout=2) + +print(f"P-cores: {len(pCores)} threads @ max {pMax//1000} MHz nominal") +for name in ("idle", "spinAlone", "spinPlusCpuDetect", "spinPlusGpuDetect"): + rows = [s for s in samples if s["phase"] == name] + spinF = statistics.median(r["spinCore"] for r in rows) + allP = statistics.median(f for r in rows for f in r["pCores"]) + gpuF = statistics.median(r["gpu"] for r in rows) if rows[0]["gpu"] is not None else 0 + w = [r["watts"] for r in rows if r["watts"]] + watts = statistics.median(w) if w else 0 + dt = detectTimes.get(name) + extra = f", detect {dt:.0f} ms" if dt else "" + print(f"{name:20s} spinCore {spinF:4.0f} MHz | P-median {allP:4.0f} MHz | GPU {gpuF:4.0f} MHz | pkg {watts:5.1f} W{extra}") diff --git a/ocl_harness/tier_burn.py b/ocl_harness/tier_burn.py new file mode 100644 index 00000000..fd5b6475 --- /dev/null +++ b/ocl_harness/tier_burn.py @@ -0,0 +1,80 @@ +import glob +import os +import resource +import statistics +import threading +import time + +import cv2 + +os.environ["OCL_ICD_VENDORS"] = "/run/opengl-driver/etc/OpenCL/vendors" +from dt_apriltags import Detector + +img = cv2.imread("/tmp/probeframe.png", cv2.IMREAD_GRAYSCALE) +det = Detector(families="tagStandard52h13", nthreads=4, quad_decimate=1.0, + searchpath=["/var/lib/vide-gputest"]) + +tempPaths = glob.glob("/sys/class/thermal/thermal_zone*/temp") +freqPath = "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" +trace = [] +phase = ["startup"] +stop = [False] + + +def sampler(): + while not stop[0]: + time.sleep(5) + trace.append((phase[0], + max(int(open(p).read()) for p in tempPaths) // 1000, + int(open(freqPath).read()) // 1000)) + + +def drmBusyNs(): + total = 0 + for f in glob.glob("/proc/self/fdinfo/*"): + try: + for line in open(f): + if line.startswith("drm-engine-compute"): + total = max(total, int(line.split()[1])) + except OSError: + pass + return total + + +threading.Thread(target=sampler, daemon=True).start() + +PHASES = [("fullGpu", {"APRILTAG_OPENCL": "1", "APRILTAG_OPENCL_FIT": "1"}, 60), + ("midway", {"APRILTAG_OPENCL": "1"}, 60), + ("stockCpu", {}, 300)] + +for name, env, seconds in PHASES: + for k in ("APRILTAG_OPENCL", "APRILTAG_OPENCL_FIT"): + os.environ.pop(k, None) + os.environ.update(env) + det.detect(img) + det.detect(img) + phase[0] = name + r0 = resource.getrusage(resource.RUSAGE_SELF) + c0 = r0.ru_utime + r0.ru_stime + g0 = drmBusyNs() + times = [] + t0 = time.time() + while time.time() - t0 < seconds: + s = time.perf_counter() + n = len(det.detect(img)) + times.append((time.time(), (time.perf_counter() - s) * 1000)) + r1 = resource.getrusage(resource.RUSAGE_SELF) + coreMs = (r1.ru_utime + r1.ru_stime - c0) * 1000 / len(times) + gpuMs = (drmBusyNs() - g0) / 1e6 / len(times) + mid = t0 + seconds / 2 + early = statistics.median(ms for ts, ms in times if ts < t0 + 30) + late = statistics.median(ms for ts, ms in times if ts > max(mid, t0 + seconds - 30)) + overall = statistics.median(ms for _, ms in times) + phaseTrace = [(t, f) for p, t, f in trace if p == name] + tFirst, fFirst = phaseTrace[0] if phaseTrace else (0, 0) + tLast, fLast = phaseTrace[-1] if phaseTrace else (0, 0) + print(f"{name:9s} {len(times):4d} detects ({n} tags) | wall med {overall:5.0f} ms " + f"(early {early:5.0f} / late {late:5.0f}) | {coreMs:5.0f} core-ms | {gpuMs:5.1f} GPU-ms | " + f"temp {tFirst}->{tLast}C cpu0 {fFirst}->{fLast}MHz", flush=True) + +stop[0] = True diff --git a/ocl_harness/tier_probe.py b/ocl_harness/tier_probe.py new file mode 100644 index 00000000..e544444b --- /dev/null +++ b/ocl_harness/tier_probe.py @@ -0,0 +1,61 @@ +import os +import resource +import statistics +import sys +import time + +import cv2 +import numpy as np + +os.environ["OCL_ICD_VENDORS"] = "/run/opengl-driver/etc/OpenCL/vendors" + +from dt_apriltags import Detector + +image = cv2.imread(sys.argv[1], cv2.IMREAD_GRAYSCALE) +print(f"frame: {image.shape[1]}x{image.shape[0]}") + + +def make(searchpath): + kwargs = dict(families="tagStandard52h13", nthreads=4, quad_decimate=1.0, + quad_sigma=0.0, refine_edges=1, decode_sharpening=0.0) + if searchpath: + kwargs["searchpath"] = searchpath + try: + return Detector(min_cluster_pixels=100, **kwargs) + except TypeError: + return Detector(**kwargs) + + +stock = make(None) +gpu = make(["/tmp"]) + +tiers = [ + ("stock", stock, {}), + ("gpu-frontend", gpu, {"APRILTAG_OPENCL": "1"}), + ("gpu-fit", gpu, {"APRILTAG_OPENCL": "1", "APRILTAG_OPENCL_FIT": "1"}), +] + +results = {} +for name, det, env in tiers: + for k in ("APRILTAG_OPENCL", "APRILTAG_OPENCL_FIT"): + os.environ.pop(k, None) + os.environ.update(env) + det.detect(image) + det.detect(image) + times = [] + r0 = resource.getrusage(resource.RUSAGE_SELF) + c0 = r0.ru_utime + r0.ru_stime + for _ in range(5): + t0 = time.perf_counter() + res = det.detect(image) + times.append((time.perf_counter() - t0) * 1000) + r1 = resource.getrusage(resource.RUSAGE_SELF) + core = (r1.ru_utime + r1.ru_stime - c0) * 1000 / 5 + results[name] = sorted(res, key=lambda d: d.tag_id) + print(f"{name:14s} {len(res)} tags, median {statistics.median(times):6.0f} ms wall, {core:5.0f} core-ms") + +ref = results["stock"] +for name in ("gpu-frontend", "gpu-fit"): + r = results[name] + same = len(ref) == len(r) and all(a.tag_id == b.tag_id and np.array_equal(a.corners, b.corners) for a, b in zip(ref, r)) + print(f"parity {name}: identical={same}") From 4c669ae0464ca5f04dfceba264b435fe4f8659fc Mon Sep 17 00:00:00 2001 From: James McVay Date: Fri, 12 Jun 2026 00:01:48 +0200 Subject: [PATCH 17/18] Record the fleet campaign results and findings --- ocl_harness/FLEET_NOTES.md | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 ocl_harness/FLEET_NOTES.md diff --git a/ocl_harness/FLEET_NOTES.md b/ocl_harness/FLEET_NOTES.md new file mode 100644 index 00000000..a62262ed --- /dev/null +++ b/ocl_harness/FLEET_NOTES.md @@ -0,0 +1,51 @@ +# Fleet campaign notes (2026-06-11) + +Live validation and benchmarking across four robots: w3cj (NUC15, +225H, Arc 140T), w3y / w3bd / w3u (NUC14, 125H, Xe-LPG). Methods and +scripts in this directory; raw frames preserved off-robot +(~/Documents/apriltag-realframes, ~/Documents/apriltag-bursts). + +## Tier results (quiet machines, real full-res robot frames) + +per-frame, prod settings (52h13, 4 threads, decimate 1.0), bit-exact +parity verified in every cell: + +NUC15 / 225H (broken balance_performance EPP, the fleet default): + stock 88-108 ms / 238-260 core-ms; midway 41-46 / 90-112; fit 34-35 / 23-24 +NUC15 / 225H (EPP=performance, sustained 60-300 s, tier_burn): + stock 94 ms / 304 core-ms, 105 C, clock 4850->2199 (throttled, wall ~flat); + midway 60 / 167, 92 C; fit 50 / 42, 68 C +NUC14 / 125H (healthy units w3y/w3bd): + stock 65-112 / 206-364; midway 35-58 / 70-168; fit 32-50 / 26-40 + +In-service (vide, dark-evening conditions): stock ~1.9-3.5 cores -> +fit tier 0.48-1.35 cores. Combo with the spectacular-vio optimisation +deploy on w3y: system 3.95 -> 2.58 cores. + +## Findings + +- "Dense-frame cliff" RETRACTED: the slow unit (w3u) runs ALL frames + ~10x slow on GPU (cross-tested both directions); content exonerated + (boundary-record and component-structure analysis showed the "dense" + frame is unexceptional). w3u GPU defect: full clocks/topology/ + bandwidth reported healthy, survives reboot. Hardware case. + => rollout needs a startup self-bench gate, not a content breaker. +- GPU->CPU power coupling: NOT material on fleet hardware. Cool quiet + boosted 225H: busy core 4900 MHz alone, 4700 beside CPU detect, 4688 + beside GPU detect (-0.3% marginal). Package 38.5 W peak vs 64 W PL2. + GPU detect draws LESS package power than CPU detect (32 vs 38.5 W) + and runs ~35 C cooler sustained (68 vs 105 C). +- FLEET BUG (ticket-worthy): NUC15/225H + kernel 6.12 intel_pstate + balance_performance clamps busy P-cores to ~1.7 GHz (below base; + hwmax misreported as 6.2 GHz). EPP=performance restores spec 4.9 GHz + turbo. NUC14 unaffected. All NUC15 CPU benchmarks under the default + EPP are ~2x pessimistic. +- Cameras deliver 7-19.5 fps (exposure-limited), not nominal 25; dark + scenes throttle vide's loop via frame-freshness rejection. +- vide --mock is broken upstream (FilesystemCameraReader lacks the + auto-exposure interface the camera-switch path calls). +- Parallel work: BuildMonumental/apriltag branch nuc15-hardware + (Bouke, same day) — independent bit-exact GPU port, run-based + frontend, CPU-fit hybrid default, detect_prepare overlap API. + Complementary: their CPU stages + overlap API, this repo's fit + chain + drop-in deployment + fleet validation. From ae7b33548ba98cfb23153ea69a7c3d6660e4596b Mon Sep 17 00:00:00 2001 From: James McVay Date: Fri, 12 Jun 2026 00:24:52 +0200 Subject: [PATCH 18/18] Add the definitive controlled cross-generation comparison --- ocl_harness/FLEET_NOTES.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ocl_harness/FLEET_NOTES.md b/ocl_harness/FLEET_NOTES.md index a62262ed..6be8074c 100644 --- a/ocl_harness/FLEET_NOTES.md +++ b/ocl_harness/FLEET_NOTES.md @@ -49,3 +49,30 @@ deploy on w3y: system 3.95 -> 2.58 cores. frontend, CPU-fit hybrid default, detect_prepare overlap API. Complementary: their CPU stages + overlap API, this repo's fit chain + drop-in deployment + fleet validation. + +## Controlled comparison (2026-06-12, definitive) + +Same 3 real scenes (Pisa 25/39/21 camera frames) run on both machines, +services stopped, 60 s cold-start, reverse order (full -> midway -> +stock), 20 s/cell, stock cells use the closure's stock libapriltag. +Ranges span the 3 scenes. EPP state per column noted. + + W3.2/225H AS-SHIPPED W3.2/225H EPP-FIXED W3.1.W3/125H + (balance_performance) (performance) (performance) +Original vide FPS 6-14 14-20 13-19 + CPU/f 221-539 166-227 178-251 +Midway FPS 19-20 22-36 21-33 +(GPU frontend) CPU/f 104-125 (-53..-77%) 55-106 (-53..-67%) 60-109 (-57..-66%) + GPU/f 13-17 10-15 10-16 +Full GPU(fit) FPS 20-25 28-34 29-36 + CPU/f 51-62 (-77..-89%) 21-25 (-87..-89%) 21-25 (-88..-90%) + GPU/f 23-30 21-27 21-27 + +Findings: +- Full GPU is 28-36 FPS / 21-25 core-ms on BOTH generations once EPP is + fixed; the generations converge to within ~5% per GPU cell. +- The W3.2 EPP bug makes the newest robots the SLOWEST as-shipped (6-14 + vs 125H's 13-19 FPS on stock) -- a CPU-governor issue independent of + this work. Full GPU rescues it regardless (20-25 FPS, -77..-89% CPU). +- Last night's burn 50 ms fit number was a context artifact; the real + fit-tier figure is 28-36 ms, matching the daytime quiet runs.