Skip to content

Commit 104dc21

Browse files
committed
fix(store): use 64-bit file offsets in the SQLite page writer
sqlite_writer.c computed page offsets as (long)(page - 1) * 65536 and passed them to fseek()/ftell(). long is 32 bits on Windows, so once a database grows past 2 GiB the offset goes negative and fseek() fails; the result is ignored, so pages land wherever the stream happens to be. Past 4 GiB the offset wraps back to small positive values and the writer overwrites the start of the file. The staging database stops at exactly 4 GiB, is corrupt, and the run ends in persist_failed. Route every seek/tell in the writer through 64-bit wrappers (_fseeki64/_ftelli64 on Windows, fseeko/ftello elsewhere) and compute offsets with a new cbm_writer_page_offset() helper that widens before multiplying. Fixes DeusData#2399 Signed-off-by: lixsp11 <30531760+Lixsp11@users.noreply.github.com>
1 parent 56700bd commit 104dc21

3 files changed

Lines changed: 40 additions & 14 deletions

File tree

‎internal/cbm/sqlite_writer.c‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,22 @@
3535
#include <process.h>
3636
#include <windows.h>
3737
#define cbm_writer_getpid _getpid
38+
/* long is 32 bits on Windows, so fseek/ftell cannot address past 2 GiB there. */
39+
#define cbm_writer_seek(fp, off, whence) _fseeki64((fp), (long long)(off), (whence))
40+
#define cbm_writer_tell(fp) ((int64_t)_ftelli64(fp))
3841
#else
3942
#include <unistd.h>
4043
#define cbm_writer_getpid getpid
44+
#define cbm_writer_seek(fp, off, whence) fseeko((fp), (off_t)(off), (whence))
45+
#define cbm_writer_tell(fp) ((int64_t)ftello(fp))
4146
#endif
4247

4348
#define CBM_PAGE_SIZE 65536
4449

50+
int64_t cbm_writer_page_offset(uint32_t page_num) {
51+
return (int64_t)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;
52+
}
53+
4554
/* SQLite reserves the page containing the 1 GiB file offset (the "pending byte"
4655
* used for file locking on Windows). This page MUST be skipped during allocation
4756
* otherwise integrity_check reports "2nd reference to page N" because it marks
@@ -601,8 +610,8 @@ static void pb_flush_leaf(PageBuilder *pb) {
601610
// Write page to file. Skip the pending byte page (SQLite reserved).
602611
pb->next_page = cbm_skip_pending_byte(pb->next_page);
603612
uint32_t page_num = pb->next_page;
604-
long offset = (long)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;
605-
(void)fseek(pb->fp, offset, SEEK_SET);
613+
int64_t offset = cbm_writer_page_offset(page_num);
614+
(void)cbm_writer_seek(pb->fp, offset, SEEK_SET);
606615
(void)fwrite(pb->page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);
607616

608617
// Record this leaf for interior page building
@@ -707,7 +716,7 @@ static int write_interior_page(PageBuilder *pb, uint8_t *page, int cell_count, i
707716
page[HDR_FRAGBYTES_OFF] = 0;
708717
put_u32(page + HDR_RIGHTCHILD_OFF, right_child_page);
709718

710-
(void)fseek(pb->fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
719+
(void)cbm_writer_seek(pb->fp, cbm_writer_page_offset(pnum), SEEK_SET);
711720
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);
712721

713722
if (parent_count >= *parent_cap) {
@@ -970,7 +979,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
970979
int data_len) {
971980
int per_page = CBM_PAGE_SIZE - BTREE_PTR_SIZE;
972981
uint32_t first_page = 0;
973-
long prev_next_ptr_offset = -SKIP_ONE;
982+
int64_t prev_next_ptr_offset = -SKIP_ONE;
974983

975984
int offset = 0;
976985
while (offset < data_len) {
@@ -983,7 +992,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
983992
if (prev_next_ptr_offset >= 0) {
984993
uint8_t ptr[BTREE_PTR_SIZE];
985994
put_u32(ptr, pnum);
986-
(void)fseek(fp, prev_next_ptr_offset, SEEK_SET);
995+
(void)cbm_writer_seek(fp, prev_next_ptr_offset, SEEK_SET);
987996
(void)fwrite(ptr, SKIP_ONE, BTREE_PTR_SIZE, fp);
988997
}
989998

@@ -997,9 +1006,9 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
9971006
put_u32(page, 0); // next-page pointer — 0 for now, backpatched on next iteration
9981007
memcpy(page + BTREE_PTR_SIZE, data + offset, chunk);
9991008

1000-
long page_offset = (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE;
1009+
int64_t page_offset = cbm_writer_page_offset(pnum);
10011010
prev_next_ptr_offset = page_offset;
1002-
(void)fseek(fp, page_offset, SEEK_SET);
1011+
(void)cbm_writer_seek(fp, page_offset, SEEK_SET);
10031012
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
10041013

10051014
offset += chunk;
@@ -1276,7 +1285,7 @@ static uint32_t write_table_btree(FILE *fp, uint32_t *next_page, const uint8_t *
12761285
put_u16(page + hdr + HDR_CELLCOUNT_OFF, 0); // 0 cells
12771286
put_u16(page + hdr + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE); // content at end of page
12781287
page[hdr + HDR_FRAGBYTES_OFF] = 0; // 0 fragmented bytes
1279-
(void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
1288+
(void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);
12801289
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
12811290
return pnum;
12821291
}
@@ -1326,7 +1335,7 @@ static uint32_t write_empty_index_leaf(FILE *fp, uint32_t *next_page) {
13261335
put_u16(page + HDR_CELLCOUNT_OFF, 0);
13271336
put_u16(page + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE);
13281337
page[HDR_FRAGBYTES_OFF] = 0;
1329-
(void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
1338+
(void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);
13301339
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
13311340
return pnum;
13321341
}
@@ -2020,7 +2029,7 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u
20202029

20212030
write_sqlite_file_header(page1, next_page - SKIP_ONE);
20222031

2023-
(void)fseek(fp, 0, SEEK_SET);
2032+
(void)cbm_writer_seek(fp, 0, SEEK_SET);
20242033
(void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp);
20252034

20262035
for (int i = 0; i < master_count; i++) {
@@ -2034,12 +2043,12 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u
20342043

20352044
/* Pad file to exact page boundary. */
20362045
static void pad_file_to_page_boundary(FILE *fp, uint32_t next_page) {
2037-
(void)fseek(fp, 0, SEEK_END);
2038-
long file_size = ftell(fp);
2039-
long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE;
2046+
(void)cbm_writer_seek(fp, 0, SEEK_END);
2047+
int64_t file_size = cbm_writer_tell(fp);
2048+
int64_t expected_size = cbm_writer_page_offset(next_page);
20402049
if (file_size < expected_size) {
20412050
uint8_t zero = 0;
2042-
(void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET);
2051+
(void)cbm_writer_seek(fp, expected_size - SKIP_ONE, SEEK_SET);
20432052
(void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp);
20442053
}
20452054
}

‎internal/cbm/sqlite_writer.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,4 +90,8 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo
9090
CBMDumpEdge *edges, int edge_count, CBMDumpVector *vectors,
9191
int vector_count, CBMDumpTokenVec *token_vecs, int token_vec_count);
9292

93+
// Byte offset of 1-based page `page_num` in the output file. 64-bit on every
94+
// platform, so databases larger than 2 GiB are addressed correctly on Windows.
95+
int64_t cbm_writer_page_offset(uint32_t page_num);
96+
9397
#endif // CBM_SQLITE_WRITER_H

‎tests/test_sqlite_writer.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -996,6 +996,18 @@ TEST(sw_publish_preserves_live_reader) {
996996

997997
/* ── Suite ─────────────────────────────────────────────────────── */
998998

999+
TEST(sw_page_offsets_do_not_wrap_past_4gib) {
1000+
/* A 32-bit long offset wraps negative at 2 GiB and back to 0 at 4 GiB: on
1001+
* Windows a staging DB stopped at exactly 4 GiB with its first pages
1002+
* overwritten, and publish failed with persist_failed (#2399). */
1003+
ASSERT_EQ(cbm_writer_page_offset(1), 0);
1004+
ASSERT_EQ(cbm_writer_page_offset(2), 65536);
1005+
ASSERT_EQ(cbm_writer_page_offset(32769), INT64_C(2147483648));
1006+
ASSERT_EQ(cbm_writer_page_offset(65537), INT64_C(4294967296));
1007+
ASSERT_EQ(cbm_writer_page_offset(UINT32_MAX), (int64_t)(UINT32_MAX - 1) * 65536);
1008+
PASS();
1009+
}
1010+
9991011
TEST(sw_open_truncated_path_names_its_reason) {
10001012
/* final_path is a fixed 4K buffer and make_writer_temp_path only fails by
10011013
* truncation. Neither sets an errno of its own, so without naming one the
@@ -1065,6 +1077,7 @@ SUITE(sqlite_writer) {
10651077
RUN_TEST(sw_publish_failure_preserves_destination_sidecars);
10661078
RUN_TEST(sw_publish_supports_non_ascii_path);
10671079
RUN_TEST(sw_publish_preserves_live_reader);
1080+
RUN_TEST(sw_page_offsets_do_not_wrap_past_4gib);
10681081
RUN_TEST(sw_open_truncated_path_names_its_reason);
10691082
RUN_TEST(sw_publish_failure_reports_the_rename_not_the_cleanup);
10701083
}

0 commit comments

Comments
 (0)