From 051fb7c6b18d645a87217fcd0c3aa8051047511f Mon Sep 17 00:00:00 2001 From: yash-pouranik Date: Thu, 9 Apr 2026 15:13:39 +0530 Subject: [PATCH 1/2] feat: implement relational population and harden mail service quota --- .../__tests__/data.controller.read.test.js | 41 ++++++++++++++++++- .../src/__tests__/mail.controller.test.js | 15 +++---- .../src/controllers/data.controller.js | 14 ++++++- docs/api-reference.md | 12 ++++++ docs/database.md | 28 ++++++++++++- packages/common/src/utils/injectModel.js | 19 +++++---- packages/common/src/utils/queryEngine.js | 11 +++++ 7 files changed, 120 insertions(+), 20 deletions(-) diff --git a/apps/public-api/src/__tests__/data.controller.read.test.js b/apps/public-api/src/__tests__/data.controller.read.test.js index eafd9d354..8efdf116d 100644 --- a/apps/public-api/src/__tests__/data.controller.read.test.js +++ b/apps/public-api/src/__tests__/data.controller.read.test.js @@ -4,6 +4,7 @@ const mockFind = jest.fn(); const mockAnd = jest.fn(); const mockFindOne = jest.fn(); const mockQueryLean = jest.fn().mockResolvedValue([]); +const mockPopulate = jest.fn(); // mockAnd returns an object with .lean() so features.query.lean() works after .and() mockAnd.mockReturnValue({ lean: mockQueryLean }); @@ -13,6 +14,7 @@ const mockQueryEngine = jest.fn((query) => { query, filter() { return engine; }, sort() { return engine; }, + populate() { return engine; }, paginate() { return engine; }, }; return engine; @@ -25,11 +27,24 @@ jest.mock('@urbackend/common', () => ({ getCompiledModel: jest.fn(() => ({ find: (...args) => { mockFind(...args); - return { and: mockAnd, lean: mockQueryLean }; + return { + and: mockAnd, + populate: (...pArgs) => { + mockPopulate(...pArgs); + return { lean: mockQueryLean }; + }, + lean: mockQueryLean + }; }, findOne: (...args) => { mockFindOne(...args); - return { lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) }; + return { + populate: (...pArgs) => { + mockPopulate(...pArgs); + return { lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) }; + }, + lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) + }; }, })), QueryEngine: mockQueryEngine, @@ -101,4 +116,26 @@ describe('data.controller read RLS filters', () => { }); expect(res.json).toHaveBeenCalled(); }); + + test('getAllData passes populate param to QueryEngine', async () => { + const req = makeReq({ query: { populate: 'author,comments' } }); + const res = makeRes(); + + // The real QueryEngine (not mocked) would call .populate() on the query + // But since we mock QueryEngine, we just check if populate() was part of the chain + // In this specific test file, mockQueryEngine is what's used. + await getAllData(req, res); + + expect(mockQueryEngine).toHaveBeenCalled(); + }); + + test('getSingleDoc calls populate on the query', async () => { + const req = makeReq({ query: { populate: 'author' } }); + const res = makeRes(); + + await getSingleDoc(req, res); + + expect(mockPopulate).toHaveBeenCalledWith('author'); + expect(res.json).toHaveBeenCalled(); + }); }); diff --git a/apps/public-api/src/__tests__/mail.controller.test.js b/apps/public-api/src/__tests__/mail.controller.test.js index a85126066..aa2c619d2 100644 --- a/apps/public-api/src/__tests__/mail.controller.test.js +++ b/apps/public-api/src/__tests__/mail.controller.test.js @@ -20,6 +20,7 @@ jest.mock('@urbackend/common', () => { incr: jest.fn(), expire: jest.fn(), decr: jest.fn(), + eval: jest.fn(), }; return { @@ -66,14 +67,13 @@ describe('mail.controller', () => { const req = makeReq(); const res = makeRes(); - mockProjectConfig({ _id: 'proj_1', resendApiKey: {} }); + mockProjectConfig({ _id: 'proj_1', resendApiKey: { encrypted: '...' } }); decrypt.mockReturnValue('byok-key'); - redis.incr.mockResolvedValue(1); + redis.eval.mockResolvedValue(1); await mailController.sendMail(req, res); - expect(redis.incr).toHaveBeenCalledTimes(1); - expect(redis.expire).toHaveBeenCalledTimes(1); + expect(redis.eval).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ success: true, @@ -87,7 +87,7 @@ describe('mail.controller', () => { mockProjectConfig({ _id: 'proj_1', resendApiKey: null }); decrypt.mockReturnValue(null); - redis.incr.mockResolvedValue(2); + redis.eval.mockResolvedValue(2); await mailController.sendMail(req, res); @@ -103,14 +103,15 @@ describe('mail.controller', () => { mockProjectConfig({ _id: 'proj_1', resendApiKey: null }); decrypt.mockReturnValue(null); - redis.incr.mockResolvedValue(101); + redis.eval.mockResolvedValue(101); await mailController.sendMail(req, res); expect(redis.decr).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(429); expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ - error: 'Monthly mail limit exceeded.', + success: false, + message: 'Monthly mail limit exceeded.', })); }); }); diff --git a/apps/public-api/src/controllers/data.controller.js b/apps/public-api/src/controllers/data.controller.js index 25d2d0171..248ebec0b 100644 --- a/apps/public-api/src/controllers/data.controller.js +++ b/apps/public-api/src/controllers/data.controller.js @@ -122,6 +122,7 @@ module.exports.getAllData = async (req, res) => { features .sort() + .populate() .paginate(); const data = await features.query.lean(); @@ -158,7 +159,18 @@ module.exports.getSingleDoc = async (req, res) => { ); const baseFilter = req.rlsFilter && typeof req.rlsFilter === 'object' ? req.rlsFilter : {}; - const doc = await Model.findOne({ $and: [{ _id: id }, baseFilter] }).lean(); + let query = Model.findOne({ $and: [{ _id: id }, baseFilter] }); + + // Handle population for single doc + const populateParam = req.query.populate || req.query.expand; + if (populateParam) { + const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); + fields.forEach(f => { + query = query.populate(f); + }); + } + + const doc = await query.lean(); if (!doc) return res.status(404).json({ error: "Document not found." }); res.json(doc); diff --git a/docs/api-reference.md b/docs/api-reference.md index 58813da8e..9968d1cc1 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,6 +24,18 @@ | **Storage** | `POST` | `/api/storage/upload` | Upload a file | | **Storage** | `DELETE` | `/api/storage/file` | Delete a file by path | +## Data Query Parameters + +| Parameter | Example | Description | +| :--- | :--- | :--- | +| `populate` | `?populate=author` | Expand Reference fields into objects | +| `expand` | `?expand=author` | Alias for `populate` | +| `sort` | `?sort=field:asc` | Sort order (`asc`, `desc`, `1`, `-1`) | +| `limit` | `?limit=50` | Limit results (default 100) | +| `page` | `?page=1` | Pagination page number | +| `field_gt` | `?price_gt=100` | Greater than filter | +| `field_lt` | `?price_lt=50` | Less than filter | + ## Common Headers | Header | Required | Purpose | diff --git a/docs/database.md b/docs/database.md index 2a76e9c50..c90563fb2 100644 --- a/docs/database.md +++ b/docs/database.md @@ -56,12 +56,36 @@ curl "https://api.ub.bitbros.in/api/data/posts" ^ ### Fetch Single Document **Endpoint**: `GET /api/data/:collectionName/:id` -```bash curl "https://api.ub.bitbros.in/api/data/posts/64fd1234abcd5678ef901234" ^ -H "x-api-key: pk_live_xxx" ``` -## 3. Update a Document +## 2.1 Query Parameters + +You can refine your `GET` requests using the following query parameters: + +| Parameter | Type | Example | Description | +| :--- | :--- | :--- | :--- | +| `populate` | `String` | `?populate=author,category` | Expand `Ref` fields into full objects. | +| `expand` | `String` | `?expand=author` | Alias for `populate`. | +| `sort` | `String` | `?sort=createdAt:desc` | Sort results by field and order. | +| `limit` | `Number` | `?limit=25` | Number of documents to return (max 100). | +| `page` | `Number` | `?page=2` | Pagination offset. | + +### Relational Data (Population) 🔗 + +If you have fields defined as `Ref` (singular) or `Array of Ref` in your schema, urBackend can automatically join those documents for you. + +By default, urBackend returns raw IDs for references. Use `?populate=` to get the full nested object. + +```bash +# Get posts with full author and category objects +curl "https://api.ub.bitbros.in/api/data/posts?populate=author,category" ^ + -H "x-api-key: pk_live_xxx" +``` + +> [!TIP] +> This solves the N+1 query problem, allowing you to fetch complex relational data in a single request. **Endpoint**: `PUT /api/data/:collectionName/:id` diff --git a/packages/common/src/utils/injectModel.js b/packages/common/src/utils/injectModel.js index da93f7310..0c350bc75 100644 --- a/packages/common/src/utils/injectModel.js +++ b/packages/common/src/utils/injectModel.js @@ -10,7 +10,7 @@ const typeMapping = { }; // Recursive field definition builder -function buildFieldDef(field) { +function buildFieldDef(field, projectId, isExternal) { // Object type — nested sub-schema if (field.type === "Object" && field.fields && field.fields.length > 0) { const subSchema = {}; @@ -18,7 +18,7 @@ function buildFieldDef(field) { const normalizedKey = normalizeKey(f.key); if (!normalizedKey) return; - subSchema[normalizedKey] = buildFieldDef(f); + subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal); }); return { type: subSchema, required: !!field.required }; } @@ -42,14 +42,15 @@ function buildFieldDef(field) { const normalizedKey = normalizeKey(f.key); if (!normalizedKey) return; - subSchema[normalizedKey] = buildFieldDef(f); + subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal); }); return { type: [subSchema], required: !!field.required }; } // Array of Ref - if (field.items.type === "Ref") { + if (field.items && field.items.type === "Ref") { + const targetRef = isExternal ? field.items.ref : `${projectId}_${field.items.ref}`; return { - type: [{ type: mongoose.Schema.Types.ObjectId }], + type: [{ type: mongoose.Schema.Types.ObjectId, ref: targetRef }], required: !!field.required, }; } @@ -61,8 +62,10 @@ function buildFieldDef(field) { // Ref type — stores ObjectId if (field.type === "Ref") { + const targetRef = isExternal ? field.ref : `${projectId}_${field.ref}`; return { type: mongoose.Schema.Types.ObjectId, + ref: targetRef, required: !!field.required, }; } @@ -79,14 +82,14 @@ function normalizeKey(key) { .trim(); } -function buildMongooseSchema(fieldsArray = []) { +function buildMongooseSchema(fieldsArray = [], projectId, isExternal) { const schemaDef = {}; fieldsArray.forEach((field) => { const normalizedKey = normalizeKey(field.key); if (!normalizedKey) return; - schemaDef[normalizedKey] = buildFieldDef(field); + schemaDef[normalizedKey] = buildFieldDef(field, projectId, isExternal); }); return new mongoose.Schema(schemaDef, { @@ -123,7 +126,7 @@ function getCompiledModel(connection, collectionData, projectId, isExternal) { } // Build schema + compile - const schema = buildMongooseSchema(collectionData.model); + const schema = buildMongooseSchema(collectionData.model, projectId, isExternal); const model = connection.model(collectionName, schema); // Cache it diff --git a/packages/common/src/utils/queryEngine.js b/packages/common/src/utils/queryEngine.js index 6a65d94bd..4b75277fe 100644 --- a/packages/common/src/utils/queryEngine.js +++ b/packages/common/src/utils/queryEngine.js @@ -60,6 +60,17 @@ class QueryEngine { return this; } + + populate() { + const populateParam = this.queryString.populate || this.queryString.expand; + if (populateParam) { + const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); + fields.forEach(f => { + this.query = this.query.populate(f); + }); + } + return this; + } } module.exports = QueryEngine; From c2eb9817ede7bc38348e9012396343b6dcd7b853 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:29:37 +0000 Subject: [PATCH 2/2] fix: address PR review feedback on populate robustness, docs, and tests Agent-Logs-Url: https://github.com/geturbackend/urBackend/sessions/2bf89d6d-c2ca-425b-ae5d-71ce054d8edb Co-authored-by: yash-pouranik <172860064+yash-pouranik@users.noreply.github.com> --- .../__tests__/data.controller.read.test.js | 45 +++++++++++++++---- .../src/controllers/data.controller.js | 5 ++- docs/database.md | 7 ++- packages/common/src/utils/queryEngine.js | 18 ++++---- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/apps/public-api/src/__tests__/data.controller.read.test.js b/apps/public-api/src/__tests__/data.controller.read.test.js index 8efdf116d..77c3627f6 100644 --- a/apps/public-api/src/__tests__/data.controller.read.test.js +++ b/apps/public-api/src/__tests__/data.controller.read.test.js @@ -5,6 +5,7 @@ const mockAnd = jest.fn(); const mockFindOne = jest.fn(); const mockQueryLean = jest.fn().mockResolvedValue([]); const mockPopulate = jest.fn(); +const mockEnginePopulate = jest.fn(); // mockAnd returns an object with .lean() so features.query.lean() works after .and() mockAnd.mockReturnValue({ lean: mockQueryLean }); @@ -14,7 +15,7 @@ const mockQueryEngine = jest.fn((query) => { query, filter() { return engine; }, sort() { return engine; }, - populate() { return engine; }, + populate() { mockEnginePopulate(...arguments); return engine; }, paginate() { return engine; }, }; return engine; @@ -38,13 +39,14 @@ jest.mock('@urbackend/common', () => ({ }, findOne: (...args) => { mockFindOne(...args); - return { + const chainable = { populate: (...pArgs) => { mockPopulate(...pArgs); - return { lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) }; + return chainable; }, - lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) + lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }), }; + return chainable; }, })), QueryEngine: mockQueryEngine, @@ -89,6 +91,7 @@ function makeRes() { describe('data.controller read RLS filters', () => { beforeEach(() => { jest.clearAllMocks(); + mockEnginePopulate.mockClear(); }); test('getAllData applies rlsFilter to find()', async () => { @@ -117,16 +120,28 @@ describe('data.controller read RLS filters', () => { expect(res.json).toHaveBeenCalled(); }); - test('getAllData passes populate param to QueryEngine', async () => { + test('getAllData calls engine.populate() when populate param is provided', async () => { const req = makeReq({ query: { populate: 'author,comments' } }); const res = makeRes(); - // The real QueryEngine (not mocked) would call .populate() on the query - // But since we mock QueryEngine, we just check if populate() was part of the chain - // In this specific test file, mockQueryEngine is what's used. await getAllData(req, res); - + expect(mockQueryEngine).toHaveBeenCalled(); + expect(mockEnginePopulate).toHaveBeenCalled(); + }); + + test('getAllData does not forward populate/expand to Mongo filter', async () => { + const req = makeReq({ query: { populate: 'author', expand: 'category', title: 'hello' } }); + const res = makeRes(); + + await getAllData(req, res); + + // mockFind is called with the raw Mongoose query (no args for Model.find()) + // The real filter exclusion is tested via the QueryEngine directly, + // but we confirm find() was invoked and the request did not error out. + expect(mockFind).toHaveBeenCalledWith(); + expect(res.json).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalledWith(500); }); test('getSingleDoc calls populate on the query', async () => { @@ -138,4 +153,16 @@ describe('data.controller read RLS filters', () => { expect(mockPopulate).toHaveBeenCalledWith('author'); expect(res.json).toHaveBeenCalled(); }); + + test('getSingleDoc handles array-format populate param without crashing', async () => { + const req = makeReq({ query: { populate: ['author', 'category'] } }); + const res = makeRes(); + + await getSingleDoc(req, res); + + expect(mockPopulate).toHaveBeenCalledWith('author'); + expect(mockPopulate).toHaveBeenCalledWith('category'); + expect(res.json).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalledWith(500); + }); }); diff --git a/apps/public-api/src/controllers/data.controller.js b/apps/public-api/src/controllers/data.controller.js index 248ebec0b..4312fdc01 100644 --- a/apps/public-api/src/controllers/data.controller.js +++ b/apps/public-api/src/controllers/data.controller.js @@ -162,8 +162,9 @@ module.exports.getSingleDoc = async (req, res) => { let query = Model.findOne({ $and: [{ _id: id }, baseFilter] }); // Handle population for single doc - const populateParam = req.query.populate || req.query.expand; - if (populateParam) { + const rawPopulateParam = req.query.populate || req.query.expand; + if (rawPopulateParam) { + const populateParam = Array.isArray(rawPopulateParam) ? rawPopulateParam.join(',') : String(rawPopulateParam); const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); fields.forEach(f => { query = query.populate(f); diff --git a/docs/database.md b/docs/database.md index c90563fb2..bfe5c5ee8 100644 --- a/docs/database.md +++ b/docs/database.md @@ -56,6 +56,7 @@ curl "https://api.ub.bitbros.in/api/data/posts" ^ ### Fetch Single Document **Endpoint**: `GET /api/data/:collectionName/:id` +```bash curl "https://api.ub.bitbros.in/api/data/posts/64fd1234abcd5678ef901234" ^ -H "x-api-key: pk_live_xxx" ``` @@ -69,8 +70,8 @@ You can refine your `GET` requests using the following query parameters: | `populate` | `String` | `?populate=author,category` | Expand `Ref` fields into full objects. | | `expand` | `String` | `?expand=author` | Alias for `populate`. | | `sort` | `String` | `?sort=createdAt:desc` | Sort results by field and order. | -| `limit` | `Number` | `?limit=25` | Number of documents to return (max 100). | -| `page` | `Number` | `?page=2` | Pagination offset. | +| `limit` | `Number` | `?limit=25` | Number of documents to return (default: 100, max: 100). | +| `page` | `Number` | `?page=2` | 1-indexed page number for pagination. | ### Relational Data (Population) 🔗 @@ -87,6 +88,8 @@ curl "https://api.ub.bitbros.in/api/data/posts?populate=author,category" ^ > [!TIP] > This solves the N+1 query problem, allowing you to fetch complex relational data in a single request. +## 3. Update a Document + **Endpoint**: `PUT /api/data/:collectionName/:id` urBackend uses `$set` logic, meaning you only need to send the fields you want to change. diff --git a/packages/common/src/utils/queryEngine.js b/packages/common/src/utils/queryEngine.js index 4b75277fe..e06588164 100644 --- a/packages/common/src/utils/queryEngine.js +++ b/packages/common/src/utils/queryEngine.js @@ -6,7 +6,7 @@ class QueryEngine { filter() { const queryObj = { ...this.queryString }; - const excludedFields = ['page', 'sort', 'limit', 'fields']; + const excludedFields = ['page', 'sort', 'limit', 'fields', 'populate', 'expand']; excludedFields.forEach(el => delete queryObj[el]); const mongoQuery = {}; @@ -53,7 +53,7 @@ class QueryEngine { paginate() { const page = parseInt(this.queryString.page, 10) || 1; - const limit = parseInt(this.queryString.limit, 10) || 100; + const limit = Math.min(parseInt(this.queryString.limit, 10) || 100, 100); const skip = (page - 1) * limit; this.query = this.query.skip(skip).limit(limit); @@ -62,13 +62,13 @@ class QueryEngine { } populate() { - const populateParam = this.queryString.populate || this.queryString.expand; - if (populateParam) { - const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); - fields.forEach(f => { - this.query = this.query.populate(f); - }); - } + const rawParam = this.queryString.populate || this.queryString.expand; + if (!rawParam) return this; + const populateParam = Array.isArray(rawParam) ? rawParam.join(',') : String(rawParam); + const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); + fields.forEach(f => { + this.query = this.query.populate(f); + }); return this; } }