diff --git a/apps/public-api/src/__tests__/data.controller.read.test.js b/apps/public-api/src/__tests__/data.controller.read.test.js index eafd9d354..77c3627f6 100644 --- a/apps/public-api/src/__tests__/data.controller.read.test.js +++ b/apps/public-api/src/__tests__/data.controller.read.test.js @@ -4,6 +4,8 @@ const mockFind = jest.fn(); const mockAnd = jest.fn(); const mockFindOne = jest.fn(); const mockQueryLean = jest.fn().mockResolvedValue([]); +const mockPopulate = jest.fn(); +const mockEnginePopulate = jest.fn(); // mockAnd returns an object with .lean() so features.query.lean() works after .and() mockAnd.mockReturnValue({ lean: mockQueryLean }); @@ -13,6 +15,7 @@ const mockQueryEngine = jest.fn((query) => { query, filter() { return engine; }, sort() { return engine; }, + populate() { mockEnginePopulate(...arguments); return engine; }, paginate() { return engine; }, }; return engine; @@ -25,11 +28,25 @@ jest.mock('@urbackend/common', () => ({ getCompiledModel: jest.fn(() => ({ find: (...args) => { mockFind(...args); - return { and: mockAnd, lean: mockQueryLean }; + return { + and: mockAnd, + populate: (...pArgs) => { + mockPopulate(...pArgs); + return { lean: mockQueryLean }; + }, + lean: mockQueryLean + }; }, findOne: (...args) => { mockFindOne(...args); - return { lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) }; + const chainable = { + populate: (...pArgs) => { + mockPopulate(...pArgs); + return chainable; + }, + lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }), + }; + return chainable; }, })), QueryEngine: mockQueryEngine, @@ -74,6 +91,7 @@ function makeRes() { describe('data.controller read RLS filters', () => { beforeEach(() => { jest.clearAllMocks(); + mockEnginePopulate.mockClear(); }); test('getAllData applies rlsFilter to find()', async () => { @@ -101,4 +119,50 @@ describe('data.controller read RLS filters', () => { }); expect(res.json).toHaveBeenCalled(); }); + + test('getAllData calls engine.populate() when populate param is provided', async () => { + const req = makeReq({ query: { populate: 'author,comments' } }); + const res = makeRes(); + + await getAllData(req, res); + + expect(mockQueryEngine).toHaveBeenCalled(); + expect(mockEnginePopulate).toHaveBeenCalled(); + }); + + test('getAllData does not forward populate/expand to Mongo filter', async () => { + const req = makeReq({ query: { populate: 'author', expand: 'category', title: 'hello' } }); + const res = makeRes(); + + await getAllData(req, res); + + // mockFind is called with the raw Mongoose query (no args for Model.find()) + // The real filter exclusion is tested via the QueryEngine directly, + // but we confirm find() was invoked and the request did not error out. + expect(mockFind).toHaveBeenCalledWith(); + expect(res.json).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalledWith(500); + }); + + test('getSingleDoc calls populate on the query', async () => { + const req = makeReq({ query: { populate: 'author' } }); + const res = makeRes(); + + await getSingleDoc(req, res); + + expect(mockPopulate).toHaveBeenCalledWith('author'); + expect(res.json).toHaveBeenCalled(); + }); + + test('getSingleDoc handles array-format populate param without crashing', async () => { + const req = makeReq({ query: { populate: ['author', 'category'] } }); + const res = makeRes(); + + await getSingleDoc(req, res); + + expect(mockPopulate).toHaveBeenCalledWith('author'); + expect(mockPopulate).toHaveBeenCalledWith('category'); + expect(res.json).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalledWith(500); + }); }); diff --git a/apps/public-api/src/__tests__/mail.controller.test.js b/apps/public-api/src/__tests__/mail.controller.test.js index a85126066..aa2c619d2 100644 --- a/apps/public-api/src/__tests__/mail.controller.test.js +++ b/apps/public-api/src/__tests__/mail.controller.test.js @@ -20,6 +20,7 @@ jest.mock('@urbackend/common', () => { incr: jest.fn(), expire: jest.fn(), decr: jest.fn(), + eval: jest.fn(), }; return { @@ -66,14 +67,13 @@ describe('mail.controller', () => { const req = makeReq(); const res = makeRes(); - mockProjectConfig({ _id: 'proj_1', resendApiKey: {} }); + mockProjectConfig({ _id: 'proj_1', resendApiKey: { encrypted: '...' } }); decrypt.mockReturnValue('byok-key'); - redis.incr.mockResolvedValue(1); + redis.eval.mockResolvedValue(1); await mailController.sendMail(req, res); - expect(redis.incr).toHaveBeenCalledTimes(1); - expect(redis.expire).toHaveBeenCalledTimes(1); + expect(redis.eval).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ success: true, @@ -87,7 +87,7 @@ describe('mail.controller', () => { mockProjectConfig({ _id: 'proj_1', resendApiKey: null }); decrypt.mockReturnValue(null); - redis.incr.mockResolvedValue(2); + redis.eval.mockResolvedValue(2); await mailController.sendMail(req, res); @@ -103,14 +103,15 @@ describe('mail.controller', () => { mockProjectConfig({ _id: 'proj_1', resendApiKey: null }); decrypt.mockReturnValue(null); - redis.incr.mockResolvedValue(101); + redis.eval.mockResolvedValue(101); await mailController.sendMail(req, res); expect(redis.decr).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(429); expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ - error: 'Monthly mail limit exceeded.', + success: false, + message: 'Monthly mail limit exceeded.', })); }); }); diff --git a/apps/public-api/src/controllers/data.controller.js b/apps/public-api/src/controllers/data.controller.js index 25d2d0171..4312fdc01 100644 --- a/apps/public-api/src/controllers/data.controller.js +++ b/apps/public-api/src/controllers/data.controller.js @@ -122,6 +122,7 @@ module.exports.getAllData = async (req, res) => { features .sort() + .populate() .paginate(); const data = await features.query.lean(); @@ -158,7 +159,19 @@ module.exports.getSingleDoc = async (req, res) => { ); const baseFilter = req.rlsFilter && typeof req.rlsFilter === 'object' ? req.rlsFilter : {}; - const doc = await Model.findOne({ $and: [{ _id: id }, baseFilter] }).lean(); + let query = Model.findOne({ $and: [{ _id: id }, baseFilter] }); + + // Handle population for single doc + const rawPopulateParam = req.query.populate || req.query.expand; + if (rawPopulateParam) { + const populateParam = Array.isArray(rawPopulateParam) ? rawPopulateParam.join(',') : String(rawPopulateParam); + const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); + fields.forEach(f => { + query = query.populate(f); + }); + } + + const doc = await query.lean(); if (!doc) return res.status(404).json({ error: "Document not found." }); res.json(doc); diff --git a/docs/api-reference.md b/docs/api-reference.md index 58813da8e..9968d1cc1 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,6 +24,18 @@ | **Storage** | `POST` | `/api/storage/upload` | Upload a file | | **Storage** | `DELETE` | `/api/storage/file` | Delete a file by path | +## Data Query Parameters + +| Parameter | Example | Description | +| :--- | :--- | :--- | +| `populate` | `?populate=author` | Expand Reference fields into objects | +| `expand` | `?expand=author` | Alias for `populate` | +| `sort` | `?sort=field:asc` | Sort order (`asc`, `desc`, `1`, `-1`) | +| `limit` | `?limit=50` | Limit results (default 100) | +| `page` | `?page=1` | Pagination page number | +| `field_gt` | `?price_gt=100` | Greater than filter | +| `field_lt` | `?price_lt=50` | Less than filter | + ## Common Headers | Header | Required | Purpose | diff --git a/docs/database.md b/docs/database.md index 2a76e9c50..bfe5c5ee8 100644 --- a/docs/database.md +++ b/docs/database.md @@ -61,6 +61,33 @@ curl "https://api.ub.bitbros.in/api/data/posts/64fd1234abcd5678ef901234" ^ -H "x-api-key: pk_live_xxx" ``` +## 2.1 Query Parameters + +You can refine your `GET` requests using the following query parameters: + +| Parameter | Type | Example | Description | +| :--- | :--- | :--- | :--- | +| `populate` | `String` | `?populate=author,category` | Expand `Ref` fields into full objects. | +| `expand` | `String` | `?expand=author` | Alias for `populate`. | +| `sort` | `String` | `?sort=createdAt:desc` | Sort results by field and order. | +| `limit` | `Number` | `?limit=25` | Number of documents to return (default: 100, max: 100). | +| `page` | `Number` | `?page=2` | 1-indexed page number for pagination. | + +### Relational Data (Population) 🔗 + +If you have fields defined as `Ref` (singular) or `Array of Ref` in your schema, urBackend can automatically join those documents for you. + +By default, urBackend returns raw IDs for references. Use `?populate=` to get the full nested object. + +```bash +# Get posts with full author and category objects +curl "https://api.ub.bitbros.in/api/data/posts?populate=author,category" ^ + -H "x-api-key: pk_live_xxx" +``` + +> [!TIP] +> This solves the N+1 query problem, allowing you to fetch complex relational data in a single request. + ## 3. Update a Document **Endpoint**: `PUT /api/data/:collectionName/:id` diff --git a/packages/common/src/utils/injectModel.js b/packages/common/src/utils/injectModel.js index da93f7310..0c350bc75 100644 --- a/packages/common/src/utils/injectModel.js +++ b/packages/common/src/utils/injectModel.js @@ -10,7 +10,7 @@ const typeMapping = { }; // Recursive field definition builder -function buildFieldDef(field) { +function buildFieldDef(field, projectId, isExternal) { // Object type — nested sub-schema if (field.type === "Object" && field.fields && field.fields.length > 0) { const subSchema = {}; @@ -18,7 +18,7 @@ function buildFieldDef(field) { const normalizedKey = normalizeKey(f.key); if (!normalizedKey) return; - subSchema[normalizedKey] = buildFieldDef(f); + subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal); }); return { type: subSchema, required: !!field.required }; } @@ -42,14 +42,15 @@ function buildFieldDef(field) { const normalizedKey = normalizeKey(f.key); if (!normalizedKey) return; - subSchema[normalizedKey] = buildFieldDef(f); + subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal); }); return { type: [subSchema], required: !!field.required }; } // Array of Ref - if (field.items.type === "Ref") { + if (field.items && field.items.type === "Ref") { + const targetRef = isExternal ? field.items.ref : `${projectId}_${field.items.ref}`; return { - type: [{ type: mongoose.Schema.Types.ObjectId }], + type: [{ type: mongoose.Schema.Types.ObjectId, ref: targetRef }], required: !!field.required, }; } @@ -61,8 +62,10 @@ function buildFieldDef(field) { // Ref type — stores ObjectId if (field.type === "Ref") { + const targetRef = isExternal ? field.ref : `${projectId}_${field.ref}`; return { type: mongoose.Schema.Types.ObjectId, + ref: targetRef, required: !!field.required, }; } @@ -79,14 +82,14 @@ function normalizeKey(key) { .trim(); } -function buildMongooseSchema(fieldsArray = []) { +function buildMongooseSchema(fieldsArray = [], projectId, isExternal) { const schemaDef = {}; fieldsArray.forEach((field) => { const normalizedKey = normalizeKey(field.key); if (!normalizedKey) return; - schemaDef[normalizedKey] = buildFieldDef(field); + schemaDef[normalizedKey] = buildFieldDef(field, projectId, isExternal); }); return new mongoose.Schema(schemaDef, { @@ -123,7 +126,7 @@ function getCompiledModel(connection, collectionData, projectId, isExternal) { } // Build schema + compile - const schema = buildMongooseSchema(collectionData.model); + const schema = buildMongooseSchema(collectionData.model, projectId, isExternal); const model = connection.model(collectionName, schema); // Cache it diff --git a/packages/common/src/utils/queryEngine.js b/packages/common/src/utils/queryEngine.js index 6a65d94bd..e06588164 100644 --- a/packages/common/src/utils/queryEngine.js +++ b/packages/common/src/utils/queryEngine.js @@ -6,7 +6,7 @@ class QueryEngine { filter() { const queryObj = { ...this.queryString }; - const excludedFields = ['page', 'sort', 'limit', 'fields']; + const excludedFields = ['page', 'sort', 'limit', 'fields', 'populate', 'expand']; excludedFields.forEach(el => delete queryObj[el]); const mongoQuery = {}; @@ -53,13 +53,24 @@ class QueryEngine { paginate() { const page = parseInt(this.queryString.page, 10) || 1; - const limit = parseInt(this.queryString.limit, 10) || 100; + const limit = Math.min(parseInt(this.queryString.limit, 10) || 100, 100); const skip = (page - 1) * limit; this.query = this.query.skip(skip).limit(limit); return this; } + + populate() { + const rawParam = this.queryString.populate || this.queryString.expand; + if (!rawParam) return this; + const populateParam = Array.isArray(rawParam) ? rawParam.join(',') : String(rawParam); + const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean); + fields.forEach(f => { + this.query = this.query.populate(f); + }); + return this; + } } module.exports = QueryEngine;