fix: harden populate robustness, clamp paginate limit, fix docs and test coverage

apps/public-api/src/__tests__/data.controller.read.test.js

@@ -4,6 +4,8 @@ const mockFind = jest.fn();
 const mockAnd = jest.fn();
 const mockFindOne = jest.fn();
 const mockQueryLean = jest.fn().mockResolvedValue([]);
+const mockPopulate = jest.fn();
+const mockEnginePopulate = jest.fn();
 
 // mockAnd returns an object with .lean() so features.query.lean() works after .and()
 mockAnd.mockReturnValue({ lean: mockQueryLean });
@@ -13,6 +15,7 @@ const mockQueryEngine = jest.fn((query) => {
         query,
         filter() { return engine; },
         sort() { return engine; },
+        populate() { mockEnginePopulate(...arguments); return engine; },
         paginate() { return engine; },
     };
     return engine;
@@ -25,11 +28,25 @@ jest.mock('@urbackend/common', () => ({
     getCompiledModel: jest.fn(() => ({
         find: (...args) => {
             mockFind(...args);
-            return { and: mockAnd, lean: mockQueryLean };
+            return { 
+                and: mockAnd, 
+                populate: (...pArgs) => {
+                    mockPopulate(...pArgs);
+                    return { lean: mockQueryLean };
+                },
+                lean: mockQueryLean 
+            };
         },
         findOne: (...args) => {
             mockFindOne(...args);
-            return { lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }) };
+            const chainable = {
+                populate: (...pArgs) => {
+                    mockPopulate(...pArgs);
+                    return chainable;
+                },
+                lean: jest.fn().mockResolvedValue({ _id: 'doc_1' }),
+            };
+            return chainable;
         },
     })),
     QueryEngine: mockQueryEngine,
@@ -74,6 +91,7 @@ function makeRes() {
 describe('data.controller read RLS filters', () => {
     beforeEach(() => {
         jest.clearAllMocks();
+        mockEnginePopulate.mockClear();
     });
 
     test('getAllData applies rlsFilter to find()', async () => {
@@ -101,4 +119,50 @@ describe('data.controller read RLS filters', () => {
         });
         expect(res.json).toHaveBeenCalled();
     });
+
+    test('getAllData calls engine.populate() when populate param is provided', async () => {
+        const req = makeReq({ query: { populate: 'author,comments' } });
+        const res = makeRes();
+
+        await getAllData(req, res);
+
+        expect(mockQueryEngine).toHaveBeenCalled();
+        expect(mockEnginePopulate).toHaveBeenCalled();
+    });
+
+    test('getAllData does not forward populate/expand to Mongo filter', async () => {
+        const req = makeReq({ query: { populate: 'author', expand: 'category', title: 'hello' } });
+        const res = makeRes();
+
+        await getAllData(req, res);
+
+        // mockFind is called with the raw Mongoose query (no args for Model.find())
+        // The real filter exclusion is tested via the QueryEngine directly,
+        // but we confirm find() was invoked and the request did not error out.
+        expect(mockFind).toHaveBeenCalledWith();
+        expect(res.json).toHaveBeenCalled();
+        expect(res.status).not.toHaveBeenCalledWith(500);
+    });
+
+    test('getSingleDoc calls populate on the query', async () => {
+        const req = makeReq({ query: { populate: 'author' } });
+        const res = makeRes();
+
+        await getSingleDoc(req, res);
+
+        expect(mockPopulate).toHaveBeenCalledWith('author');
+        expect(res.json).toHaveBeenCalled();
+    });
+
+    test('getSingleDoc handles array-format populate param without crashing', async () => {
+        const req = makeReq({ query: { populate: ['author', 'category'] } });
+        const res = makeRes();
+
+        await getSingleDoc(req, res);
+
+        expect(mockPopulate).toHaveBeenCalledWith('author');
+        expect(mockPopulate).toHaveBeenCalledWith('category');
+        expect(res.json).toHaveBeenCalled();
+        expect(res.status).not.toHaveBeenCalledWith(500);
+    });
 });

apps/public-api/src/__tests__/mail.controller.test.js

@@ -20,6 +20,7 @@ jest.mock('@urbackend/common', () => {
         incr: jest.fn(),
         expire: jest.fn(),
         decr: jest.fn(),
+        eval: jest.fn(),
     };
 
     return {
@@ -66,14 +67,13 @@ describe('mail.controller', () => {
         const req = makeReq();
         const res = makeRes();
 
-        mockProjectConfig({ _id: 'proj_1', resendApiKey: {} });
+        mockProjectConfig({ _id: 'proj_1', resendApiKey: { encrypted: '...' } });
         decrypt.mockReturnValue('byok-key');
-        redis.incr.mockResolvedValue(1);
+        redis.eval.mockResolvedValue(1);
 
         await mailController.sendMail(req, res);
 
-        expect(redis.incr).toHaveBeenCalledTimes(1);
-        expect(redis.expire).toHaveBeenCalledTimes(1);
+        expect(redis.eval).toHaveBeenCalledTimes(1);
         expect(res.status).toHaveBeenCalledWith(200);
         expect(res.json).toHaveBeenCalledWith(expect.objectContaining({
             success: true,
@@ -87,7 +87,7 @@ describe('mail.controller', () => {
 
         mockProjectConfig({ _id: 'proj_1', resendApiKey: null });
         decrypt.mockReturnValue(null);
-        redis.incr.mockResolvedValue(2);
+        redis.eval.mockResolvedValue(2);
 
         await mailController.sendMail(req, res);
 
@@ -103,14 +103,15 @@ describe('mail.controller', () => {
 
         mockProjectConfig({ _id: 'proj_1', resendApiKey: null });
         decrypt.mockReturnValue(null);
-        redis.incr.mockResolvedValue(101);
+        redis.eval.mockResolvedValue(101);
 
         await mailController.sendMail(req, res);
 
         expect(redis.decr).toHaveBeenCalledTimes(1);
         expect(res.status).toHaveBeenCalledWith(429);
         expect(res.json).toHaveBeenCalledWith(expect.objectContaining({
-            error: 'Monthly mail limit exceeded.',
+            success: false,
+            message: 'Monthly mail limit exceeded.',
         }));
     });
 });

apps/public-api/src/controllers/data.controller.js

@@ -122,6 +122,7 @@ module.exports.getAllData = async (req, res) => {
 
     features
       .sort()
+      .populate()
       .paginate();
 
     const data = await features.query.lean();
@@ -158,7 +159,19 @@ module.exports.getSingleDoc = async (req, res) => {
     );
 
     const baseFilter = req.rlsFilter && typeof req.rlsFilter === 'object' ? req.rlsFilter : {};
-    const doc = await Model.findOne({ $and: [{ _id: id }, baseFilter] }).lean();
+    let query = Model.findOne({ $and: [{ _id: id }, baseFilter] });
+
+    // Handle population for single doc
+    const rawPopulateParam = req.query.populate || req.query.expand;
+    if (rawPopulateParam) {
+      const populateParam = Array.isArray(rawPopulateParam) ? rawPopulateParam.join(',') : String(rawPopulateParam);
+      const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean);
+      fields.forEach(f => {
+        query = query.populate(f);
+      });
+    }
+
+    const doc = await query.lean();
     if (!doc) return res.status(404).json({ error: "Document not found." });
 
     res.json(doc);

docs/api-reference.md

@@ -24,6 +24,18 @@
 | **Storage** | `POST` | `/api/storage/upload` | Upload a file |
 | **Storage** | `DELETE` | `/api/storage/file` | Delete a file by path |
 
+## Data Query Parameters
+
+| Parameter | Example | Description |
+| :--- | :--- | :--- |
+| `populate` | `?populate=author` | Expand Reference fields into objects |
+| `expand` | `?expand=author` | Alias for `populate` |
+| `sort` | `?sort=field:asc` | Sort order (`asc`, `desc`, `1`, `-1`) |
+| `limit` | `?limit=50` | Limit results (default 100) |
+| `page` | `?page=1` | Pagination page number |
+| `field_gt` | `?price_gt=100` | Greater than filter |
+| `field_lt` | `?price_lt=50` | Less than filter |
+
 ## Common Headers
 
 | Header | Required | Purpose |

docs/database.md

@@ -61,6 +61,33 @@ curl "https://api.ub.bitbros.in/api/data/posts/64fd1234abcd5678ef901234" ^
   -H "x-api-key: pk_live_xxx"
 ```
 
+## 2.1 Query Parameters
+
+You can refine your `GET` requests using the following query parameters:
+
+| Parameter | Type | Example | Description |
+| :--- | :--- | :--- | :--- |
+| `populate` | `String` | `?populate=author,category` | Expand `Ref` fields into full objects. |
+| `expand` | `String` | `?expand=author` | Alias for `populate`. |
+| `sort` | `String` | `?sort=createdAt:desc` | Sort results by field and order. |
+| `limit` | `Number` | `?limit=25` | Number of documents to return (default: 100, max: 100). |
+| `page` | `Number` | `?page=2` | 1-indexed page number for pagination. |
+
+### Relational Data (Population) 🔗
+
+If you have fields defined as `Ref` (singular) or `Array of Ref` in your schema, urBackend can automatically join those documents for you. 
+
+By default, urBackend returns raw IDs for references. Use `?populate=` to get the full nested object.
+
+```bash
+# Get posts with full author and category objects
+curl "https://api.ub.bitbros.in/api/data/posts?populate=author,category" ^
+  -H "x-api-key: pk_live_xxx"
+```
+
+> [!TIP]
+> This solves the N+1 query problem, allowing you to fetch complex relational data in a single request.
+
 ## 3. Update a Document
 
 **Endpoint**: `PUT /api/data/:collectionName/:id`

packages/common/src/utils/injectModel.js

@@ -10,15 +10,15 @@ const typeMapping = {
 };
 
 // Recursive field definition builder
-function buildFieldDef(field) {
+function buildFieldDef(field, projectId, isExternal) {
   // Object type — nested sub-schema
   if (field.type === "Object" && field.fields && field.fields.length > 0) {
     const subSchema = {};
     field.fields.forEach((f) => {
       const normalizedKey = normalizeKey(f.key);
       if (!normalizedKey) return;
 
-      subSchema[normalizedKey] = buildFieldDef(f);
+      subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal);
     });
     return { type: subSchema, required: !!field.required };
   }
@@ -42,14 +42,15 @@ function buildFieldDef(field) {
         const normalizedKey = normalizeKey(f.key);
         if (!normalizedKey) return;
 
-        subSchema[normalizedKey] = buildFieldDef(f);
+        subSchema[normalizedKey] = buildFieldDef(f, projectId, isExternal);
       });
       return { type: [subSchema], required: !!field.required };
     }
     // Array of Ref
-    if (field.items.type === "Ref") {
+    if (field.items && field.items.type === "Ref") {
+      const targetRef = isExternal ? field.items.ref : `${projectId}_${field.items.ref}`;
       return {
-        type: [{ type: mongoose.Schema.Types.ObjectId }],
+        type: [{ type: mongoose.Schema.Types.ObjectId, ref: targetRef }],
         required: !!field.required,
       };
     }
@@ -61,8 +62,10 @@ function buildFieldDef(field) {
 
   // Ref type — stores ObjectId
   if (field.type === "Ref") {
+    const targetRef = isExternal ? field.ref : `${projectId}_${field.ref}`;
     return {
       type: mongoose.Schema.Types.ObjectId,
+      ref: targetRef,
       required: !!field.required,
     };
   }
@@ -79,14 +82,14 @@ function normalizeKey(key) {
     .trim();
 }
 
-function buildMongooseSchema(fieldsArray = []) {
+function buildMongooseSchema(fieldsArray = [], projectId, isExternal) {
   const schemaDef = {};
 
   fieldsArray.forEach((field) => {
     const normalizedKey = normalizeKey(field.key);
     if (!normalizedKey) return;
 
-    schemaDef[normalizedKey] = buildFieldDef(field);
+    schemaDef[normalizedKey] = buildFieldDef(field, projectId, isExternal);
   });
 
   return new mongoose.Schema(schemaDef, {
@@ -123,7 +126,7 @@ function getCompiledModel(connection, collectionData, projectId, isExternal) {
   }
 
   // Build schema + compile
-  const schema = buildMongooseSchema(collectionData.model);
+  const schema = buildMongooseSchema(collectionData.model, projectId, isExternal);
   const model = connection.model(collectionName, schema);
 
   // Cache it

packages/common/src/utils/queryEngine.js

@@ -6,7 +6,7 @@ class QueryEngine {
 
     filter() {
         const queryObj = { ...this.queryString };
-        const excludedFields = ['page', 'sort', 'limit', 'fields'];
+        const excludedFields = ['page', 'sort', 'limit', 'fields', 'populate', 'expand'];
         excludedFields.forEach(el => delete queryObj[el]);
 
         const mongoQuery = {};
@@ -53,13 +53,24 @@ class QueryEngine {
 
     paginate() {
         const page = parseInt(this.queryString.page, 10) || 1;
-        const limit = parseInt(this.queryString.limit, 10) || 100;
+        const limit = Math.min(parseInt(this.queryString.limit, 10) || 100, 100);
         const skip = (page - 1) * limit;
 
         this.query = this.query.skip(skip).limit(limit);
 
         return this;
     }
+
+    populate() {
+        const rawParam = this.queryString.populate || this.queryString.expand;
+        if (!rawParam) return this;
+        const populateParam = Array.isArray(rawParam) ? rawParam.join(',') : String(rawParam);
+        const fields = populateParam.split(',').map(f => f.trim()).filter(Boolean);
+        fields.forEach(f => {
+            this.query = this.query.populate(f);
+        });
+        return this;
+    }
 }
 
 module.exports = QueryEngine;