Adding Pagination to Your Node.js API

This tutorial builds on our tested API by implementing robust pagination for GET requests. We'll cover different pagination strategies, cursor-based pagination, and how to handle large datasets efficiently.

Prerequisites

• Completed Parts 1-6 of the tutorial
• Basic understanding of REST APIs
• MongoDB and Node.js installed

Project Setup

Install the required dependencies:

npm install mongoose-cursor-pagination query-string

Updated Project Structure

Add these new files to your project:

books-api/
├── src/
│   ├── config/
│   │   └── pagination.js
│   ├── middleware/
│   │   └── pagination.js
│   ├── utils/
│   │   └── paginationHelpers.js
│   └── types/
│       └── pagination.js

Pagination Configuration

Create src/config/pagination.js:

module.exports = {
    // Default pagination settings
    DEFAULT_PAGE_SIZE: 10,
    MAX_PAGE_SIZE: 100,
    
    // Sorting defaults
    DEFAULT_SORT_FIELD: 'createdAt',
    DEFAULT_SORT_ORDER: 'desc',
    
    // Cursor settings
    CURSOR_ENCODING: 'base64',
    
    // Response field names
    META_FIELDS: {
        totalCount: 'totalCount',
        pageSize: 'pageSize',
        currentPage: 'currentPage',
        totalPages: 'totalPages',
        hasNextPage: 'hasNextPage',
        hasPrevPage: 'hasPrevPage',
        nextCursor: 'nextCursor',
        prevCursor: 'prevCursor'
    }
};

Pagination Middleware

Create src/middleware/pagination.js:

const { 
    DEFAULT_PAGE_SIZE,
    MAX_PAGE_SIZE,
    DEFAULT_SORT_FIELD,
    DEFAULT_SORT_ORDER
} = require('../config/pagination');

// Offset-based pagination middleware
const offsetPagination = (req, res, next) => {
    const page = parseInt(req.query.page) || 1;
    const limit = Math.min(
        parseInt(req.query.limit) || DEFAULT_PAGE_SIZE,
        MAX_PAGE_SIZE
    );
    const skip = (page - 1) * limit;
    const sortField = req.query.sortBy || DEFAULT_SORT_FIELD;
    const sortOrder = req.query.order || DEFAULT_SORT_ORDER;
    
    req.pagination = {
        page,
        limit,
        skip,
        sort: { [sortField]: sortOrder === 'desc' ? -1 : 1 }
    };
    
    next();
};

// Cursor-based pagination middleware
const cursorPagination = (req, res, next) => {
    const limit = Math.min(
        parseInt(req.query.limit) || DEFAULT_PAGE_SIZE,
        MAX_PAGE_SIZE
    );
    const cursor = req.query.cursor;
    const sortField = req.query.sortBy || DEFAULT_SORT_FIELD;
    const sortOrder = req.query.order || DEFAULT_SORT_ORDER;
    
    req.pagination = {
        limit,
        cursor,
        sort: { [sortField]: sortOrder === 'desc' ? -1 : 1 }
    };
    
    next();
};

module.exports = {
    offsetPagination,
    cursorPagination
};

Pagination Helpers

Create src/utils/paginationHelpers.js:

const { META_FIELDS, CURSOR_ENCODING } = require('../config/pagination');

// Helper for offset-based pagination
const createOffsetPaginationResponse = async (
    model,
    query,
    pagination,
    projection = {},
    populate = []
) => {
    const { page, limit, skip, sort } = pagination;
    
    // Execute queries in parallel
    const [data, totalCount] = await Promise.all([
        model
            .find(query, projection)
            .sort(sort)
            .skip(skip)
            .limit(limit)
            .populate(populate),
        model.countDocuments(query)
    ]);
    
    const totalPages = Math.ceil(totalCount / limit);
    
    return {
        data,
        meta: {
            [META_FIELDS.totalCount]: totalCount,
            [META_FIELDS.pageSize]: limit,
            [META_FIELDS.currentPage]: page,
            [META_FIELDS.totalPages]: totalPages,
            [META_FIELDS.hasNextPage]: page < totalPages,
            [META_FIELDS.hasPrevPage]: page > 1
        }
    };
};

// Helper for cursor-based pagination
const createCursorPaginationResponse = async (
    model,
    query,
    pagination,
    projection = {},
    populate = []
) => {
    const { limit, cursor, sort } = pagination;
    const sortField = Object.keys(sort)[0];
    const sortOrder = sort[sortField];
    
    // Add cursor to query if provided
    if (cursor) {
        const decodedCursor = Buffer.from(cursor, CURSOR_ENCODING).toString('utf8');
        const [cursorValue, cursorId] = decodedCursor.split('_');
        
        const cursorQuery = {
            $or: [
                {
                    [sortField]: sortOrder === 1 
                        ? { $gt: cursorValue }
                        : { $lt: cursorValue }
                },
                {
                    [sortField]: cursorValue,
                    _id: sortOrder === 1 
                        ? { $gt: cursorId }
                        : { $lt: cursorId }
                }
            ]
        };
        
        query = { $and: [query, cursorQuery] };
    }
    
    // Fetch one extra record to determine if there's a next page
    const data = await model
        .find(query, projection)
        .sort(sort)
        .limit(limit + 1)
        .populate(populate);
    
    const hasNextPage = data.length > limit;
    if (hasNextPage) {
        data.pop(); // Remove the extra record
    }
    
    // Create cursors
    const nextCursor = hasNextPage
        ? Buffer.from(
            `${data[data.length - 1][sortField]}_${data[data.length - 1]._id}`
        ).toString(CURSOR_ENCODING)
        : null;
    
    const prevCursor = cursor
        ? Buffer.from(
            `${data[0][sortField]}_${data[0]._id}`
        ).toString(CURSOR_ENCODING)
        : null;
    
    return {
        data,
        meta: {
            [META_FIELDS.pageSize]: limit,
            [META_FIELDS.hasNextPage]: hasNextPage,
            [META_FIELDS.hasPrevPage]: !!cursor,
            [META_FIELDS.nextCursor]: nextCursor,
            [META_FIELDS.prevCursor]: prevCursor
        }
    };
};

// Helper for generating pagination links
const generatePaginationLinks = (req, meta) => {
    const { protocol, hostname, originalUrl } = req;
    const baseUrl = `${protocol}://${hostname}`;
    const queryParams = new URLSearchParams(req.query);
    
    const links = {};
    
    if (meta.hasNextPage) {
        queryParams.set('page', meta.currentPage + 1);
        links.next = `${baseUrl}${originalUrl.split('?')[0]}?${queryParams}`;
    }
    
    if (meta.hasPrevPage) {
        queryParams.set('page', meta.currentPage - 1);
        links.prev = `${baseUrl}${originalUrl.split('?')[0]}?${queryParams}`;
    }
    
    return links;
};

module.exports = {
    createOffsetPaginationResponse,
    createCursorPaginationResponse,
    generatePaginationLinks
};

Updated Book Routes

Update src/routes/books.js to implement pagination:

const express = require('express');
const router = express.Router();
const { Book } = require('../models/book');
const { offsetPagination, cursorPagination } = require('../middleware/pagination');
const { 
    createOffsetPaginationResponse,
    createCursorPaginationResponse,
    generatePaginationLinks 
} = require('../utils/paginationHelpers');

// Get all books with offset-based pagination
router.get('/', offsetPagination, async (req, res) => {
    try {
        const query = {};
        
        // Apply filters
        if (req.query.genre) {
            query.genre = req.query.genre;
        }
        if (req.query.author) {
            query.author = new RegExp(req.query.author, 'i');
        }
        
        const result = await createOffsetPaginationResponse(
            Book,
            query,
            req.pagination,
            {}, // Projection
            ['author'] // Populate
        );
        
        // Generate HATEOAS links
        const links = generatePaginationLinks(req, result.meta);
        
        res.json({
            ...result,
            links
        });
    } catch (error) {
        res.status(500).json({ error: error.message });
    }
});

// Get all books with cursor-based pagination
router.get('/cursor', cursorPagination, async (req, res) => {
    try {
        const query = {};
        
        // Apply filters
        if (req.query.genre) {
            query.genre = req.query.genre;
        }
        if (req.query.author) {
            query.author = new RegExp(req.query.author, 'i');
        }
        
        const result = await createCursorPaginationResponse(
            Book,
            query,
            req.pagination,
            {}, // Projection
            ['author'] // Populate
        );
        
        res.json(result);
    } catch (error) {
        res.status(500).json({ error: error.message });
    }
});

Response Headers

Add pagination metadata to response headers:

const addPaginationHeaders = (req, res, next) => {
    const sendResponse = res.json;
    res.json = function(body) {
        if (body && body.meta) {
            res.set({
                'X-Total-Count': body.meta.totalCount,
                'X-Page-Size': body.meta.pageSize,
                'X-Current-Page': body.meta.currentPage,
                'X-Total-Pages': body.meta.totalPages,
                'X-Has-Next-Page': body.meta.hasNextPage,
                'X-Has-Prev-Page': body.meta.hasPrevPage
            });
        }
        return sendResponse.call(this, body);
    };
    next();
};

Pagination Tests

Create __tests__/integration/pagination.test.js:

const request = require('supertest');
const app = require('../../src/server');
const { createTestBook } = require('../../src/utils/testUtils');

describe('Pagination Tests', () => {
    // Create test data
    beforeEach(async () => {
        const books = Array(15).fill().map(() => createTestBook());
        await Promise.all(books);
    });
    
    describe('Offset-based Pagination', () => {
        it('should return paginated results with default settings', async () => {
            const response = await request(app)
                .get('/api/books')
                .expect(200);
            
            expect(response.body.data).toHaveLength(10); // Default page size
            expect(response.body.meta.currentPage).toBe(1);
            expect(response.body.meta.hasNextPage).toBe(true);
        });
        
        it('should respect custom page size', async () => {
            const response = await request(app)
                .get('/api/books?limit=5')
                .expect(200);
            
            expect(response.body.data).toHaveLength(5);
        });
        
        it('should enforce maximum page size', async () => {
            const response = await request(app)
                .get('/api/books?limit=200')
                .expect(200);
            
            expect(response.body.data.length).toBeLessThanOrEqual(100);
        });
    });
    
    describe('Cursor-based Pagination', () => {
        it('should return cursor-based results', async () => {
            const response = await request(app)
                .get('/api/books/cursor')
                .expect(200);
            
            expect(response.body.data).toBeDefined();
            expect(response.body.meta.nextCursor).toBeDefined();
        });
        
        it('should handle cursor navigation', async () => {
            // Get first page
            const firstResponse = await request(app)
                .get('/api/books/cursor?limit=5')
                .expect(200);
            
            // Get next page using cursor
            const nextResponse = await request(app)
                .get(`/api/books/cursor?cursor=${firstResponse.body.meta.nextCursor}`)
                .expect(200);
            
            expect(nextResponse.body.data).toHaveLength(5);
            expect(nextResponse.body.meta.hasPrevPage).toBe(true);
        });
    });
});

Best Practices Implemented

1. Multiple Strategies

   • Offset-based pagination
   • Cursor-based pagination
   • Flexible configuration
   • HATEOAS compliance

2. Performance Optimization

   • Parallel query execution
   • Cursor-based efficiency
   • Index utilization
   • Resource limiting

3. Developer Experience

   • Clear documentation
   • Consistent responses
   • Error handling
   • Flexible configuration

4. Client Integration

   • Response headers
   • HATEOAS links
   • Clear metadata
   • Cursor encoding

Next Steps

To enhance your API further:

1. Implementing proper logging

Conclusion

You now have a robust pagination system that:

• Handles large datasets efficiently
• Provides multiple pagination strategies
• Maintains consistent performance
• Follows REST best practices
• Supports various use cases

Remember to:

• Monitor pagination performance
• Index relevant fields
• Update documentation
• Test edge cases
• Consider caching strategies

The next tutorial will cover implementing proper logging for your API endpoints.