File Upload with NodeJS and GridFS – MongoDB

February 28, 2017
by Rahil Shaikh
  • Fileupload nodejs and gridfs

Sometimes it is best to store files directly into the database instead of the filesystem. It may be that there is a limit to the number of files that can be stored on the filesystem and it’s always more manageable to have the metadata and files stored at the same place. Storing files and metadata in the DB together also makes replication easier.

MongoDB offers the GridFS driver for handling large files. The maximum size a document can hold in MongoDB is 16mb. Instead of storing the files in a single document GridFS divides it into multiple documents known as chunks. GridFS works by creating two collections for storing your files. fs.files stores the file metadata and other details while the fs.chunks stores the chunks.

Not going into much more details of the what and why part, in this tutorial, we will focus on the “How” part of it, we will learn how we can upload files into MongoDB with nodejs and GridFS.

For the front-end I will use the Angular 2  app we had built for one of our earlier tutorials on file upload.

DOWNLOADDEMO

Prerequisite

First and foremost we need to have MongoDB installed and running on our machine. You can get it installed if you haven’t by following the instructions on the official website. We also need node and npm installed ofcourse.

What we will be building?

For this tutorial, we will have 2 APIs, built with Node.js and ExpressJS. The first API will be responsible for uploading the files into MongoDB using GridFS driver and the second API will get the uploaded file to be viewed/downloaded. We will also need a front-end app that will allow us to upload files. We will use an Angular 2 app which I had built for one of my previous tutorials, however, this article itself won’t include the explanation for browser app but I will provide links to the same.

Build our Application

Start in an empty project directory by running npm init --yes. This will create a package.json for us.

Change the package.json file to look like below.

package.json
{
  "name": "file-upload-gridfs",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo "Error: no test specified" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "body-parser": "1.16.1",
    "express": "4.14.1",
    "gridfs-stream": "1.1.1",
    "mongoose": "4.8.4",
    "multer": "1.3.0",
    "multer-gridfs-storage": "1.0.0"
  }
}

Save this and run npm i. This will install all the project dependencies for us.

You might have noticed a few unfamiliar packages.
We are using mongoose as our ORM for working with MongoDB. I have chosen this over the official MongoDB driver because many of the people prefer mongoose and there are good examples of working with GridFS with the official driver but not so much with mongoose.
We are using multer to upload files and multer-gridfs-storage to setup the storage option for the same. gridfs-stream module provides all the utilities for working with GridFS using Node.js

Create a file named app.js. This is where our application code will go.

package.json
    var express = require('express');
    var app = express();
    var bodyParser = require('body-parser');
    var mongoose = require('mongoose');
    mongoose.connect('mongodb://127.0.0.1/gridfstest');
    var conn = mongoose.connection;
    var multer = require('multer');
    var GridFsStorage = require('multer-gridfs-storage');
    var Grid = require('gridfs-stream');
    Grid.mongo = mongoose.mongo;
    var gfs = Grid(conn.db);

    /** Seting up server to accept cross-origin browser requests */
    app.use(function(req, res, next) { //allow cross origin requests
        res.setHeader("Access-Control-Allow-Methods", "POST, PUT, OPTIONS, DELETE, GET");
        res.header("Access-Control-Allow-Origin", "http://localhost:3000");
        res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
        res.header("Access-Control-Allow-Credentials", true);
        next();
    });

    app.use(bodyParser.json());

    /** Setting up storage using multer-gridfs-storage */
    var storage = GridFsStorage({
        gfs : gfs,
        filename: function (req, file, cb) {
            var datetimestamp = Date.now();
            cb(null, file.fieldname + '-' + datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length -1]);
        },
        /** With gridfs we can store aditional meta-data along with the file */
        metadata: function(req, file, cb) {
            cb(null, { originalname: file.originalname });
        },
        root: 'ctFiles' //root name for collection to store files into
    });

    var upload = multer({ //multer settings for single upload
        storage: storage
    }).single('file');

    /** API path that will upload the files */
    app.post('/upload', function(req, res) {
        upload(req,res,function(err){
            if(err){
                 res.json({error_code:1,err_desc:err});
                 return;
            }
             res.json({error_code:0,err_desc:null});
        });
    });

    app.get('/file/:filename', function(req, res){
        gfs.collection('ctFiles'); //set collection name to lookup into

        /** First check if file exists */
        gfs.files.find({filename: req.params.filename}).toArray(function(err, files){
            if(!files || files.length === 0){
                return res.status(404).json({
                    responseCode: 1,
                    responseMessage: "error"
                });
            }
            /** create read stream */
            var readstream = gfs.createReadStream({
                filename: files[0].filename,
                root: "ctFiles"
            });
            /** set the proper content type */
            res.set('Content-Type', files[0].contentType)
            /** return response */
            return readstream.pipe(res);
        });
    });

    app.listen('3002', function(){
        console.log('running on 3002...');
    });

This is about the code that we need to upload files using Node.js and GridFS. We are woking with streams because GridFS streams file in and out of MongoDB. Also, if you see we are using filename to retrieve files from MongoDB, if you follow this you need to make sure the filenames are unique, or else you can always use _id provided by MongoDB. The original file name can be stored as metadata.

In your command line tool, run node app.js to start the server. We now have a node server running that can upload files to MongoDB, but we also need a browser app from where we can upload it.

The explanation to the client side application is not in the scope of this article, but we already have two file uploader apps done in AngularJS and Angular 2 respectively, you can follow the guides below to build one.

  • For Angular 1 app refer here
  • For Angular 2 app refer here

Quick Setup

I have included the Angular 2 file uploader app along with the GridFS module in the repository. Follow the below steps to get it running quickly.

  • git clone https://github.com/rahil471/fileupload-nodejs-gridfs-angular2.git
  • Naviagte into the directory
  • npm install
  • MongoDB should be installed and running on default port
  • node app.js this will start the node server on port 3002
  • cd angular2-app
  • npm install
  • npm start
  • angular2 app will be running on port 3000

Watch The Demo!

Conclusion

In this article, we have seen how we can upload files into MongoDB using NodeJS and GridFS. GridFS takes care of most of the complexity and leaves us with very little work to do. Hope this article showed you an easy way for performing File Upload with GridFS. Happy File Uploading!

Further Links

About

Engineer. Blogger. Thinker. Loves programming and working with emerging tech. We can also talk on Football, Gaming, World Politics, Monetary Systems.

Free PDF

Subscribe and get AngularJS Tips PDF. We never spam!
First Name:
Email:

10 comments

  1. Saahil
    |

    “Cannot GET /file/file-1489260162515.jpg.”

    Error keeps coming if i try to access the image.
    Please help, any input would be greatly appreciated.

    • Rahil Shaikh
      |

      I did a rerun on this and it works fine for me. Would you perhaps email me your code and I’ll look into it? Would recommend trying from the github repo which has been provided.

  2. |

    Its a good tutorial.
    I have a question
    how to handle text and file content from a form which Has to be stored as a single document in the database?

    • |

      By file content you mean you are only interested in storing the content and not uploading the entire file?

  3. |

    Its a good tutorial.
    I have a question

    how to get response json file?

  4. |

    There is an error in GridFsStorage init, gfs:gfs is useless, but instead of it you need to set url property (or an error is thrown)

    var storage = GridFsStorage({
            gfs : gfs,  <----- replace with: url: 'mongodb://127.0.0.1/gridfstest'
            filename: function (req, file, cb) {
                var datetimestamp = Date.now();
                cb(null, file.fieldname + '-' + datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length -1]);
            },
            /** With gridfs we can store aditional meta-data along with the file */
            metadata: function(req, file, cb) {
                cb(null, { originalname: file.originalname });
            },
            root: 'ctFiles' //root name for collection to store files into
        });
  5. |

    Hello,
    I followed your guidelines how to implement the file upload with nodeJS and Gridfs.
    For unknown reasons I get this bizzare error when trying to run the code.

    if (!db) throw new Error(‘missing db argument\nnew Grid(db, mongo)’);
    ^

    Error: missing db argument
    new Grid(db, mongo)

    Any ideas how to solve this problem?

    • |

      I’ll have to have a look into it.

  6. |

    Excellent tutorial Rahil. I am struggling to find out the way to read this image file from the mongodb and display in webpage (below the form). Could you please provide the code sample that I need to include in my angular 2 project.

  7. |

    How do I get the object-ID of the file after it has been saved in mongoDB ?

Leave a Comment