For a recent project I had to build a service that when the user uploads a zip file. It would unpack it, process it and upload it to S3, on other platforms like Rails this could be achieved by Paperclip or some similar library but as we were building everything in Node.JS this would have to be done from the ground up.

Upload

To build the uploader I used a React package called DropZone which creates an area of the form a user can drag-and-drop a file on, or use the native file select.

import Dropzone from 'react-dropzone';

render() {
  return (<div>
    <Dropzone
      disableClick={true}
      multiple={false}
      accept={'application/zip'}
      onDrop={this.onDrop}>
      <div className="dropzone-text">Drop your zip file here</div>
    </Dropzone>
    { (this.state.percent > 0) &&
       <div className="progress-upload">
         <Progress
            bar
            color="success"
            value={this.state.percent}
            className="upload-story-bar">
              {Math.round(this.state.percent)}%
          </Progress>
       </div>
     }
  </div>
)}

As you can see the dropzone component is wired to only accept files with a mime-type of 'zip' so only zip files will be allowed for upload.

I also use the Progress component from ReactStrap (a React wrapper for Bootstrap to render a progress bar as the upload is taking place).

When the onDrop action here is the resulting handler:

onDrop = (file) => {
  this.setState({ percent: 0 });
  let data = new FormData();
  let singleFile = file[0];
  data.append('media', singleFile);
  var that = this;

  SuperAgent.put(`/v1/story/upload_video/${this.state.id}`)
    .send(data)
    .on('progress', e => {
      if(e.direction === "upload") {
        this.setState({
          percent: e.percent,
          percent_message: `${e.direction} is done ${e.percent}%`
        });
      }
    })
    .end(function(err, resp) {
      if (err) {
        this.handleError(err);
      }
      that.setState({
        upload_filename: resp.body.upload_filename,
        upload_filepath: resp.body.upload_filepath,
        extract_location: resp.body.extract_location,
        complete: true
      });

      return resp;
    });
}

As you can see we're using the SuperAgent library to handle the put request to the server with .on('progress') updates the progress state and updates the progress bar on the render.

Backend

All the while this is happening we need to have a way of handling the upload to the backend, unzipping it and logging a task to so we can later upload it to s3.
This we'll do via the decompress & decompressUnzip libraries, multer for multi-file upload and more importantly Agenda to handle task logging.

var story = require('../../models/story.js');
const decompress = require('decompress');
const decompressUnzip = require('decompress-unzip');
var multer = require('multer');
const path = require('path');
require('dotenv').config();

const Agenda = require('agenda');
var agenda = new Agenda({db: {address: process.env.MONGODB_URI}});

var storage = multer.diskStorage({
  destination: function (req, file, cb) {
    cb(null, 'media')
  },
  filename: function (req, file, cb) {
    cb(null, req.params.id + '.zip')
  }
})

// upload file handler
exports.upload = multer({ storage: storage })

// upload api route
exports.uploadMedia = function(req, res) {
  const query =  { id: req.params.id }
  story.findById(query, function(err, record) {
    if (req.files.length > 0) {
      try {
        let filename = req.files[0].filename
        let filepath = req.files[0].path
        let extract_location = filepath.replace('.zip', '')

        record.upload_filename = filename
        record.upload_filepath = filepath
        record.extract_location = extract_location.replace('public/', '')

        // decompress zipfile
        decompress(filepath, extract_location, {
          plugins: [
            decompressUnzip()
          ]
        }).then(() => {
          console.log('zipfile decompressed');

          // delete zipfile
          let fs = require('fs');
          fs.unlink(filepath, function() {
            console.log('zipfile deleted');

            record.s3_extract_status = "processing"
            record.save()

            var job = agenda.create('upload ready', {
              extract_location: record.extract_location,
              story_id: record.id
            });

            job.save(function(err) {
              console.log('Job successfully saved');
            });

          });
        });

      } catch(err) {
        res.status(422).send({ message: err.responseText})
      }
    }
    record.save()
    .then(function(record) {
      res.status(200).send(record);
    }).catch(function(err) {
      res.status(422).send({ message: err.responseText);
    });
  });
};

At the start you can see where we define our disk storage directory and filename, the handler and when we get into the working of the put handler a try / catch block to decompress the zipFile, update the status and more importantly log the task 'upload ready' to Agenda.

Agenda is a background task worker that while running on a different process, watches the 'agendaJobs' table for new tasks. When one is added it then checks it's lastRunAt and whether it's been disabled. If it hasn't been disabled or run yet it defers it for processing which it then hands to the related task handler. In the next post we'll go through Agenda and how we managed to do the recursive file uploading.