Sunday, 13 January 2019

node upload & download with gridfs

project site: https://chuanshuoge1-gridfs.herokuapp.com/

In MongoDB, use GridFS for storing files larger than 16 MB.
GridFS divides the document into chunks of size chunkSize  255 kilobytes (kB).

support all file format

extract download zip file

2 collections created, meta data and binary chunks


metadata

[ { _id: 5c3bf5a015291c001743024b,
    length: 574974,
    chunkSize: 261120,
    uploadDate: 2019-01-14T02:36:18.472Z,
    filename: '521476_013.gif',
    md5: '55d75f257bd0354ca843763306f6c90d',
    contentType: 'image/gif' },
  ...
    chunkSize: 261120,
    uploadDate: 2019-01-14T02:37:04.404Z,
    filename: 'IMG_20170517_215456.3gp',
    md5: '74c29e75141e225894f109a2fc0de00c',
    contentType: 'video/3gpp' } ]

--------------------------------------------------

binary chunks

{
    "_id": {
        "$oid": "5c3bf5a315291c0017430255"
    },
    "files_id": {
        "$oid": "5c3bf5a015291c001743024d"
    },
    "n": 1,
    "data": "<Binary Data>"
}
{
    "_id": {
        "$oid": "5c3bf5a315291c0017430256"
    },
    "files_id": {
        "$oid": "5c3bf5a015291c001743024d"
    },
    "n": 2,
    "data": "<Binary Data>"
}

------------------------------------------

chunks assembled into binary file 

IMG_20170517_215456.3gp

<Buffer ff d8 ff e1 30 7d 45 78 69 66 00 00 49 49 2a 00 08 00 00 00 11 00 0e 01 02 00 20 00 00 00 da 00 00 00 0f 01 02 00 20 00 00 00 fa 00 00 00 10 01 02 00 ... >


package.json

{
  "name": "upload",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "start": "node server.js",
    "dev": "nodemon server.js",
    "heroku-postbuild": "cd client && npm install && npm run build"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "body-parser": "^1.18.3",
    "express": "^4.16.4",
    "gridfs-stream": "^1.1.1",
    "jszip": "^3.1.5",
    "mongoose": "^5.4.2",
    "multer": "^1.4.1",
    "multer-gridfs-storage": "^3.2.3"
  }
}

------------------------------------------

server.js

const express = require('express')
const bodyParser = require('body-parser')
const multer = require('multer');
const path = require('path');
const mongoose = require('mongoose');
const jszip = require('jszip');
const gridStorage = require('multer-gridfs-storage');
const gridStream = require('gridfs-stream');

//fix finOne bug
eval(`gridStream.prototype.findOne = ${gridStream.prototype.findOne.toString().replace('nextObject', 'next')}`);

const app = express();

//Body parser Middleware
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));

//public folder
//app.use(express.static('./public'));
app.use(express.static(path.join(__dirname, 'client/build')));


const mongoURL = "mongodb://xxxxxxx@ds149984.mlab.com:49984/upload";

//gridfs multer
const storage = new gridStorage({
    url: mongoURL,
    file: (req, file) => {

        const fileInfo = {
            filename: file.originalname,
            bucketName: 'gridfs'
        }

        return fileInfo
    }
})

const gridUpload = multer({ storage });

//gridfs upload
app.post('/gridUpload', gridUpload.any(), (req, res, next) => {
    console.log(req.files)
    res.send('ok')
})

//gridfs search
app.get('/gridSearch', (req, res, next) => {
    try {
        //only download id and name, ignore actual file 
        gfs.files.find().toArray((err, files) => {

            console.log(files)
            res.send(files)
        })
    } catch (err) {
        return next(err)
    }
})


//gridfs download
app.post('/gridDownload', async (req, res, next) => {
    try {
        const lists = req.body;
        const zip = new jszip();
        let downloaded = 0;

        lists.map(async (item, index) => {

            let fileName;
            //find name by id from metadata
            await gfs.findOne({ _id: item }, (err, search) => {
                if (err) {
                    console.log(err)
                }

                console.log(search.filename)
                fileName = search.filename;
            })

            const readStream = gfs.createReadStream({ _id: item });

            let fileChunks = [];
            //store all chunks into an array
            readStream.on('data', chunk => { fileChunks.push(chunk) });

            //when all chunks of a files have been read...
            readStream.on('end', async () => {

                //assemble all chunks into a file
                const file = Buffer.concat(fileChunks);
                console.log(file);

                //store file in zip folder
                await zip.file(fileName, file, { base64: true })
                console.log(index)

                //record # of files donwloaded
                downloaded++;

                //when all downloaded files are in zip folder...
                if (downloaded === lists.length) {
                    zip.generateAsync({ type: "nodebuffer" })
                        .then(function (content) {

                            //deliver zip folder to font end
                            res.send(content);
                        })
                        .catch(err => {
                            next(err)
                        });
                }
            })
        })


    } catch (err) {
        return next(err)
    }
})

//gridfs delete
app.delete('/gridDelete', (req, res, next) => {

    try {
        const lists = req.body;
        let deleted = 0;

        lists.map((item, index) => {
            gfs.remove({ _id: item, root: 'gridfs' }, (err, gridStore) => {
                if (err) { return next(err) }

                deleted++;

                if (deleted === lists.length) {
                    res.send('deleted')
                }
            })
        })

    } catch (err) {
        return next(err);
    }
});

app.get('*', (req, res) => {
    res.sendFile(path.join(__dirname + '/client/build/index.html'));
});

const port = process.env.PORT || 5000;
app.listen(port, () => {

    mongoose.connect(mongoURL, {
        useNewUrlParser: true
    });

});

const db = mongoose.connection;

let gfs;

db.on('error', (err) => console.log(err));

db.once('open', () => {
    gfs = gridStream(db.db, mongoose.mongo);
    gfs.collection('gridfs');

    console.log('server started');
});

---------------------------------------------------

client/app.js

import React, { Component } from 'react';
import './App.css';
import ButtonUpload from './buttonUpload';
import Delete from './delete';
import Download from './download';
import { Row, Col, Checkbox, Icon } from 'antd';
import 'antd/dist/antd.css';
import axios from 'axios';
import DragUpload from './dragUpload';

class App extends Component {
  constructor(props) {
    super(props);

    this.state = {
      files: [],
      downloadCheck: [],
      deleteCheck: [],
    };
  }

  componentWillMount() {
    this.cloudSearch();
  }

  cloudSearch() {
    axios({
      method: 'get',
      url: '/gridSearch'
    })
      .then(res => {
        this.setState({ files: res.data })
      })
      .catch(err => {
        console.log(err)
      })
  }

  downloadCheck(e) {
    this.setState({ downloadCheck: e })
  }

  deleteCheck(e) {
    this.setState({ deleteCheck: e })
  }

  render() {



    return (
      <div style={{ padding: '16px' }}>
        <h1 style={{ textAlign: 'center' }}><Icon type="laptop" />  <Icon type="swap" /> <Icon type="global" /> <Icon type="swap" /> <Icon type="database" /> <Icon type="swap" /> <Icon type="global" /> <Icon type="swap" /> <Icon type="cloud" /></h1>
        <Row>
          <Col xs={12} sm={8}>
            <ButtonUpload refresh={() => this.cloudSearch()}></ButtonUpload><br />

            <DragUpload refresh={() => this.cloudSearch()}></DragUpload>

          </Col>
          <Col xs={12} sm={8}>
            <h3>Download Files from Cloud</h3>

            <Checkbox.Group onChange={(e) => this.downloadCheck(e)} style={{ width: '100%' }} >
              <Row>
                {this.state.files.map((item, index) =>
                  <Col key={index}>
                    <Checkbox value={item._id} >{item.filename}</Checkbox>
                  </Col>
                )}
              </Row>
            </Checkbox.Group><br /><br />

            <Download check={this.state.downloadCheck}></Download><br />

          </Col>
          <Col xs={12} sm={8}>
            <h3>Delete Files on Cloud</h3>

            <Checkbox.Group onChange={(e) => this.deleteCheck(e)} style={{ width: '100%' }} >
              <Row>
                {this.state.files.map((item, index) =>
                  <Col key={index}>
                    <Checkbox value={item._id} >{item.filename}</Checkbox>
                  </Col>
                )}
              </Row>
            </Checkbox.Group><br /><br />

            <Delete refresh={() => this.cloudSearch()} check={this.state.deleteCheck}></Delete>

          </Col>
        </Row>

      </div>
    );
  }
}

export default App;

--------------------------------

client/buttonUpload.js

import React, { Component } from 'react';
import './App.css';
import 'antd/dist/antd.css';
import { Button, Upload, Icon, message, Progress } from 'antd';
import axios from 'axios';

class ButtonUpload extends Component {
    constructor(props) {
        super(props);

        this.state = {
            fileList: [],
            uploading: false,
            progress: 0,
        };
    }

    handleUpload = () => {
        const { fileList } = this.state;
        const formData = new FormData();
        fileList.forEach((file) => {
            formData.append('files[]', file);
        });

        this.setState({
            uploading: true,
        });

        axios({
            method: 'post',
            url: '/gridUpload',
            data: formData,
            headers: { 'Content-Type': 'multipart/form-data', "Access-Control-Allow-Origin": "*" },
            onUploadProgress: progressEvent => {
                const p = parseInt(progressEvent.loaded / progressEvent.total * 99);
                this.setState({ progress: p });
            },
        })
            .then(res => {
                console.log(res.data);
                if (res.data === 'ok') {

                    this.setState({
                        fileList: [],
                        uploading: false,
                        progress: 100,
                    });

                    message.success('upload successful');
                    this.props.refresh();
                }
            })
            .catch((err) => {
                message.error(err.toString());
            })
    }

    render() {

        const { uploading, fileList } = this.state;
        const configs = {
            multiple: true,

            onRemove: (file) => {
                this.setState((state) => {
                    const index = state.fileList.indexOf(file);
                    const newFileList = state.fileList.slice();
                    newFileList.splice(index, 1);
                    return {
                        fileList: newFileList,
                    };
                });
            },

            beforeUpload: (file) => {
                this.setState(state => ({
                    fileList: [...state.fileList, file],
                    progress: 0,
                }));
                return false;
            },

            fileList,

        };

        return (
            <div >
                <Upload {...configs}>
                    <Button>
                        <Icon type="upload" /> Select File
          </Button>
                </Upload>
                <Button
                    type="primary"
                    onClick={this.handleUpload}
                    disabled={fileList.length === 0}
                    loading={uploading}
                    style={{ marginTop: 16 }}
                >
                    {uploading ? 'Uploading' : 'Start Upload'}
                </Button>{' '}
                <Progress type="circle" percent={this.state.progress} width={38} />
            </div>
        );
    }
}

export default ButtonUpload;

----------------------------------------------

client/dragUpload.js

import React, { Component } from 'react';
import './App.css';
import 'antd/dist/antd.css';
import { Upload, Icon, message } from 'antd';

const Dragger = Upload.Dragger;

class DragUpload extends Component {
    constructor(props) {
        super(props);

        this.state = {

        };
    }

    onChange(info) {
        const status = info.file.status;
        if (status !== 'uploading') {
            console.log(info.file, info.fileList);
        }
        if (status === 'done') {
            message.success(`${info.file.name} file uploaded successfully.`);

            this.props.refresh();

        } else if (status === 'error') {
            message.error(`${info.file.name} file upload failed.`);
        }
    }

    render() {
        const baseUrl = 'https://chuanshuoge1-gridfs.herokuapp.com' || 'http://localhost:3000';
        const url = baseUrl + '/gridUpload'


        return (
            <div style={{ paddingRight: '16px' }}>
                <Dragger multiple={true} action={url} onChange={(e) => this.onChange(e)}>
                    <p className="ant-upload-drag-icon">
                        <Icon type="inbox" />
                    </p>
                    <p className="ant-upload-text">Click or drag file to this area to upload</p>
                    <p className="ant-upload-hint">Support for a single or bulk upload. Strictly prohibit from uploading company data or other band files</p>
                </Dragger>
            </div>
        );
    }
}

export default DragUpload;

---------------------------------------

client/download.js

import React, { Component } from 'react';
import './App.css';
import 'antd/dist/antd.css';
import { Button, message, Progress } from 'antd';
import axios from 'axios';
import saveAs from 'file-saver';

class Download extends Component {
    constructor(props) {
        super(props);

        this.state = {
            progress: 0,
        };
    }


    downloadAxios = () => {
        this.setState({ progress: 0 })
        axios({
            url: '/gridDownload',
            method: 'post',
            data: this.props.check,
            responseType: 'blob', // important
            onDownloadProgress: progressEvent => {
                const p = parseInt(progressEvent.loaded / progressEvent.total * 100);
                this.setState({ progress: p });
            },
        })
            .then(res => {
                console.log(res.data)
                const name = Date.now() + '.zip'
                saveAs(res.data, name);

            })
            .catch(err => {
                message.error(err)
            })
    }

    render() {

        return (
            <div  >
                <Button type='dashed' onClick={() => { this.downloadAxios() }} disabled={this.props.check.length === 0 ? true : false}>Download zip</Button>{' '}
                <Progress type="circle" percent={this.state.progress} width={38} />
            </div>
        );
    }
}

export default Download;

-------------------------------------

client/delete.js

import React, { Component } from 'react';
import './App.css';
import 'antd/dist/antd.css';
import { Button, message } from 'antd';
import axios from 'axios';


class Delete extends Component {
    constructor(props) {
        super(props);

        this.state = {

        };
    }

    delete = () => {
        axios({
            method: 'delete',
            url: '/gridDelete',
            data: this.props.check,
        })
            .then(res => {
                message.success('deleted all')
                this.props.refresh();
            })
            .catch(err => {
                console.log(err)
            })
    }

    render() {



        return (
            <div  >
                <Button type='danger' onClick={() => { this.delete() }} disabled={this.props.check.length === 0 ? true : false}>Delete</Button>
            </div>
        );
    }
}

export default Delete;

-----------------------------------------
reference:
Posted by at

1 comment:

  1. Anonymous18 September 2020 at 10:38

    do you have any github repo for the code?

    ReplyDelete

Subscribe to: Post Comments (Atom)