Creating a multer storage engine with a Nextjs placeholder

A Custom Multer Storage Engine in Express with Nextjs Image placeholder

We will be creating a custom Multer Storage Engine to upload an image and a placeholder that we will use for Nextjs Image component.

We are going to use the following packages to build our application:

express: A very popular Node server.
multer: A package for extracting files from multipart/form-data requests.
jimp: An image manipulation package.
dotenv: A package for adding .env variables to process.env.
aws-sdk: A package for uploading images to s3 bucket.
concat-stream: A package for creating a writable stream that concatenates all the data from a stream and calls a callback with the result.
streamifier: A package to convert a Buffer/String into a readable stream.

Creating the Multer Storage Engine create the handler for the upload request. We are going to implement the /upload route to actually handle the upload and we will be using the Multer package for that.

// server.ts
import express, { Application } from "express";
import cors from "cors";
import dotenv from "dotenv";
import multer from "multer";
import Storage from "./helpers/storage";
import AWS from "aws-sdk";

dotenv.config();

const app: Application = express();

// Set S3 endpoint
const spacesEndpoint = new AWS.Endpoint(process.env.SPACES_BUCKET_ENDPOINT);

const s3 = new AWS.S3({
  endpoint: spacesEndpoint,
  accessKeyId: process.env.SPACES_ACCESS_KEY_ID,
  secretAccessKey: process.env.SPACES_ACCESS_SECRET_KEY,
});

app.use(cors());

app.use("/media", express.static("public"));

// setup a new instance of the AvatarStorage engine
const storage = Storage({
  s3,
  bucket: process.env.SPACES_BUCKET_NAME,
  acl: "public-read",
  threshold: 1000,
  storage: "locale",
  dir: "public",
  output: "jpg",
});

const limits = {
  files: 1, // allow only 1 file per request
  fileSize: 5 * (1024 * 1024), // 10 MB (max file size)
};

// setup multer
const upload = multer({
  storage,
  limits: limits,
});

interface CustomFileResult extends Partial<Express.Multer.File> {
  image: string;
  placeholder: string;
  bucket?: string;
}

app.post("/upload", upload.single("photo"), function (req, res) {
  const file = req.file as CustomFileResult;
  const { mimetype, originalname, image, placeholder, bucket } = file;
  res.json({ mimetype, originalname, image, placeholder, bucket });
});

const PORT = 5000;

app.listen(PORT, function () {
  console.log(`Express Server started on port ${PORT}`);
});

The filename for normal image and its placeholder takes the format [generated_filename]_placeholder.[output_extension]
and [generated_filename].[output_extension] Then the image clone and the stream are put in a batch for processing.

Creating the Multer Storage Engine

We will have to create a custom storage engine to use with Multer. Let’s create a new folder in our project root named helpers and create a new file storage.ts inside it for our custom storage engine. The file should contain the following blueprint code snippet:

// storage.ts

// Load dependencies
import { Request } from "express";
import multer from "multer";
import path from "path";
import fs from "fs";
import Jimp from "jimp";
import concat from "concat-stream";
import streamifier from "streamifier";
import slugify from "slugify";
import { customAlphabet } from "nanoid";
import type AWS from "aws-sdk";

const nanoid = customAlphabet("abcdefghijklmnopqrstuvwxyz", 10);

const PNG = "png";
const JPEG = "jpeg" || "jpg";
const typeS3 = "s3";
const typeLocal = "locale";

type nameFnType = (
  file: Express.Multer.File,
  output: typeof PNG | typeof JPEG
) => string;

type Options = {
  s3: AWS.S3 | null;
  bucket: string | null;
  acl: string;
  output?: typeof PNG | typeof JPEG;
  storage?: typeof typeS3 | typeof typeLocal;
  quality?: number;
  threshold?: number | null;
  placeholderSize?: number;
  dir: string;
};

interface CustomFileResult extends Partial<Express.Multer.File> {
  image: string;
  placeholder: string;
  bucket?: string;
}

class CustomStorageEngine implements multer.StorageEngine {
  defaultOptions: Options;
  options: Options;
  placeholder: string;
  image: string;
  filepath: string;
  fileSharedName: string;

  constructor(opts: Options) {
    this.options = opts || undefined;
    this.placeholder;
    this.image;
    this.filepath;
    this.fileSharedName;

    // fallback for options
    this.defaultOptions = {
      s3: null,
      bucket: null,
      acl: null,
      dir: null,
      output: "png",
      storage: "locale",
      quality: 90,
      threshold: null,
      placeholderSize: 26,
    };

    // You can add more options
    const allowedOutputFormats = ["jpg", "jpeg", "png"];

    if (this.options.dir && !fs.existsSync(this.options.dir)) {
      fs.mkdirSync(this.options.dir);
    }

    // If the option value is undefined or null it will fall back to the default option
    const allowedOutput = allowedOutputFormats?.includes(
      String(this.options.output ?? this.defaultOptions.output)?.toLowerCase()
    );

    if (!allowedOutput) throw new Error("Output is not allowed");
    if (!this.options.dir) throw new Error("Expected dir to be string");

    switch (typeof opts.s3) {
      case "object":
        if (!this.options.acl) throw new Error("Expected acl to be string");
        if (!this.options.bucket)
          throw new Error("Expected bucket to be string");
        break;
      default:
        if (this.options.storage === typeS3)
          throw new TypeError("Expected opts.s3 to be object");
        break;
    }
  }

  // Create a file path based on date
  private getPath = () => {
    const newDate = new Date();
    const Month = newDate.getMonth() + 1;
    const Year = newDate.getFullYear();

    const dir = this.options.dir ?? this.defaultOptions.dir;
    const dirPath = `${Year}/${Month}`;

    const filePath = path.resolve(`${dir}/${Year}/${Month}`);

    if (!fs.existsSync(filePath)) {
      fs.mkdirSync(filePath, { recursive: true });
    }

    return { dirPath, filePath };
  };

  private _getMime = () => {
    // resolve the Jimp output mime type
    const output = this.options.output ?? this.defaultOptions.output;
    switch (output) {
      case "jpg":
      case "jpeg":
        return Jimp.MIME_JPEG;
      case "png":
        return Jimp.MIME_PNG;
      default:
        return Jimp.MIME_PNG;
    }
  };

  // return as filename with the output extension
  private generateFilename: nameFnType = (
    file: Express.Multer.File,
    output: typeof PNG | typeof JPEG
  ) => {
    const newDate = new Date();
    const DateAsInt = Math.round(newDate.getTime() / 1000); // in seconds

    // trim a file extension from image and remove any possible dots in file name
    const filename = file?.originalname
      ?.replace(/\.[^/.]+$/, "")
      ?.replace(/\./g, "");

    if (filename) {
      const cleanedTitle = slugify(
        filename.replace(/[^A-Za-z0-9\s!?]/g, "").trim(),
        "_"
      );
      return (
        (cleanedTitle + "__" + DateAsInt + "_" + nanoid())?.toLowerCase() +
        "." +
        output
      );
    }
    return (DateAsInt + "_" + nanoid())?.toLowerCase() + "." + output;
  };

  _createOutputStream = (
    filepath: string,
    cb: (error?: Error | null, info?: CustomFileResult) => void
  ) => {
    const output = fs.createWriteStream(filepath);

    // set callback fn as handler for the error event
    output.on("error", cb);

    // set handler for the finish event
    output.on("finish", () => {
      cb(null, {
        destination: this.filepath,
        mimetype: this._getMime(),
        image: this.image,
        placeholder: this.placeholder,
      });
    });

    // return the output stream
    return output;
  };

  private writeImage = (
    filepath: string,
    image: Jimp,
    cb: (error?: Error, info?: CustomFileResult) => void,
    { isPlaceholder }: { isPlaceholder: boolean }
  ) => {
    try {
      // get the buffer of the Jimp image using the output mime type
      image.getBuffer(this._getMime(), (err, buffer) => {
        const storage = this.options.storage ?? this.defaultOptions.storage;

        switch (storage) {
          case typeLocal: {
            // create a writable stream for it
            const outputStream = this._createOutputStream(filepath, cb);
            // create a read stream from the buffer and pipe it to the output stream
            streamifier.createReadStream(buffer).pipe(outputStream);
            break;
          }
          case typeS3:
            this.options.s3.upload(
              {
                Bucket: this.options.bucket,
                Key: isPlaceholder ? this.placeholder : this.image,
                Body: streamifier.createReadStream(buffer),
                ACL: this.options.acl,
                ContentType: "application/octet-stream",
              },
              (error, response) => {
                if (!error) {
                  cb(null, {
                    destination: this.filepath,
                    mimetype: this._getMime(),
                    image: this.image,
                    placeholder: this.placeholder,
                    bucket: response.Bucket,
                  });
                } else {
                  cb(error);
                }
              }
            );
            break;
          default:
            break;
        }
      });
    } catch (error) {
      console.log("error :>", error);
    }
  };

  _processImage = (
    image: Jimp,
    cb: (error?: Error, info?: CustomFileResult) => void,
    file: Express.Multer.File
  ) => {
    // Get options
    const output = this.options.output ?? this.defaultOptions.output;
    const quality = this.options.quality ?? this.defaultOptions.quality;
    const threshold = this.options.threshold ?? this.defaultOptions.threshold;
    const placeholderSize =
      this.options.placeholderSize ?? this.defaultOptions.placeholderSize;

    const filename = this.generateFilename(file, output);
    this.fileSharedName = filename;

    // create a clone of the Jimp image
    let clone = image.clone();

    // Auto scale the image dimensions to fit the threshold requirement
    if (threshold) {
      clone = clone.resize(threshold, Jimp.AUTO);
    }

    // Set the image output quality
    clone = clone.quality(quality);

    const filenameSplit = filename.split(".");
    const _filename = filenameSplit[0];
    const _output = filenameSplit[1];

    const { filePath, dirPath } = this.getPath();

    this.filepath = filePath;

    // Original image processing
    const originalImage = clone.clone();
    const originalFilename = _filename + "." + _output;
    // Set original image upload path
    this.image = `${dirPath}/${originalFilename}`;
    // create the complete filepath
    const originalFilepath = path.join(this.filepath, originalFilename);
    this.writeImage(originalFilepath, originalImage, cb, {
      isPlaceholder: false,
    });

    // Placeholder image processing
    const placeholderImage = clone.resize(placeholderSize, Jimp.AUTO);
    const placeholderFilename = _filename + "_" + "placeholder" + "." + _output;
    // Set placeholder image upload path
    this.placeholder = `${dirPath}/${placeholderFilename}`;
    // create the complete filepath
    const placeholderFilepath = path.join(this.filepath, placeholderFilename);
    this.writeImage(placeholderFilepath, placeholderImage, cb, {
      isPlaceholder: true,
    });
  };

  _handleFile = (
    req: Request,
    file: Express.Multer.File,
    cb: (error?: Error | null, info?: CustomFileResult) => void
  ): void => {
    // create a writable stream using concat-stream that will
    // concatenate all the buffers written to it and pass the
    // complete buffer to a callback fn
    const fileManipulate = concat((imageData) => {
      // read the image buffer with Jimp
      // returns a promise
      Jimp.read(imageData)
        .then((image) => {
          // process the Jimp image buffer
          this._processImage(image, cb, file);
        })
        .catch(cb);
    });

    // write the uploaded file buffer to the fileManipulate stream
    file.stream.pipe(fileManipulate);
  };

  _removeFile = (
    _req: Request,
    file: Express.Multer.File & { name: string },
    cb: (error: Error | null) => void
  ): void => {
    if (file.path) {
      fs.unlink(file.path, cb);
    }
    return;
  };
}

export default (opts: Options) => {
  return new CustomStorageEngine(opts);
};

Nextjs Images placeholder

After we upload our image we received a response with the image path and it’s placeholder.
Let’s create a hook useGetDataUrl.ts to Fetch blob and convert the placeholder to base64.

import { useEffect, useState } from "react";

export function useGetDataUrl(customPlaceholder: string) {
  const [Base64Placeholder, setBase64Placeholder] = useState<string>(
    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8+utrPQAJNQNlcqdyCgAAAABJRU5ErkJggg=="
  );

  useEffect(() => {
    async function toBase64() {
      try {
        const data = await fetch(customPlaceholder);
        const blob = await data.blob();

        // eslint-disable-next-line no-undef
        return await new Promise<string>((resolve) => {
          const reader = new window.FileReader();
          reader.readAsDataURL(blob);
          reader.onloadend = () => {
            const base64data = reader.result as string;
            return resolve(base64data);
          };
        })
          .then((res: string) => {
            setBase64Placeholder(res);
            return res;
          })
          .catch((error) => {
            console.log("error :>", error);
          });
      } catch (error) {
        console.log("error :>", error);
      }
    }

    if (customPlaceholder) {
      toBase64();
    }
  }, [customPlaceholder]);

  return Base64Placeholder;
}

GitHub

View Github

Creating a multer storage engine with a Nextjs placeholder

A Custom Multer Storage Engine in Express with Nextjs Image placeholder

Creating the Multer Storage Engine

Nextjs Images placeholder

GitHub

A simple, ok-sized tooltip for React with minimal boilerplate and a really nice feel

A lottery simulator of the Loto-Quebec 6/49 lottery game Built With React.js

A Custom Multer Storage Engine in Express with Nextjs Image placeholder

Creating the Multer Storage Engine

Nextjs Images placeholder

GitHub

A simple, ok-sized tooltip for React with minimal boilerplate and a really nice feel

A lottery simulator of the Loto-Quebec 6/49 lottery game Built With React.js

You might also like...

Subscribe to React.js Examples