Homelab, Linux & ABAP (~˘▾˘)~
 

[nodejs] Parsing multipart/mixed response (containing a file stream)

Recently I had to consume an API which returned multipart/mixed data. A response looked like this:

--Boundary_0000000000001
Content-Type: application/octet-stream
Content-Disposition: attachment; filename"test.pdf"

%PDF-1.7
%�������
1 0 obj
...
%%EOF

--Boundary_0000000000001
Content-Type: application/json

{"data":[]}
--Boundary_0000000000001--

There are some node packages for parsing multipart responses, but most can only handle multipart/formData and not multipart/mixed. The most recommended package for multipart/mixed is Dicer, but to be honest, I wasn’t sure how to use it properly. Therefore I built my own parser. Luckely the user idbehold provided a function to parse a response string into an json object here. To get it working I just had to change the regex expressions in the split function. The most important step is to convert the data from the arrayBuffer to a String in binary encoding before parsing.

Also I wrote two helper functions. The first one to parse the boundary string from the Content-Type and the second one to parse the filename from the Content-Dispositon Header of your response.

module.exports = new class multipartMixedParser {

    parse(boundary, buffer) {
        const body = buffer.toString('binary') //toString encodes to utf-8 as default, this would lead to corrupted pdf's     
        return body.split(boundary).reduce((parts, part) => {
            if (part && part !== '--\r\n') {
                const [head, body] = part.trim().split(/\r\n\r\n/g)
                console.log({ body })
                parts.push({
                    body: body,
                    headers: head.split(/\r\n/g).reduce((headers, header) => {
                        const [key, value] = header.split(/:\s+/)
                        headers[key.toLowerCase()] = value
                        return headers
                    }, {})
                })
            }
            return parts
        }, [])
    }

    getBoundaryFromResponseHeaders(headers) {
        //example: multipart/mixed;boundary=Boundary_0000000000001 -> --Boundary_0000000000001
        const contentType = headers.get('content-type')
        return '--' + contentType.split("=")[1].split(";")[0]
    }

    getFileNameFromContentDisposition(cd) {
        //example: 'attachment; filename="example.pdf"' -> example.pdf
        return cd.slice(
            cd.indexOf('"') + 1,
            cd.lastIndexOf('"')
        )
    }

}

And that’s how I’m calling the API and using the multipartMixedParser Class to parse the response. The API I was using is expecting a file as formData and is also returning a file (as part of the multipart/mixed response).
It’s important to get the buffer from the response. If you would use response.getText() it would convert the data to an utf-8 encoded string which will lead to corrupted files.

const btoa = require('btoa')
const FormData = require('form-data')
const fetch = require('node-fetch')
const multipartMixedParser = require('./multipartMixedParser') 

function callAPI(file) {

        const form = new FormData()
        form.append('file', file.content, {
            contentType: file.mediaType,
            filename: file.fileName
        })

        const headers = {
            'Authorization': 'Basic ' + btoa(username + ':' + password),
            ...form.getHeaders()
        }

        const url = /my/api/path

        try {
            const response = await fetch(url, {
                method: 'POST',
                headers: headers,
                body: form
            })
            if (!response.ok) throw new Error(response.statusText)

            //parse the response
            const buffer = await response.buffer() 
            const boundary = multipartMixedParser.getBoundaryFromResponseHeaders(response.headers)

            const result = multipartMixedParser.parse(boundary, buffer)

            // in my case I only returned the file content as buffer and filename 
            return {
                fileContent: Buffer.from(result[0].body, 'binary'),
                fileName: multipartMixedParser.getFileNameFromContentDisposition(result[0].headers["content-disposition"])
            }
        } catch (err) {
            console.log("Error message: " + err.message)
        }

}

[ABAP] Convert JavaScript Timestamp to yyyy-MM-dd’T’HH:mm:ss

DATA(lv_js_timestamp) = "/Date(1615161600000)/".

"Extract /Date(1615161600000)/ to 1615161600000
FIND REGEX '([0-9]+)' IN lv_js_timestamp IGNORING CASE SUBMATCHES DATA(js_timestamp).

cl_pco_utility=>convert_java_timestamp_to_abap( EXPORTING iv_timestamp = js_timestamp
                                                IMPORTING ev_date      = DATA(lv_date)
                                                          ev_time      = DATA(lv_time) ).
"2021-03-08T00:00:00
CONVERT DATE lv_date TIME lv_time INTO TIME STAMP DATA(timestamp) TIME ZONE 'UTC'.
rv_datetime = |{ timestamp TIMESTAMP = ISO }|.

[nodejs] APIs and Microservices Projects

These are my notes while doing the course APIs and Microservices on https://www.freecodecamp.org. I highly recommend it if you prefer to try things directly rather than watching videos.


Timestamp Microservice

https://repl.it/@nocin/boilerplate-project-timestamp#server.js

app.get("/api/timestamp/", (req, res) => {
  res.json({ unix: Date.now(), utc: Date() });
});

app.get("/api/timestamp/:date?", (req, res) => {

  //utc date?
  let date = new Date(req.params.date)
  if (date != "Invalid Date") {
   res.json({unix: date.getTime(), utc: date.toUTCString()});
  }

  //unix timestamp?
  const dateInt = parseInt(req.params.date);
  date = new Date(dateInt).toUTCString();
  if (date != "Invalid Date") {
    res.json({unix: dateInt, utc: date});
  }
  
  //invalid input
  res.json({ error: date });
});


Request Header Parser Microservice

https://repl.it/@nocin/boilerplate-project-headerparser#server.js

https://www.npmjs.com/package/express-useragent
https://www.npmjs.com/package/express-request-language

var useragent = require('express-useragent');
var cookieParser = require('cookie-parser');
var requestLanguage = require('express-request-language');

// stuff...

app.use(useragent.express());
app.use(cookieParser());
app.use(requestLanguage({
  languages: ['en-US', 'zh-CN'],
  cookie: {
    name: 'language',
    options: { maxAge: 24*3600*1000 },
    url: '/languages/{language}'
  }
}));

app.get("/api/whoami", (req, res) => {
  res.json({"ipaddress": req.ip,
            "language": req.language,
            "software": req.useragent.source });
});


URL Shortener Microservice

https://repl.it/@nocin/boilerplate-project-urlshortener#server.js

require('dotenv').config();
const express = require('express');
const cors = require('cors');
const app = express();
const bodyParser = require('body-parser');
const dns = require('dns');


// Basic Configuration
const port = process.env.PORT || 3000;

app.use(cors());

app.use('/public', express.static(`${process.cwd()}/public`));

app.get('/', function(req, res) {
  res.sendFile(process.cwd() + '/views/index.html');
});

app.use(bodyParser.urlencoded({extended: false}));

let urls = [];

//POST
app.post("/api/shorturl/new", function(req, res) {
  
  const getHostnameFromRegex = (url) => {
  // run against regex
  const matches = url.match(/^https?\:\/\/([^\/?#]+)(?:[\/?#]|$)/i);
  // extract hostname (will be null if no match is found)
  return matches && matches[1];
  }

  hostname = getHostnameFromRegex(req.body.url);
  console.log("Hostname: " + hostname);

  // if no hostname found, return here
  if (!hostname) res.json({ error: 'invalid url' });

  // check if url is valid
  dns.lookup(hostname, (error, addresses) => {
    console.error(error);
    console.log(addresses);

    if (!error) {
       let newUrl = { original_url : req.body.url, short_url : urls.length + 1};
      urls.push(newUrl);
      res.json(newUrl);
    } else {
      res.json({ error: 'invalid url' });
    }

  });

});

//GET
app.get('/api/shorturl/:num', function(req, res) {

  for (let i = 0; i < urls.length; i++) {
    console.log(urls[i].original_url);
    if (urls[i].short_url == req.params.num) {
        res.redirect(urls[i].original_url);
    }
  }

});

app.listen(port, function() {
  console.log(`Listening on port ${port}`);
});


Exercise Tracker

https://repl.it/@nocin/boilerplate-project-exercisetracker#server.js

const express = require('express')
const app = express()
const cors = require('cors')
require('dotenv').config()
const bodyParser = require('body-parser');
const mongoose = require('mongoose');

app.use(cors())
app.use(express.static('public'))
app.get('/', (req, res) => {
    res.sendFile(__dirname + '/views/index.html')
});


const listener = app.listen(process.env.PORT || 3000, () => {
    console.log('Your app is listening on port ' + listener.address().port)
})


//BodyParser
app.use(bodyParser.urlencoded({ extended: false }));

//DB connect
mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: true, useUnifiedTopology: true });

const { Schema } = mongoose;

//User Schema
const userSchema = new Schema({
    username: { type: String, required: true },
});
const User = mongoose.model("User", userSchema);

//Exercise Schema
const exerciseSchema = new Schema({
    userId: Schema.Types.ObjectId,
    description: { type: String, required: true },
    duration: { type: Number, required: true },
    date: { type: Date, default: Date.now }
});
const Exercise = mongoose.model("Exercise", exerciseSchema);


//POST user to DB
app.post("/api/exercise/new-user", (req, res) => {

    let user = new User({ username: req.body.username });

    user.save((err, data) => {
        //console.log("created User: " + data);
        if (err) return console.error(err);
        res.json({ username: data.username, _id: data._id });
    });

});


//GET all users from DB
app.get("/api/exercise/users", (req, res) => {
    User.find((err, usersFound) => {
        if (err) return console.error(err);
        //console.error("users found: " + usersFound);
        res.json(usersFound);
    })
});


//POST exercise form data
app.post("/api/exercise/add", (req, res) => {

    let exercise = new Exercise({
        userId: req.body.userId,
        description: req.body.description,
        duration: req.body.duration,
        date: req.body.date ? req.body.date : Date.now()
    });

    exercise.save((err, data) => {
        //console.log("created exercise: " + data);
        if (err) return console.error(err);
        User.findById(exercise.userId, (err, userFound) => {
            if (err) return console.error(err);
            //console.log("userFound " + userFound.username); 
            res.json({
                _id: data.userId,
                username: userFound.username,
                date: data.date.toDateString(),
                duration: data.duration,
                description: data.description
            });
        });
    });
});


//GET exercise log
app.get("/api/exercise/log", (req, res) => {
    console.log(req.query.userId);
    console.log(req.query.from);
    console.log(req.query.to);
    console.log(req.query.limit);

    let userId = req.query.userId;
    let limit = Number(req.query.limit);

    //create query filter
    let filter = {};
    filter.userId = userId;

    if (req.query.from && req.query.to) {
        let fromDate = new Date(req.query.from);
        let toDate = new Date(req.query.to);
        filter.date = { $gte: fromDate, $lte: toDate };
    }

    console.log("Filter " + JSON.stringify(filter));

    const queryExercises = (done) => {
        Exercise.find(filter)
            .limit(limit)
            .exec((err, exercices) => {
                if (err) return console.error(err);
                done(exercices);
            })
    };

    const paseExercises = (exercices) => {
        let logArray = [];

        for (let i = 0; i < exercices.length; i++) {
            var obj = exercices[i];
            logArray.push({
                description: obj.description,
                duration: obj.duration,
                date: obj.date.toDateString()
            });
        }
        console.log(logArray);

        User.findById(userId, (err, userFound) => {
            if (err) return console.error(err);
            let logger = {
                _id: userId,
                username: userFound.username,
                count: logArray.length,
                log: logArray
            };
            res.json(logger);
        });
    }

    //Execute Query
    queryExercises(paseExercises);

});

File Metadata Microservice

https://repl.it/@nocin/boilerplate-project-filemetadata#server.js

https://www.npmjs.com/package/multer

var express = require('express');
var cors = require('cors');
require('dotenv').config()
var multer  = require('multer')
var upload = multer({ dest: 'uploads/' });

var app = express();

app.use(cors());
app.use('/public', express.static(process.cwd() + '/public'));

app.get('/', function (req, res) {
    res.sendFile(process.cwd() + '/views/index.html');
});


const port = process.env.PORT || 3000;
app.listen(port, function () {
  console.log('Your app is listening on port ' + port)
});


//POST 
app.post('/api/fileanalyse', upload.single('upfile'), (req, res, next) => {
  res.json({ name: req.file.originalname, type: req.file.mimetype, size: req.file.size  });
})