I am trying to detect the content/mime type of a given Buffer/string. In other words, I am trying to guess what on earth is a given string.
My approach is somewhat brute-force:
/**
* Check if valid url encoded.
* For now I set the value as true
* @param {String} body
* @returns {Boolean}
*/
const isValidFormUrlEncoded = (body) => {
return /^(?:(?:\w+)(?:\[(?:\d*|'[^']*')\])*=[\w\-\+\.%]*(?:&|$))+$/.test(body);
}
/**
* Detects mime type and file extention from request body
* @param {String} body request Body
* @param {Function} callback (err,mime,extention,dataBuffer)
*/
const detectBodyMime = (body,callback) => {
let buffer = isBase64(body)?Buffer.from(body,'base64'):Buffer.from(body);
const magic = new mmm.Magic(mmm.MAGIC_MIME_TYPE);
magic.detect(buffer, function(err, result) {
if (err) { return callback(err); }
if(result == 'text/plain') {
let text = buffer.toString();
if (text.codePointAt(0) === 0xFEFF) { // UTF8 BOM
text = text.substring(1);
}
try {
JSON.parse(text);
return callback(null,'application/json','json',buffer);
}catch(e){
// Do nothing keep it silent we need to just verify that content is Json
}
if(isValidFormUrlEncoded(text)){
return callback(null,'application/x-www-form-urlencoded',null,buffer);
}
if(text.indexOf('Content-Disposition: form-data; name="')>1){
return callback(null,'multipart/form-data',null,buffer);
}
try {
yaml.load(text);
return callback(null,'text/yaml','yml',buffer);
} catch (e) {
//
}
}
return callback(null,result,mime.extension(result),buffer);
});
}
I made a dummy usage stript:
const {detectBodyMime} = require('../src/common/http_utils');
const yaml = `
# it is used for development and contributions
# not intented to use for production
version: "3"
services:
manipulator:
build:
context: .
dockerfile: DockerfileDev
networks:
frontend:
ipv4_address: 172.21.0.2
volumes:
- .:/home/node/app
- ./config/config.json:/etc/http_manipulator/config.json
- ./config/cert.conf:/etc/http_manipulator/cert.conf
- ./config_vol:/etc/http_manipulator
# manipulator_prod:
# build:
# context: .
# dockerfile: Dockerfile
# networks:
# frontend:
# ipv4_address: 172.21.0.3
# volumes:
# - ./config_vol_prod:/etc/http_manipulator
networks:
frontend:
ipam:
config:
- subnet: 172.21.0.0/24
`;
detectBodyMime(yaml,(err,mime,extention,buffer) => {
console.log("3",mime,extention);
});
const json=`
{
"name":"hello",
"val":"json"
}
`
detectBodyMime(json,(err,mime,extention,buffer) => {
console.log("2",mime,extention);
});
const multipart=`
--------------------------ca72190f498016c6
Content-Disposition: form-data; name="key1"
value1
--------------------------ca72190f498016c6
Content-Disposition: form-data; name="key2"
value2
--------------------------ca72190f498016c6
Content-Disposition: form-data; name="myfile"; filename="exclude.txt"
Content-Type: text/plain
pleas=help&me=plz&var[]=true&var[2]=false&var['blahblah']=ipsum
--------------------------ca72190f498016c6--
`
detectBodyMime(multipart,(err,mime,extention,buffer) => {
console.log("1",mime,extention);
});
const urlencodedForm=`pleas=help&me=plz&var[]=true&var[2]=false&var['blahblah']=ipsum`;
detectBodyMime(urlencodedForm,(err,mime,extention)=>{
console.log("5",mime,extention);
});
const urlencodedForms=`pleas=Content-Disposition%3A%20form-data%3B%20name%3D%22myfile%22%3B%20filename%3D%22exclude.txt%22&me=plz&var[]=true&var[2]=false&var['blahblah']=ipsum`;
detectBodyMime(urlencodedForms,(err,mime,extention)=>{
console.log("6",mime,extention);
});
And tested as:
const fs = require('node:fs');
const path = require('path');
const {detectBodyMime} = require('../../../src/common/http_utils.js');
const fileDataPath = path.resolve(__dirname,'__testdata__');
test("detectsBase64EncodedImage",(done)=>{
const imgdata = path.resolve(fileDataPath,"img.jpeg.base64");
const content = fs.readFileSync(imgdata);
detectBodyMime(content.toString(),(err,mime,extention)=>{
expect(mime).toBe('image/jpeg');
expect(extention).toBe('jpeg');
expect(err).toBe(null);
done();
});
});
test("detectsBase64EncodedImagePng",(done)=>{
const imgdata = path.resolve(fileDataPath,"img.png.base64");
const content = fs.readFileSync(imgdata);
detectBodyMime(content.toString(),(err,mime,extention)=>{
expect(mime).toBe('image/png');
expect(extention).toBe('png');
expect(err).toBe(null);
done();
});
});
test("detectsBase64EncodedJson",(done)=>{
const content = Buffer.from("{\"param\":1}", 'utf8');
detectBodyMime(content.toString('base64'),(err,mime,extention)=>{
expect(mime).toBe('application/json');
expect(extention).toBe('json');
expect(err).toBe(null);
done();
});
});
test("detectsJson",(done)=>{
const content = "{\"param\":1}";
detectBodyMime(content,(err,mime,extention)=>{
expect(mime).toBe('application/json');
expect(extention).toBe('json');
expect(err).toBe(null);
done();
});
});
test("detectsXml",(done)=>{
const content = `<?xml version="1.0" encoding="UTF-8"?>
<note>
<to>Tove</to>
<from>Jani</from>
<heading>Reminder</heading>
<body>Don't forget me this weekend!</body>
</note>
`;
detectBodyMime(content,(err,mime,extention)=>{
expect(['application/xml','text/xml']).toContain(mime);
expect(extention).toBe('xml');
expect(err).toBe(null);
done();
});
})
test("detectsXmlBase64 encoded",(done)=>{
const content = `<?xml version="1.0" encoding="UTF-8"?>
<note>
<to>Tove</to>
<from>Jani</from>
<heading>Reminder</heading>
<body>Don't forget me this weekend!</body>
</note>
`;
const base64EncodedXml = Buffer.from(content).toString('base64');
detectBodyMime(base64EncodedXml,(err,mime,extention)=>{
expect(['application/xml','text/xml']).toContain(mime);
expect(extention).toBe('xml');
expect(err).toBe(null);
done();
});
})
test("form url encoded",(done)=>{
const content = `val=21221`;
detectBodyMime(content,(err,mime,extention)=>{
expect('application/x-www-form-urlencoded').toEqual(mime);
expect(extention).toBe(null);
expect(err).toBe(null);
done();
});
});
test("form url encoded multiple",(done)=>{
const content = `pleas=help&me=plz&var[]=true&var[2]=false&var['blahblah']=ipsum`;
detectBodyMime(content,(err,mime,extention)=>{
expect('application/x-www-form-urlencoded').toEqual(mime);
expect(extention).toBe(null);
expect(err).toBe(null);
done();
});
});
test("form url encoded multiple array",()=>{
const content=`pleas=ContentDisposition%3A%20formdata%3B%20name%3D%22myfile%22%3B%20filename%3D%22exclude.txt%22&me=plz&var[]=true&var[2]=false&var['blahblah']=ipsum`
detectBodyMime(content,(err,mime,extention)=>{
expect('application/x-www-form-urlencoded').toEqual(mime);
expect(extention).toBe(null);
expect(err).toBe(null);
done();
});
})
test("wrong formurl encoded data to be detected ad text/plain",(done)=>{
const content = `dawdwqeewqwqeewqwqewqeqw&me=plz&*3032-vcsjmvar[]=true&var[2]=false&var['blahblah']=ipsum`;
detectBodyMime(content,(err,mime,extention)=>{
expect('text/plain').toEqual(mime);
expect(extention).toBe('txt');
expect(err).toBe(null);
done();
});
});
test("random text to be detected ad text/plain",(done)=>{
const content = `bxuiqwbuwqpniqbnxiqwpxbipwqxbiwqpxbqwipbqip`;
detectBodyMime(content,(err,mime,extention)=>{
expect('text/plain').toEqual(mime);
expect(extention).toBe('txt');
expect(err).toBe(null);
done();
});
});
test("base64 text to be detected ad text/plain",(done)=>{
const content = `bxuiqwbuwqpniqbnxiqwpxbipwqxbiwqpxbqwipbqip`;
const contentBase64 = Buffer.from(content).toString('base64');
detectBodyMime(contentBase64,(err,mime,extention)=>{
expect('text/plain').toEqual(mime);
expect(extention).toBe('txt');
expect(err).toBe(null);
done();
});
});
Another simple usage example is using connect:
const connect = require('connect');
const {detectBodyMime} = require('../src/common/http_utils');
const app = connect();
app.use(function(req,res,next){
req.id = 1;
var body = [];
req.on('data',(data) => body.push(data) );
req.on('end',() => {
body = Buffer.concat(body).toString();
detectBodyMime(body,(err,mime,extention,buffer)=>{
// Here I can either log, analyze or check for mime
res.setHeader("Content-Type",'text/plain');
res.writeHead(200,{'X-val':3});
res.end("MIME: "+mime+"\n"+"Suggested file extention: "+extention+"\n");
});
});
});
// app.use(function(req,res){
// console.log("id:",req.id);
// res.setHeader("Content-Type",'text/plain');
// res.writeHead(200,{'X-val':3});
// res.end("jhello");
// });
app.listen(8090)
The idea is to have a generic http body content-type analysis tool. The goal is to use it in my project: https://github.com/pc-magas/http-manipulator
One of the goals of the project is to have an http analyser, that will help me distinguish what requests perform once I develop, analyze or reverse engineer web applications.
Sometimes I need to verify if content-type is mismatched. Many apps may not follow the Http guidelines and post whatever developer has thought.
Algorithmic approach
The idea behind this is to check whether the body matches a given content type of check for magic number; then for specific detected types, I perform further checks.
Current use is in this incoming request logger
