Jimp allocates 200 MB memory on reading a 3.3 MB jpg

[ahaus]

When I try to read a 3.3 MB sized jpeg image jimp allocates around 200 MB local memory (which is a lot). The main problem however is that the memory is not deallocated.
The nodejs process got killed on reading about 5 images.
I am only using Jimp.read(filename).then(...).catch(...).

[oliver-moran]

Thanks for reporting this. Could you post a test case and I'll try and reproduce?

[LeZuse]

Also having problems with high memory usage. We are using Jimp as a dependency of https://github.com/haydenbleasel/favicons This is a dealbreaker for us.

[oliver-moran]

Can you try this again on the latest release. Can't reproduce using a 10MB file and the following test case:

var i = 0;

function loop(){
    console.log(i++);
    Jimp.read("large.jpg").then(function(image) {
        loop();
    }).catch(function (err) {
        console.log(err);
    });
}

loop();

Running on Mac.

[ahaus]

I used a modified version of your loop to check the memory usage. I added the output showing memory consumption. The image i am using is a 3.3 MB Jpeg.

I used the latest version 0.2.27 running with node 5.11 on Mac OS X.

'use strict';

var fs      = require('fs');
var path    = require('path');
var Jimp    = require('jimp');

console.log("Start jimp-test");

var INTERVAL = 100;
var MAX_COUNT = 20;

// Test images
var image = "./data/large-02.jpg"; // 3.3 MB jpeg
//var image = "./data/large-01.png"; // 2.3 MB png
//var image = "./data/medium-01.png"; // 500kb png


// show memory usage periodically, period is set by const INTERVAL
function showMemoryUsage() {
    console.log("Process: %s - %s MB ", new Date(), process.memoryUsage().rss / 1048576, process.memoryUsage());
}

var i = 0;

// loop for reading the same image until process will be killed/exited

function loop(filename, cb) {
    console.log('Loop iteration: %s', i++);
    Jimp.read(filename).then(function(image) {
        loop(filename);
    }).catch(function (err) {
        console.log(err);
    });
}

// show memory usage after processes is spawned
showMemoryUsage();

// check if allocated memory got freed
setInterval(showMemoryUsage, INTERVAL);

// read images
loop(image);

Output for memory usage:

Andreas-Hausleitners-MacBook-Pro:test andreas$ node jimp-test2.js 
Start jimp-test
Process: Wed Aug 31 2016 16:57:36 GMT+0200 (CEST) - 40.375 MB  { rss: 42336256, heapTotal: 31246112, heapUsed: 15693368 }
Loop iteration: 0
Loop iteration: 1
Process: Wed Aug 31 2016 16:57:37 GMT+0200 (CEST) - 196.7265625 MB  { rss: 206282752, heapTotal: 52916768, heapUsed: 18959488 }
Loop iteration: 2
Process: Wed Aug 31 2016 16:57:38 GMT+0200 (CEST) - 298.1953125 MB  { rss: 312680448, heapTotal: 76639264, heapUsed: 51481064 }
Loop iteration: 3
Process: Wed Aug 31 2016 16:57:40 GMT+0200 (CEST) - 433.83984375 MB  { rss: 454914048, heapTotal: 112757024, heapUsed: 85917080 }
Loop iteration: 4
Process: Wed Aug 31 2016 16:57:41 GMT+0200 (CEST) - 446.1328125 MB  { rss: 467804160, heapTotal: 83862816, heapUsed: 49162256 }
Loop iteration: 5
Process: Wed Aug 31 2016 16:57:42 GMT+0200 (CEST) - 556.375 MB  { rss: 583401472, heapTotal: 108629280, heapUsed: 85251936 }
Loop iteration: 6
Process: Wed Aug 31 2016 16:57:43 GMT+0200 (CEST) - 449.12890625 MB  { rss: 470945792, heapTotal: 81786912, heapUsed: 49112912 }
Loop iteration: 7
Process: Wed Aug 31 2016 16:57:44 GMT+0200 (CEST) - 443.4921875 MB  { rss: 465035264, heapTotal: 72523552, heapUsed: 49439472 }
Loop iteration: 8
Process: Wed Aug 31 2016 16:57:46 GMT+0200 (CEST) - 441.19921875 MB  { rss: 462630912, heapTotal: 70459680, heapUsed: 49200456 }
Loop iteration: 9
Process: Wed Aug 31 2016 16:57:47 GMT+0200 (CEST) - 494.5 MB  { rss: 518520832, heapTotal: 114832928, heapUsed: 84796232 }
Loop iteration: 10
Process: Wed Aug 31 2016 16:57:48 GMT+0200 (CEST) - 585.9765625 MB  { rss: 614440960, heapTotal: 144759072, heapUsed: 121414592 }
Loop iteration: 11
Process: Wed Aug 31 2016 16:57:49 GMT+0200 (CEST) - 569.96875 MB  { rss: 597655552, heapTotal: 109673248, heapUsed: 84869800 }
Loop iteration: 12
Process: Wed Aug 31 2016 16:57:51 GMT+0200 (CEST) - 535.70703125 MB  { rss: 561729536, heapTotal: 93162272, heapUsed: 49247288 }
Loop iteration: 13
Process: Wed Aug 31 2016 16:57:53 GMT+0200 (CEST) - 565.28125 MB  { rss: 592740352, heapTotal: 109673248, heapUsed: 85073568 }
Loop iteration: 14
Process: Wed Aug 31 2016 16:57:56 GMT+0200 (CEST) - 512.27734375 MB  { rss: 537161728, heapTotal: 114832928, heapUsed: 87756488 }
Loop iteration: 15
Process: Wed Aug 31 2016 16:57:58 GMT+0200 (CEST) - 608.34765625 MB  { rss: 637898752, heapTotal: 149918752, heapUsed: 120869608 }
Loop iteration: 16
Process: Wed Aug 31 2016 16:58:01 GMT+0200 (CEST) - 573.22265625 MB  { rss: 601067520, heapTotal: 114832928, heapUsed: 87769336 }
Loop iteration: 17
Process: Wed Aug 31 2016 16:58:04 GMT+0200 (CEST) - 689.06640625 MB  { rss: 722538496, heapTotal: 149918752, heapUsed: 120902528 }
Loop iteration: 18
Process: Wed Aug 31 2016 16:58:06 GMT+0200 (CEST) - 617.09765625 MB  { rss: 647073792, heapTotal: 114832928, heapUsed: 87784312 }
Loop iteration: 19
Process: Wed Aug 31 2016 16:58:09 GMT+0200 (CEST) - 696.62890625 MB  { rss: 730468352, heapTotal: 149918752, heapUsed: 120905560 }
Loop iteration: 20
Process: Wed Aug 31 2016 16:58:11 GMT+0200 (CEST) - 617.7578125 MB  { rss: 647766016, heapTotal: 114832928, heapUsed: 87797888 }
Loop iteration: 21
Process: Wed Aug 31 2016 16:58:14 GMT+0200 (CEST) - 700.00390625 MB  { rss: 734007296, heapTotal: 149918752, heapUsed: 120914616 }
Loop iteration: 22
Process: Wed Aug 31 2016 16:58:17 GMT+0200 (CEST) - 616.09765625 MB  { rss: 646025216, heapTotal: 114832928, heapUsed: 87788648 }
Loop iteration: 23
Process: Wed Aug 31 2016 16:58:19 GMT+0200 (CEST) - 693.984375 MB  { rss: 727695360, heapTotal: 149918752, heapUsed: 120870816 }
Loop iteration: 24
Process: Wed Aug 31 2016 16:58:22 GMT+0200 (CEST) - 610.96875 MB  { rss: 640647168, heapTotal: 114832928, heapUsed: 88223232 }
Loop iteration: 25
Process: Wed Aug 31 2016 16:58:25 GMT+0200 (CEST) - 525.3046875 MB  { rss: 550821888, heapTotal: 81810976, heapUsed: 49201992 }
Loop iteration: 26
Process: Wed Aug 31 2016 16:58:27 GMT+0200 (CEST) - 506.25390625 MB  { rss: 530845696, heapTotal: 73555488, heapUsed: 51376456 }
Loop iteration: 27
Process: Wed Aug 31 2016 16:58:30 GMT+0200 (CEST) - 553.28125 MB  { rss: 580157440, heapTotal: 113800992, heapUsed: 87766136 }
Loop iteration: 28
Process: Wed Aug 31 2016 16:58:33 GMT+0200 (CEST) - 638.06640625 MB  { rss: 669061120, heapTotal: 149918752, heapUsed: 120991856 }
Loop iteration: 29
Process: Wed Aug 31 2016 16:58:35 GMT+0200 (CEST) - 628.60546875 MB  { rss: 659140608, heapTotal: 114832928, heapUsed: 87799544 }
Loop iteration: 30
Process: Wed Aug 31 2016 16:58:38 GMT+0200 (CEST) - 702.32421875 MB  { rss: 736440320, heapTotal: 148886816, heapUsed: 120923632 }

[siobhandougall]

Another data point: I'm hitting this on both OS X and Linux, using both the promises-based and callback-based API.

ETA: Also when using var img = new Jimp(path);. I've tried multiple Jimp versions, all the way back to 0.1.0, and I'm still hitting it.

[Streemo]

Is this still happening?

[haydenbleasel]

@Streemo Yep.

[iwsfg]

Well, I cannot confirm this either. With Node 6.2.0 running at Windows 8 I see this. Heap grows by 20-30 MB every time I read PNG image (974x974 px, 30 KB), which is acceptable in my opinion. Calculator says that for storing 1500x1500 bitmap with 32 bpp I would need only 8.5 MB however, but whatever, could be garbage from decoding phase. GC is kicking in from time to time and does its job. Here is the image I was using if somebody needs it

Also having problems with high memory usage. We are using Jimp as a dependency of https://github.com/haydenbleasel/favicons This is a dealbreaker for us.

@LeZuse, in your case high memory usage is actually favicons' fault. It's reading the same image from fs multiple times in parallel for every icon it's generating. If icons were generated in sequence that would not be a problem.

I'm working quietly on a series of changes to that package (favicons) which fixes that among many other things. I was able to reduce heap size down to ~300 MB instead of 1.3 GB that it was using before by generating only one icon at the time (this also includes icons that's already generated and stored in memory), so in my case memory usage seems reasonable. Hopefully I'll finish it by the end of the month (also waiting on #159 to get merged)

---

UPD: actually, yes, can confirm. Tried bigger image (JPEG, 3456x2304px, 611KB) and it used about 320 MB every time. What's interesting, same image saved as 3MB PNG needed only 220 MB for a copy. And calculator says we need only 30MB buffer. (Tests performed with latest version published on npm)
https://gist.github.com/Iwasawafag/3033e0df86a1bd23cd8a4c5ef66602da

[haydenbleasel]

@iwasawafag What would the most performant process be? I can read the file once and use the buffer multiple times but running everything in sequence would take forever.

[iwsfg]

@haydenbleasel No, it wont. Everything Jimp or underlying modules (like png-js or jpegjs) do takes time to process and isn't asynchronous (except reading/writing from fs). So it doesn't really matter if you start generating icons in parallel or in sequence - either way we are waiting untill we're done iterating over pixels of the one image untill we can start iterating pixels of the other. On the other hand when V8 sees high memory usage it's trying to clear garbage, but there's nothing to clear, so I would speculate that we're actually wasting some processing time here while GC runs.

On a dirt cheap laptop sequential generation was only 2-3 seconds slower (~14 seconds instead of 16-17) and I would say it's probably caused by reading from fs. But memory usage was 70% lower. I'd say worth it. Keeping copy of the image in-memory and cloning it would definitely make things faster because decoding PNGs is expensive task and it will be run 40 times less often

[iwsfg]

It seems that root of the problem is somewhere in 'pngjs' and 'jpeg-js' modules. I tried to repeat the test using these modules directly and got very simillar numbers.

I then found another JPEG decoder from Mozilla's PNGJS, which is published on Bower, but not on NPM for some reason. Code looks nearly identical, but needs about 4.5 times less memory.

#	pngjs	jpeg-js	jpgjs
1	149.265625 MB	316.2421875 MB	69.37109375 MB
2	219.859375 MB	584.2109375 MB	115.16015625 MB
3	376.56640625 MB	364.39453125 MB	85.02734375 MB
4	340.45703125 MB	364.73828125 MB	130.69921875 MB
5	338.9609375 MB	606.28515625 MB	161.6796875 MB
6	327.7109375 MB	650.3515625 MB	207.3515625 MB
7	326.20703125 MB	575.26953125 MB	253.07421875 MB
8	321.609375 MB	363.11328125 MB	161.38671875 MB
9	312.109375 MB	368.37890625 MB	207.0625 MB

Still not a 30 MB though

[santiagovdk]

Having the same problem here but much worse, with a 18.7mb and 5751×3347px memory goes from ~60mb to over ~630mb