Get Even More Visitors To Your Blog, Upgrade To A Business Listing >>

Is this leaking memory?

Is this leaking memory?

Problem

My code is leaking memory. After a couple of hours, it fills up the entire memory and crashes. I've simplified my code here, would anybody be able to tell if this looks like it leaks? Thanks.

var request = require('request').forever(), // as per [1]
    async = require('async'),
    kue = require('kue'),
    jobs = kue.createQueue(),
    pool = { maxSockets: 1 };



function main (job, done) {
    async.series(
        [function (callback) {
            var req = request({url: job.data.URL1, pool: pool}, function (err, resp, body) {
                //stuff...
                callback(err);
            });
        },
        function (callback) {
            var req = request({url: job.data.URL2}, function (err, resp, body) {
                //stuff...
                callback(err);
            });
        }
        ],
        function (err) {
            //stuff...
            done();
        }
    );

}

jobs.process('job_name', function (job, done) {  //many jobs with 'job_name' in the queue
    main (job, done);
});

[1] https://groups.google.com/d/msg/nodejs/ZI6WnDgwwV0/sFm4QKK7ODEJ

Problem courtesy of: WindChimes

Solution

I don't think your code is to blame. I had the very same issue using kue, to be sure that I wasn't doing anything wrong I made a super simple worker like this:

var Redis       = require('redis'),
    kue         = require('kue'),
    config      = require("../../config/local.js"),
    redisClient = Redis.createClient(config.redis),
    jobs        = kue.createQueue({ redis : config.redis });

jobs.process('testjobs', function processJob(job, done, error) {
    console.log(job.data, error);
    done();
});

Running this code made me realize that is kue the one that leaks. The workaround is to use pm2, this guy will run your program and restart it if the memory is going to the roof, I'm using the JSON App Declaration to configure a maximum amount of memory allowed before restarting the process.

{
  "apps" : [
    {
      "name": "test_worker",
      "script": "test.js",
      "instances": 1,
      "max_restarts": 10,
      "max_memory_restart" : "10M",
      "ignore_watch": [
        "[\\/\\\\]\\./",
        "node_modules"
      ],
      "merge_logs": true,
      "exec_interpreter": "node",
      "exec_mode": "fork_mode"
    }
  ]
}

Hope this helps.

Solution courtesy of: Gus Ortiz

Discussion

View additional discussion.



This post first appeared on Node.js Recipes, please read the originial post: here

Share the post

Is this leaking memory?

×

Subscribe to Node.js Recipes

Get updates delivered right to your inbox!

Thank you for your subscription

×