0

I have node.js file that is connecting to the mongoDB and getting the number of the documents in the collection then it's running in for loop and updating each document with a value them I'm getting from an external server, the thing is that i have a lot of connections open at a time and the program is crashing, my question is how can i use 1 connection at a time and multiple connections this is my code:

var request = require('request'); var cheerio = require ('cheerio'); var fs = require('fs'); var MongoClient = require('mongodb').MongoClient; //var config = JSON.parse(fs.readFileSync("config.json")); //var host = config.host; //var port = config.port; // Variable Deceleration //var dbName = "ystocks"; var port = "27017"; var host = "localhost"; var collection = "stocks" var dbName = "yahooStocks"; function updateStock(){ // getting the number of docs in the collection getNumOfDocs("stocks", host, port, dbName, function(err, count) { if (err) { return console.log(err.message); } console.log('number of documents', count); for(i = 0 ; i <= 20; i++) { // Finding each document by _id with a callbak GetDocumentByID("stocks", host, port, dbName, i, function(err, findOne) { if (err) { return console.log(err.message); } console.log('Here is the doc that was found', findOne.ticker); var stockName = findOne.ticker; console.log("\n* stock name : " +stockName + "\n"); var yUrl = "http://finance.yahoo.com/q/ks?s=" + stockName; getStockValue(stockName, yUrl); }); // end of GetDocumentByID }//end of for loop }); // end of getNumOfDocs call }// end of update function ////passed arguments //stockName = for printing stock name to console //yUrl = the specific url for the current stock ticker function getStockValue(stockName, yUrl){ request(yUrl, stockName, function (error, response, body) { var date = setTimeToLocal(new Date(), 3); if (!error && response.statusCode == 200) { var $ = cheerio.load(body); // the keys - We get them from a certain class attribute var span = $('.time_rtq_ticker>span'); var stockValue = $(span).text(); console.log("checking the type of the stockValue " + typeof(stockValue)); //************************************* // parsing the value to int in case it was a String var parsedValue = parseFloat(stockValue); console.log("checking the type of the stockValue " + typeof(parsedValue) + " " + parsedValue); // Calling the setStockValue function which will update the stock value setStockValue(stockName, parsedValue); //writehead ??? //response.writeHead(200, {'Content-Type':'text/plain'}); console.log("Response received from -> " + yUrl); console.log(date); console.log("Stock - " + stockName + " --> " + stockValue ); //response.write is for printing in browser when we have a server running //response.write("Welcome to stockwatch\n"); //response.write("Stock - " + ticker + " current value--> " + stockValue + "\n" ); //TODO #1 //ALEX : lines 61-72 -> should be replaced with a function call that // connects to the DB and updates the specified stock //i think it sould look like this : funcName( tickerName , currenValue ){...} var json = {stockName : "", price : "", date : ""}; json.price = stockValue; json.stockName = stockName; json.date = date; var tempName = json.stockName +'.json'; fs.writeFile( tempName ,JSON.stringify(json, null, 4) , function(err) { if(!err) { console.log("File successfully written"); } });//end of filewrite //end of TODO #1 //response.end("res end"); }//end of !error && response.statusCode == 200 else if (response.statusCode == 404){ console.log("Response failed from " + yUrl + " --> error code: " + response.statusCode); }//end of statusCode == 400 });// end of request }//end of getStockValue // updating a stock with a provided ticker and value and the MongoClient // edited by Alex Brodov on : AUG 21 2014 function setStockValue(ticker, value) { // open the connection the DB server MongoClient.connect("mongodb://localhost:27017/yahooStocks", function (error, db){ if(error) throw error; var query = {'ticker' : ticker}; operator ={'$set' : {'value' : value}}; // find one document in our collection db.collection('stocks').update(query, operator, function(error, updated){ if (error) throw error; //print the result console.dir("Successfully updated " + updated + " document"); // close the DB db.close(); });// end of update }); // Connection to the DB } // end of setStockValue // Gets the local date and the desired offset time // set function setTimeToLocal(date, offset ) { // getting the local tome in millseconds var localtime = date.getTime(); // getting the local offset in millseconds var localOffset = date.getTimezoneOffset()*60000; var utc = localOffset + localtime; // Jerusalem offset // var offset = 3; // Jerusalem time in millseconds var jerusalem = utc + (offset*3600000); var d = new Date(jerusalem); console.log("Jerusalem Local Time: " + d.toLocaleString()); return d; } // end of SetTimeToLocal /** * Gets the number of documents from a collection * With a provided collection name, DB host, DB port, DB name and a callback function * Edited by Alex */ function getNumOfDocs (collectionName, host, port, dbName, callback) { var mongoConnection = MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){ if(error) return callback(err); db.collection(collectionName).count({}, function(error, numOfDocs){ if(error) return callback(err); db.close(); callback(null, numOfDocs); }); // end of count }); // end of connection to DB } // end of getNumOfDocs /** * Gets a document * With a provided collection name, DB host, DB port, DB name, the document ID and a callback * Edited by Alex Brodov on Aug 24th 2014 */ function GetDocumentByID (collectionName, host, port, dbName, docID, callback) { // open the connection to DB server MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){ if(error) return callback(error); var query = {'_id' : docID}; // find one document in our collection db.collection(collectionName).findOne(query, function(error, doc) { if (error) return callback(error); console.log(doc); // close the DB db.close(); callback(null, doc); });// end of findOne }); // Connection to the DB } var refreshInterval = setInterval(updateStock, 30000); 

1 Answer 1

2

In a MongoDB/Node application you should do the MongoClient.connect() once, at the startup, and then reuse the db object directly as much as you need. The MongoClient is in fact a connection pool and efficient.

So you should review you code, to connect only once, and it should solve the problem (faster program and less connections)

  1. remove all the mongoclientconnect, callback and db.close from your programm

  2. they add the following "around your code"

MongoClient.connect("mongodb://localhost:27017/yahooStocks", function (error, db){ if(error) throw error; // all your methods here // ... // .. var refreshInterval = setInterval(updateStock, 3000); }); 
Sign up to request clarification or add additional context in comments.

3 Comments

My problem is placing the db.close() in the right place, as you can see i have a lot of functions and a for loop that has inside findOne function, if i'm placing the db.close() in the wrong place it will not execute the whole code that is related to the MongoDB..
It looks to me that your program is a persistent program (stay up all the time). What is the issue to never close the connection until the process is stopped/killed?
I have to many connections, but if I'm using only one connection i don't know where to place db.close so all the code will be executed, i heard that there is an option to use when module or the async

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.