Last active
May 14, 2022 01:06
-
-
Save dayitv89/fcd7cac4762fe28f1d719800a2f86015 to your computer and use it in GitHub Desktop.
Transform mysql 8 data into mongodb;
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"dependencies": { | |
"mongodb": "^4.6.0", | |
"mysql2": "^2.3.3" | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var mysql = require('mysql2'); | |
var MongoClient = require('mongodb').MongoClient; | |
var mysql_config = { | |
host: 'localhost', | |
user: 'root', | |
password: 'root', | |
port: 3306, | |
database: 'mysql_db_name', | |
}; | |
var mongo_uri = 'mongodb://root:root@localhost:27017/?authSource=admin'; | |
var mongo_db_name = 'mongo_db_name'; | |
function getMysqlTables(mysqlConnection, callback) { | |
mysqlConnection.query( | |
`SELECT table_name FROM information_schema.tables WHERE table_schema = '${mysql_config.database}' ORDER BY table_name`, | |
function (error, results, fields) { | |
if (error) { | |
callback(error); | |
} else { | |
var tables = results.map((r) => r.TABLE_NAME); | |
console.log('total mysql tables:', tables.length); | |
callback(null, tables); | |
} | |
}, | |
); | |
} | |
function tableToCollection(mysqlConnection, tableName, mongoCollection, callback) { | |
var sql = 'SELECT * FROM `' + tableName + '`;'; | |
mysqlConnection.query(sql, function (error, results, fields) { | |
console.log('loading data from mysql', tableName, 'result:', !error); | |
if (error) { | |
callback(error); | |
} else { | |
console.log('inserting:', tableName, 'records:', results.length); | |
if (results.length == 0) { | |
results = [ | |
fields.reduce((acc, f) => { | |
acc[f.name] = 'BLANK'; | |
return acc; | |
}, {}), | |
]; | |
} | |
mongoCollection.insertMany(results, {}, function (error) { | |
if (error) { | |
callback(error); | |
} else { | |
callback(null); | |
} | |
}); | |
} | |
}); | |
} | |
MongoClient.connect(mongo_uri, async function (error, client) { | |
if (error) throw error; | |
var db = client.db(mongo_db_name); | |
var MysqlCon = mysql.createConnection(mysql_config); | |
MysqlCon.connect(); | |
var jobs = 0; | |
getMysqlTables(MysqlCon, function (error, tables) { | |
tables.forEach(function (table) { | |
var collection = db.collection(table); | |
++jobs; | |
tableToCollection(MysqlCon, table, collection, function (error) { | |
if (error) throw error; | |
--jobs; | |
}); | |
}); | |
}); | |
console.log('remaining:', jobs); | |
var interval = setInterval(function () { | |
console.log('remaining:', jobs); | |
if (jobs <= 0) { | |
clearInterval(interval); | |
client.close(); | |
MysqlCon.end(); | |
console.log('done!'); | |
console.log('tap `cmd/ctrl+c` to exit'); | |
} | |
}, 300); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment