dyno/scripts/loadActivity.js

const progress = require('cli-progress');
const db = require('../src/core/database.js');
const config = require('../src/core/config');
const redis = require('../src/core/redis');

const clientId = config.client.id;
const shardCount = 1152;

const multibar = new progress.MultiBar({
  format: '{name} |{bar}| {percentage}% | {duration_formatted} | {value}/{total}',
  barCompleteChar: '\u2588',
  barIncompleteChar: '\u2591',
  clearOnComplete: false,
  stopOnComplete: true,
  hideCursor: true,
});

let mongoBar;
let redisBar;

const buffer = [];

async function flushToRedis() {
  const batchSize = 1000;
  const batchCount = Math.ceil(buffer.length / batchSize);

  let barProgress = 0;
  redisBar = multibar.create(batchCount, barProgress, { name: 'Buffer -> Redis' });

  let itemsInPipeline = 0;
  let pipeline = redis.pipeline();

  for (let e of buffer) {
    const shardId = ~~((e._id / 4194304) % shardCount);
    pipeline.hset(`guild_activity:${clientId}:${shardCount}:${shardId}`, e._id, e.lastActive);

    itemsInPipeline++;

    if (itemsInPipeline >= batchSize) {
      await pipeline.exec();
      pipeline = redis.pipeline();
      itemsInPipeline = 0;
      barProgress++;
      redisBar.update(barProgress);
    }
  }
  if (itemsInPipeline > 0) {
    await pipeline.exec();
  }
  barProgress++;
  redisBar.update(barProgress);
  redisBar.stop();
}

async function fetchMongoDocs() {
  const coll = await db.collection('servers');
  const count = await coll.count({ deleted: false });

  let i = 0;
  mongoBar = multibar.create(count, i, { name: 'Mongo -> Buffer' });

  coll.find({ deleted: false }, { projection: { lastActive: 1 } }).forEach((doc) => {
    i++;

    if (!doc.lastActive) {
      return mongoBar.update(i);
    }

    buffer.push(doc);

    mongoBar.update(i);
  },
  (err) => {
    if (err) {
      console.error(err);
      process.exit(1);
    }
    mongoBar.stop();
    flushToRedis();
  });
}

setTimeout(fetchMongoDocs, 5000);
// models.Server.countDocuments({ deleted: false }).then(count => {
//   let i = 0, p = 0;
//   mongoBar = multibar.create(count, i, { name: 'Mongo -> Buffer' });

//   models.Server.find({ deleted: false }, { lastActive: 1 })
//     .cursor()
//     .on('data', doc => {
//       i++;

//       if (!doc.lastActive) {
// 	      return mongoBar.update(i);
//       }

//       buffer.push(doc);

//       mongoBar.update(i);
//     })
//     .on('end', () => {
//       mongoBar.stop();
//       flushToRedis();
//     });
// });