.gitignore

node_modules
.vscode
.clinic

README.md

OptimalBits/bull#1906

index.js

const Queue = require('bull');
const uuid = require('uuid-random');
const pImmediate = require('p-immediate');

const jobQueue = new Queue('my-test-queue');

jobQueue.process(5, async ({opts}) => {
  const {jobId} = opts;
  try {
    await pImmediate();
    console.log(`I AM A JOB ${jobId}`);
  } catch(e) {
    console.error('An error occurred during processing.');
    throw e;
  }
});

function _generateJobDefs() {
  const jobDefinitions = [];
  for(let i = 0; i < 10; ++i) {
    jobDefinitions.push({
      opts: {
        jobId: uuid(),
        // comment out removeOnComplete and run completes
        removeOnComplete: true,
      },
    });
  }
  return jobDefinitions;
}

(async () => {
  for(let i = 0; i < 100000000; ++i) {
    await pImmediate();
    const jobs = await jobQueue.addBulk(_generateJobDefs());
    await pImmediate();
    const result = await Promise.all(jobs.map(j => j.finished()));
      console.log(`${Date.now()} COMPLETED ${i}`);
    jobQueue.clean(0);
  }
  jobQueue.close();
})().catch(console.error);

package.json

{
  "name": "bull-break",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "dependencies": {
    "bull": "^3.19.0",
    "p-immediate": "^3.1.0",
    "uuid-random": "^1.3.2"
  },
  "devDependencies": {},
  "scripts": {
    "start": "redis-cli flushall & node index.js",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "repository": {
    "type": "git",
    "url": "git+ssh://[email protected]/3f78a3e108f29650ad2c3e97885c2cee.git"
  },
  "author": "",
  "license": "ISC",
  "bugs": {
    "url": "https://gist.github.com/3f78a3e108f29650ad2c3e97885c2cee"
  },
  "homepage": "https://gist.github.com/3f78a3e108f29650ad2c3e97885c2cee"
}