Skip to content

Instantly share code, notes, and snippets.

@mzubairsaleem
Created July 19, 2023 22:37
Show Gist options
  • Save mzubairsaleem/9a66730f3adf1f9605252c304d1e1a41 to your computer and use it in GitHub Desktop.
Save mzubairsaleem/9a66730f3adf1f9605252c304d1e1a41 to your computer and use it in GitHub Desktop.
async function searchWithScroll(indexName, pageSize, query, maxResults) {
let scrollId;
let results = [];
let _query = Object.keys(query).length === 0 ? {
query: {
"match_all": {}
}
} : {
query: query
};
while (true) {
const response = scrollId ?
await client.scroll({
scroll_id: scrollId,
scroll: '1m',
}) :
await client.search({
index: indexName,
size: pageSize,
scroll: '1m',
body: _query,
sort: [
{
"_fieldsCount": {
"order": "desc"
}
}
]
});
const hits = response.hits.hits;
scrollId = response._scroll_id;
if (!hits.length || results.length >= maxResults) break;
results = results.concat(hits.slice(0, maxResults - results.length));
return true;
}
// Clear the scroll to free resources in Elasticsearch
await client.clearScroll({
scroll_id: scrollId
});
results = results.map(t => cleanLead(t._source, true));
return results;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment