Playwright Infinite Scroll Scraper Example

infinite-scroll.js

// npm install playwright

const { chromium } = require('playwright');

async function scrapeInfiniteScroll() {
  const browser = await chromium.launch({
    headless: true,
  });

  const page = await browser.newPage();

  await page.goto('https://scrapingsandbox.com/infinite-scroll', {
    waitUntil: 'networkidle',
  });

  let previousCount = 0;

  while (true) {
    const currentCount = await page.locator('.product-card').count();

    console.log(`Loaded products: ${currentCount}`);

    // Stop when no new products are loaded
    if (currentCount === previousCount) {
      break;
    }

    previousCount = currentCount;

    // Scroll to bottom
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight);
    });

    // Wait for new items to load
    await page.waitForTimeout(1000);
  }

  const products = await page.evaluate(() => {
    return [...document.querySelectorAll('.product-card')].map(
      (card) => ({
        title: card.querySelector('.product-name')?.textContent?.trim(),
        price: card.querySelector('.price')?.textContent?.trim(),
      })
    );
  });

  console.log(`\nTotal products scraped: ${products.length}`);
  console.log(products.slice(0, 5));

  await browser.close();
}

scrapeInfiniteScroll().catch(console.error);

Playwright web scraping example using Scraping Sandbox website that demonstrates how to handle infinite scrolling pages and extract dynamically loaded content. This script automatically scrolls to the bottom of the page, waits for new data to load, and collects all available items.

Example page - https://scrapingsandbox.com/infinite-scroll