const puppeteer = require('puppeteer');

async function scrapeWebsite() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto('https://example.com');
    await page.screenshot({ path: 'screenshot.png' });
    await browser.close();
}

scrapeWebsite();

async function scrapeInfiniteScroll() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    await page.goto('https://example.com/feed');

    let items = [];
    let previousHeight = 0;

    while (items.length < 100) { // Collect 100 items
        items = await page.evaluate(() => {
            return Array.from(document.querySelectorAll('.item')).map(item => ({
                title: item.querySelector('.title').innerText,
                description: item.querySelector('.description').innerText
            }));
        });

        previousHeight = await page.evaluate('document.body.scrollHeight');
        await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
        await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`);
        await page.waitForTimeout(1000); // Wait for new content to load
    }

    await browser.close();
    return items;
}

const puppeteer = require('puppeteer');

async function resilientScrape(url, maxRetries = 3) {
    let browser;
    try {
        browser = await puppeteer.launch({
            headless: true,
            args: ['--no-sandbox', '--disable-setuid-sandbox']
        });

        const page = await browser.newPage();
        await page.setViewport({ width: 1920, height: 1080 });
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

        let retries = 0;
        while (retries < maxRetries) {
            try {
                await page.goto(url, { waitUntil: 'networkidle0' });
                // Your scraping logic here
                break;
            } catch (error) {
                retries++;
                if (retries === maxRetries) throw error;
                await page.waitForTimeout(1000 * retries);
            }
        }

    } catch (error) {
        console.error('Scraping failed:', error);
        throw error;
    } finally {
        if (browser) await browser.close();
    }
}