web-scraper

Web Scraper Skill

Description

Extract and process data from web pages with intelligent parsing capabilities.

Trigger

• /scrape command
• User requests web data extraction
• User needs to parse HTML

Prompt

You are a web scraping expert that extracts data efficiently and ethically.

Puppeteer Scraper (TypeScript)

import puppeteer from 'puppeteer';

interface Product {
  name: string;
  price: number;
  rating: number;
  url: string;
}

async function scrapeProducts(url: string): Promise<Product[]> {
  const browser = await puppeteer.launch({ headless: 'new' });
  const page = await browser.newPage();
  
  // Set user agent to avoid detection
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
  
  await page.goto(url, { waitUntil: 'networkidle2' });
  
  // Wait for products to load
  await page.waitForSelector('.product-card');
  
  const products = await page.evaluate(() => {
    const items = document.querySelectorAll('.product-card');
    return Array.from(items).map(item => ({
      name: item.querySelector('.product-name')?.textContent?.trim() ?? '',
      price: parseFloat(item.querySelector('.price')?.textContent?.replace('$', '') ?? '0'),
      rating: parseFloat(item.querySelector('.rating')?.getAttribute('data-rating') ?? '0'),
      url: item.querySelector('a')?.href ?? '',
    }));
  });
  
  await browser.close();
  return products;
}

Cheerio Parser (Node.js)

import axios from 'axios';
import * as cheerio from 'cheerio';

async function parseArticle(url: string) {
  const { data } = await axios.get(url, {
    headers: { 'User-Agent': 'Mozilla/5.0' }
  });
  
  const $ = cheerio.load(data);
  
  return {
    title: $('h1.article-title').text().trim(),
    author: $('span.author-name').text().trim(),
    date: $('time').attr('datetime'),
    content: $('article.content p').map((_, el) => $(el).text()).get().join('\n\n'),
    tags: $('a.tag').map((_, el) => $(el).text()).get(),
  };
}