trending-crawler

Trending Crawler

Discovers new AI Agent Skills from GitHub Trending pages and adds them to skills.json.

Workflow

1. Scrape GitHub Trending

cd /tmp && \
curl -s "https://github.com/trending?since=daily" | \
python3 << 'PYEOF'
import sys
import re
from datetime import datetime

html = sys.stdin.read()

# Extract repos from trending page
# Pattern: data-href="/owner/repo"
pattern = r'data-href="(/[^/]+/[^"]+)"'
matches = re.findall(pattern, html)

seen = set()
repos = []
for m in matches:
    if m not in seen and '/tre' not in m:  # skip trending itself
        seen.add(m)
        repos.append(m)

print(f"Found {len(repos)} repos on trending")
for r in repos[:10]:
    print(f"  {r}")
PYEOF

2. Filter for Skill-Related Repos

cd /tmp && \
TRENDING_HTML=$(curl -s "https://github.com/trending?since=daily") && \
echo "$TRENDING_HTML" | python3 << 'PYEOF'
import sys
import re
import json

html = sys.stdin.read()

# Extract repo paths and descriptions
# Pattern for repo: <h2><a href="/owner/repo">
repo_pattern = r'<h2>\s*<a href="(/[^/]+/[^"]+)"[^>]*>'
desc_pattern = r'<p class="col-9[^"]*"[^>]*>([^<]+)</p>'
star_pattern = r'aria-label="(\d[\d,]+) star'

repo_matches = re.findall(repo_pattern, html)
desc_matches = re.findall(desc_pattern, html)
star_matches = re.findall(star_pattern, html)

print(f"Repos found: {len(repo_matches)}")
print(f"Descs found: {len(desc_matches)}")
print(f"Stars found: {len(star_matches)}")

# Keywords that indicate a skill-related repo
SKILL_KEYWORDS = [
    'skill', 'agent', 'claude', 'openai', 'mcp', 'tool',
    'prompt', 'workflow', 'task', 'automation', 'cli',
    'awesome', 'framework', 'hub', 'marketplace'
]

results = []
for i, repo in enumerate(repo_matches[:20]):
    desc = desc_matches[i].strip() if i < len(desc_matches) else ""
    stars = star_matches[i] if i < len(star_matches) else "0"
    
    desc_lower = desc.lower()
    if any(k in desc_lower or k in repo.lower() for k in SKILL_KEYWORDS):
        results.append({
            'name': repo.lstrip('/'),
            'desc': desc,
            'stars': stars
        })

print(f"\nSkill-related repos: {len(results)}")
for r in results:
    print(f"  {r['name']} ({r['stars']}★) - {r['desc'][:60]}")

# Save for next step
with open('/tmp/trending_skills.json', 'w') as f:
    json.dump(results, f)
PYEOF