link-doctor

Link Doctor

Detects broken links in README.md, tutorials, blogs, and papers, then attempts to repair them.

Workflow

1. Scan All Markdown Files for Links

cd /Volumes/waku/github-维护/awesome/awesome-skills-repos && \
find . -name "*.md" -not -path "./.git/*" | head -20 && \
echo "---" && \
# Extract all GitHub links
grep -rhops "https://github.com/" --include="*.md" . | \
grep -oE "https://github\.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+" | \
sort -u | head -50

2. Check Links with curl (Batch)

cd /Volumes/waku/github-维护/awesome/awesome-skills-repos && \
python3 << 'PYEOF'
import subprocess
import re
from concurrent.futures import ThreadPoolExecutor, as_completed

REPO_PATH = "/Volumes/waku/github-维护/awesome/awesome-skills-repos"

# Collect all GitHub URLs from markdown files
github_urls = set()
for ext in ['*.md']:
    result = subprocess.run(
        f'grep -rhops "https://github.com/" {REPO_PATH} --include="*.md" 2>/dev/null | grep -oE "https://github\.com/[^ )\"\\']+" | sort -u',
        shell=True, capture_output=True, text=True
    )
    for url in result.stdout.strip().split('\n'):
        if url:
            # Normalize - remove trailing slashes, .git, etc.
            url = url.rstrip('/').replace('.git', '')
            github_urls.add(url)

urls = list(github_urls)
print(f"Total unique GitHub URLs to check: {len(urls)}")

def check_url(url):
    """Check a URL and return status"""
    try:
        result = subprocess.run(
            ['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-L', '--max-time', '10', url],
            capture_output=True, text=True, timeout=15
        )
        code = result.stdout.strip()
        return (url, code)
    except Exception as e:
        return (url, f"ERROR: {e}")

broken = []
working = []

with ThreadPoolExecutor(max_workers=10) as executor:
    futures = {executor.submit(check_url, url): url for url in urls[:100]}  # Check first 100
    
    for i, future in enumerate(as_completed(futures)):
        url, code = future.result()
        if code not in ['200', '301', '302']:
            broken.append((url, code))
            print(f"  BROKEN ({code}): {url}")
        else:
            working.append((url, code))
        
        if (i+1) % 20 == 0:
            print(f"Progress: {i+1}/{len(futures)}")

print(f"\n=== Results ===")
print(f"Working: {len(working)}")
print(f"Broken: {len(broken)}")

if broken:
    with open('/tmp/broken_links.json', 'w') as f:
        import json
        json.dump(broken, f, indent=2)
    print("Broken links saved to /tmp/broken_links.json")
PYEOF