{"owner":"commoncrawl","github":"https://github.com/commoncrawl","claimed":false,"inventory":[],"indexed":[{"repo":"commoncrawl/news-crawl","github":"https://github.com/commoncrawl/news-crawl","description":"News crawling with StormCrawler - stores content as WARC","language":"Java","stars":375,"topics":["crawler","news","warc","web-crawler","apache-storm","common-crawl","commoncrawl","storm-crawler"],"license":"Apache-2.0","category":"scrapers-browser-automation"},{"repo":"commoncrawl/web-languages","github":"https://github.com/commoncrawl/web-languages","description":"Crowd-sourced lists of urls to help Common Crawl crawl under-resourced languages. See https://github.com/commoncrawl/web-languages-code/ for the code","language":null,"stars":71,"topics":["crawling","dataset","language-detection"],"license":null,"category":"scrapers-browser-automation"}],"how_to_buy":"GET /r/commoncrawl/<repo> (Accept: application/json) for any listed repo here: tree, README, price and the checkout to pay (x402; rehearse first at its test twin, simulated money). Repos under 'indexed' are free: clone them from GitHub."}