index.html

---
layout: default
title: A Fast and Powerful Scraping and Web Crawling Framework
---
{% assign stable = site.data.scrapy.stable %}
{% assign oldstable = site.data.scrapy.oldstable %}
{% assign devel = site.data.scrapy.development %}

<div class="container">

  <div class="first-row">

    <div class="block-left">
      <h1>Meet Scrapy</h1>
      <p>An <span class="highlight">open source</span> and collaborative framework for <span class="highlight">extracting the data you need</span> from websites. In a fast, simple, yet extensible way.
      </p>
    </div>

    <div class="block-right">
      <div class="big-button">
        <h2>Install latest version:</h2>
        <p> <i class="fa fa-download"> </i> Scrapy {{ stable.version }} </p>
        <div class="install-code"><span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> pip install scrapy</div>
        <a href="http://pypi.python.org/pypi/Scrapy"><button>PyPI</button></a>
        <a href="http://doc.scrapy.org/en/{{ stable.version }}/topics/ubuntu.html"><button>Ubuntu Package</button></a>
        <a href="https://github.com/scrapy/scrapy/tarball/{{ stable.version }}"><button>Tarball</button></a>
        <a href="https://github.com/scrapy/scrapy/zipball/{{ stable.version }}"><button>Zip</button></a>
      </div>
    </div>

    {% include badges-bar.html %}

  </div>
</div>

<div class="second-row">
  <div class="container">
    <div class="logo-big"></div>

    <div class="code-box">
      <div class="box-header">
        <p>Sample Codes:</p>
        <ul class="simple-tabs" id="code-tabs">
          <li class="scrapy-10 active">Scrapy 1.0</li>
          <li class="scrapy-024">Scrapy 0.24 (old stable)</li>
        </ul>
      </div>
      <div class="box-code tab-page active-page" id="scrapy-10">
        <pre>
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> pip install scrapy
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> cat > myspider.py &lt;&lt;EOF
{% highlight python %}
import scrapy

class BlogSpider(scrapy.Spider):
    name = 'blogspider'
    start_urls = ['http://blog.scrapinghub.com']

    def parse(self, response):
        for url in response.css('ul li a::attr("href")').re(r'.*/\d\d\d\d/\d\d/$'):
            yield scrapy.Request(response.urljoin(url), self.parse_titles)

    def parse_titles(self, response):
        for post_title in response.css('div.entries > ul > li a::text').extract():
            yield {'title': post_title}
{% endhighlight %}EOF
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> scrapy runspider myspider.py
</pre>
      </div>
      <div class="box-code tab-page" id="scrapy-024">
        <pre>
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> pip install scrapy
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> cat > myspider.py &lt;&lt;EOF
{% highlight python %}
import scrapy
from urlparse import urljoin

class Post(scrapy.Item):
    title = scrapy.Field()
class BlogSpider(scrapy.Spider):
    name, start_urls = 'blogspider', ['http://blog.scrapinghub.com']
    def parse(self, response):
        for url in response.css('ul li a::attr("href")').re(r'.*/\d\d\d\d/\d\d/$'):
            yield scrapy.Request(urljoin(response.url, url), self.parse_titles)
    def parse_titles(self, response):
        for post_title in response.css('div.entries > ul > li a::text').extract():
            yield Post(title=post_title)
{% endhighlight %}EOF
<span class="prompt" onselectstart="return false"><i class="fa fa-dollar"></i></span> scrapy runspider myspider.py
        </pre>
      </div>
    </div>

    <div class="code-subs"><p>Build your <span class="highlight">own webcrawlers</span></p></div>

  </div>

</div>

<div class="container">

  <div class="third-row">
    <div class="block-01">
      <i class="fa fa-flash fa-4x"> </i>
      <h3>Fast and powerful</h3>
      <p>write the rules to extract the data and let Scrapy do the rest</p>
    </div>
    <div class="block-02">
      <i class="fa fa-puzzle-piece fa-4x"> </i>
      <h3>Easily extensible</h3>
      <p>extensible by design, plug new functionality easily without having to touch the core</p>
    </div>
    <div class="block-03">
      <i class="fa fa-cubes fa-4x"> </i>
      <h3>Portable, Python</h3>
      <p>written in Python and runs on Linux, Windows, Mac and BSD</p>
    </div>
  </div>
</div>

<div class='fourth-row'>
  <div class="container">
    <div class="block-left">
      <h2>Healthy community:</h2>
      <ul>
        <li>- 10k stars, 2.8k forks and 900 watchers on <a href="https://github.com/scrapy/scrapy">GitHub</a></li>
        <li>- 2.2k followers on <a href="https://twitter.com/ScrapyProject">Twitter</a></li>
        <li>- 4.1k questions on <a href="http://stackoverflow.com/tags/scrapy/info">StackOverflow</a></li>
        <li>- 2.5k members on <a href="https://groups.google.com/forum/?fromgroups#!aboutgroup/scrapy-users">mailing list</a></li>
      </ul>
    </div>
    <div class="block-right">
      <a href="http://doc.scrapy.org/en/{{ stable.version }}/intro/overview.html">
        <h2><span class="regular">Want to know more?</span> <br /> Scrapy at a glance</h2>
        <i class="fa fa-file-text-o fa-3x"></i>
      </a>
      <p>Wondering what Scrapy can do? <br /> <a href="../companies/">Meet the companies using Scrapy</a></p>
    </div>
  </div>
</div>

<script src="{{ "/js/simple-tabs-pure-js.js" | prepend: site.baseurl }}"></script>

<script>       
window.onload = function() {
var codeTabs = new SimpleTabs(document.getElementById('code-tabs'));
};
</script>