آموزش جامع Scrapy به فارسی

scrapy fetch --headers https://example.com/
scrapy parse https://example.com/product/1 -c parse_product
scrapy settings --get DOWNLOAD_DELAY
scrapy runspider standalone.py -O data.json
scrapy version -v

yield scrapy.FormRequest.from_response(
    response,
    formdata={"username": user, "password": password},
    callback=self.after_login,
    meta={"cookiejar": user_id},
)

yield response.follow(
    "/account/orders",
    callback=self.parse_orders,
    meta={"cookiejar": response.meta["cookiejar"]},
)

def parse_product(self, response):
    """
    @url https://example.com/product/1
    @returns items 1 1
    @returns requests 0 0
    @scrapes name price url
    """
    yield {
        "name": response.css("h1::text").get(),
        "price": response.css(".price::text").get(),
        "url": response.url,
    }

scrapy check
scrapy check products
scrapy check -l

# settings.py
ADDONS = {
    "my_package.addon.AnalyticsAddon": 500,
}

# داخل پکیج
class AnalyticsAddon:
    def update_settings(self, settings):
        settings.setdict(
            {"EXTENSIONS": {"my_package.ext.Analytics": 500}},
            priority="addon",
        )

yield scrapy.Request(
    "https://example.com/api/search?page=1",
    headers={"Accept": "application/json"},
    callback=self.parse_api,
)

def parse_api(self, response):
    data = response.json()
    yield from data["results"]

# سنجش پایه بدون منطق پروژه
scrapy bench

# آمار حافظه
scrapy crawl products -s MEMUSAGE_ENABLED=True

# در Telnet Console توسعه
prefs()
prefs(MySpider)

python -m venv .venv
source .venv/bin/activate  # macOS / Linux
.venv\Scripts\Activate.ps1 # Windows
python -m pip install --upgrade pip
python -m pip install scrapy
scrapy version

scrapy startproject shopbot
cd shopbot
scrapy genspider products example.com
scrapy list

scrapy shell "https://quotes.toscrape.com/"
response.status
response.css("title::text").get()
response.css("div.quote").getall()[:2]
view(response)

import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    allowed_domains = ["quotes.toscrape.com"]
    start_urls = ["https://quotes.toscrape.com/"]

    def parse(self, response):
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
                "tags": quote.css("a.tag::text").getall(),
            }
        next_url = response.css("li.next a::attr(href)").get()
        if next_url:
            yield response.follow(next_url, callback=self.parse)

scrapy crawl quotes -O quotes.json:json

card = response.css("article.product")[0]
name = card.css("h2::text").get(default="").strip()
price = card.css("[data-price]::attr(data-price)").get()
links = response.css("a::attr(href)").getall()
absolute = response.urljoin(links[0])

def parse(self, response):
    for card in response.css(".product-card"):
        yield response.follow(
            card.css("a::attr(href)").get(),
            callback=self.parse_product,
            cb_kwargs={"category": "books"},
            errback=self.handle_error,
        )

def parse_product(self, response, category):
    yield {"category": category, "url": response.url}

import scrapy
class ProductItem(scrapy.Item):
    name = scrapy.Field()
    price = scrapy.Field()
    currency = scrapy.Field()
    url = scrapy.Field()

item = ProductItem()
item["name"] = response.css("h1::text").get()
item["url"] = response.url
yield item

from decimal import Decimal
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem

class CleanProductPipeline:
    def process_item(self, item, spider):
        data = ItemAdapter(item)
        name = (data.get("name") or "").strip()
        if not name:
            raise DropItem("محصول بدون نام")
        data["name"] = name
        data["price"] = Decimal(str(data["price"]).replace(",", ""))
        return item

ITEM_PIPELINES = {
    "shopbot.pipelines.CleanProductPipeline": 300,
}

FEEDS = {
    "exports/%(name)s-%(time)s.jsonl": {
        "format": "jsonlines",
        "encoding": "utf-8",
        "overwrite": False,
    }
}

def parse(self, response):
    payload = response.json()
    yield from payload["results"]

python -m pip install scrapy-playwright
playwright install chromium

DOWNLOAD_HANDLERS = {
    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}
yield scrapy.Request(url, meta={"playwright": True})

ROBOTSTXT_OBEY = True
USER_AGENT = "shopbot/1.0 (+https://example.org/bot)"
CONCURRENT_REQUESTS_PER_DOMAIN = 4
DOWNLOAD_DELAY = 0.5
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_MAX_DELAY = 30
RETRY_TIMES = 2
HTTPCACHE_ENABLED = True  # فقط توسعه

scrapy settings --get CONCURRENT_REQUESTS
scrapy crawl products -s CLOSESPIDER_ITEMCOUNT=20 -L INFO
scrapy check

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

class CatalogSpider(CrawlSpider):
    name = "catalog"
    allowed_domains = ["example.com"]
    start_urls = ["https://example.com/products/"]
    rules = (
        Rule(LinkExtractor(allow=r"/product/\d+"),
             callback="parse_product"),
        Rule(LinkExtractor(allow=r"/products\?page="), follow=True),
    )

    def parse_product(self, response):
        yield {"title": response.css("h1::text").get(),
               "url": response.url}

class CrawlerHeaderMiddleware:
    def process_request(self, request, spider):
        request.headers.setdefault("X-Crawler", spider.name)
        return None

DOWNLOADER_MIDDLEWARES = {
    "shopbot.middlewares.CrawlerHeaderMiddleware": 543,
}

from scrapy import signals

class RunSummary:
    @classmethod
    def from_crawler(cls, crawler):
        ext = cls()
        ext.stats = crawler.stats
        crawler.signals.connect(ext.closed, signal=signals.spider_closed)
        return ext

    def closed(self, spider, reason):
        count = self.stats.get_value("item_scraped_count", 0)
        spider.logger.info("items=%s reason=%s", count, reason)

yield {
    "image_urls": [
        response.urljoin(url)
        for url in response.css(".gallery img::attr(src)").getall()
    ]
}

ITEM_PIPELINES = {"scrapy.pipelines.images.ImagesPipeline": 100}
IMAGES_STORE = "data/images"
IMAGES_THUMBS = {"small": (120, 120)}

scrapy crawl catalog -s JOBDIR=jobs/catalog-2026
# توقف محترمانه: یک بار Ctrl+C
# ادامه با همان فرمان

CONCURRENT_REQUESTS = 32
CONCURRENT_REQUESTS_PER_DOMAIN = 4
REACTOR_THREADPOOL_MAXSIZE = 20
DNS_TIMEOUT = 20
DOWNLOAD_TIMEOUT = 30
LOG_LEVEL = "INFO"
MEMUSAGE_ENABLED = True

from scrapy.crawler import AsyncCrawlerProcess
from scrapy.utils.project import get_project_settings

process = AsyncCrawlerProcess(get_project_settings())
process.crawl("catalog", category="books")
process.start()

import os
API_TOKEN = os.environ["SHOP_API_TOKEN"]
TELNETCONSOLE_ENABLED = False
COOKIES_DEBUG = False