id = "scrapy"
name = "Scrapy"
homepage = "https://docs.scrapy.org/"
executables = ["scrapy"]
[dotfiles]
paths = [
"[configHome]/scrapy.cfg",
"~/.scrapy.cfg",
]
secret = []
[commands]
format = 1
label = "Scrapy"
description = "Run a spider, fetch a URL or create a project. Interactive shell, parse, settings and custom commands are left out."
quality = "ai-draft"
parser = { end_of_options = "--", interleave = "permute", long_equals = false, short_attached = false, short_bundles = false }
[[commands.subcommand]]
path = ["crawl"]
label = "Run a spider"
description = "Start crawling with a spider from the current project and optionally save items to a file."
[[commands.subcommand.source]]
id = "docs"
kind = "manual"
url = "https://docs.scrapy.org/en/latest/topics/commands.html"
revision = "latest 2026-10"
evidence = """
Syntax: scrapy crawl <spider>
-a NAME=VALUE: set a spider argument (may be repeated)
--output FILE or -o FILE: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. -o FILE:FORMAT)
--overwrite-output FILE or -O FILE: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. -O FILE:FORMAT)
"""
[[commands.subcommand.field]]
id = "spider"
label = "Spider name"
type = "string"
position = 0
min = 1
sources = ["docs"]
[[commands.subcommand.field]]
id = "spider_arg"
label = "Spider arguments"
description = "One NAME=VALUE per entry."
type = "string"
option = ["-a"]
multiple = true
sources = ["docs"]
[[commands.subcommand.field]]
id = "output"
label = "Append items to file"
description = "Append to the file; add :FORMAT to the name to pick the format."
type = "string"
option = ["--output", "-o"]
sources = ["docs"]
[[commands.subcommand.field]]
id = "overwrite_output"
label = "Overwrite items file"
description = "Replace the file's existing content; add :FORMAT to the name to pick the format."
type = "string"
option = ["--overwrite-output", "-O"]
effect = "overwrite"
sources = ["docs"]
[[commands.subcommand.task]]
id = "crawl"
label = "Run a spider"
fields = ["spider", "spider_arg"]
required = ["spider"]
[[commands.subcommand.task]]
id = "export"
label = "Run and export items"
fields = ["spider", "output", "overwrite_output", "spider_arg"]
required = ["spider"]
[[commands.subcommand.group]]
id = "basic"
label = "Basic"
fields = ["spider", "output"]
[[commands.subcommand.group]]
id = "advanced"
label = "Advanced"
fields = ["overwrite_output", "spider_arg"]
[[commands.subcommand]]
path = ["fetch"]
label = "Fetch a URL"
description = "Download a URL with the Scrapy downloader and print it to standard output."
[[commands.subcommand.source]]
id = "docs"
kind = "manual"
url = "https://docs.scrapy.org/en/latest/topics/commands.html"
revision = "latest 2026-10"
evidence = """
Syntax: scrapy fetch <url>
--spider=SPIDER: bypass spider autodetection and force use of specific spider
--headers: print the request's and response's HTTP headers instead of the response's body
--no-redirect: do not follow HTTP 3xx redirects (default is to follow them)
"""
[[commands.subcommand.field]]
id = "url"
label = "URL"
type = "url"
position = 0
min = 1
sources = ["docs"]
[[commands.subcommand.field]]
id = "headers"
label = "Print headers only"
type = "boolean"
option = ["--headers"]
sources = ["docs"]
[[commands.subcommand.field]]
id = "no_redirect"
label = "Do not follow redirects"
type = "boolean"
option = ["--no-redirect"]
sources = ["docs"]
[[commands.subcommand.field]]
id = "spider"
label = "Force spider"
type = "string"
option = ["--spider"]
sources = ["docs"]
[[commands.subcommand.task]]
id = "fetch"
label = "Fetch a page"
fields = ["url", "headers", "no_redirect", "spider"]
required = ["url"]
[[commands.subcommand.group]]
id = "basic"
label = "Basic"
fields = ["url", "headers", "no_redirect"]
[[commands.subcommand.group]]
id = "advanced"
label = "Advanced"
fields = ["spider"]
[[commands.subcommand]]
path = ["startproject"]
label = "Create a project"
description = "Create a new Scrapy project directory."
[[commands.subcommand.source]]
id = "docs"
kind = "manual"
url = "https://docs.scrapy.org/en/latest/topics/commands.html"
revision = "latest 2026-10"
evidence = """
Syntax: scrapy startproject <project_name> [project_dir]
"""
[[commands.subcommand.field]]
id = "project_name"
label = "Project name"
type = "string"
position = 0
min = 1
sources = ["docs"]
[[commands.subcommand.field]]
id = "project_dir"
label = "Project directory"
description = "Defaults to the project name."
type = "path"
path = { kind = "directory", role = "output", must_exist = false }
position = 1
sources = ["docs"]
[[commands.subcommand.task]]
id = "new"
label = "New project"
fields = ["project_name", "project_dir"]
required = ["project_name"]
[[commands.subcommand.group]]
id = "basic"
label = "Basic"
fields = ["project_name", "project_dir"]
