> ## Documentation Index
> Fetch the complete documentation index at: https://docs.supadata.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Crawl

> Create a crawl job to extract content from all pages on a website.



## OpenAPI

````yaml v1-openapi POST /web/crawl
openapi: 3.1.0
info:
  title: Supadata
  description: Web & Social Media Content API for Developers
  version: 1.3.0
servers:
  - url: https://api.supadata.ai/v1
security:
  - apiKeyAuth: []
paths:
  /web/crawl:
    post:
      description: Create a crawl job to extract content from all pages on a website.
      operationId: postWebCrawl
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                url:
                  type: string
                  format: uri
                  description: URL of the website to crawl
                  example: https://supadata.ai
                limit:
                  type: number
                  minimum: 1
                  maximum: 5000
                  description: Maximum number of pages to crawl
                  example: 100
                  default: 100
              required:
                - url
      responses:
        '200':
          description: Successfully started crawl job
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/JobId'
        '400':
          description: Invalid Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '429':
          description: Limit Exceeded
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Internal Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
      x-codeSamples:
        - lang: TypeScript
          label: Node.js
          source: >-
            import { Supadata } from '@supadata/js';


            const supadata = new Supadata({
              apiKey: 'YOUR_API_KEY',
            });


            // Start a crawl job

            const crawl = await supadata.web.crawl({
              url: 'https://example.com',
              limit: 10,
            });


            // Get crawl job results

            // This automatically handles pagination and returns all pages

            const crawlResults = await
            supadata.web.getCrawlResults(crawl.jobId);

            console.log(crawlResults);
        - lang: Python
          label: Python
          source: |-
            from supadata import Supadata, SupadataError

            supadata = Supadata(api_key="YOUR_API_KEY")

            # Start a crawl job
            crawl_job = supadata.web.crawl(
                url="https://example.com",
                limit=10  # Optional: limit the number of pages to crawl
            )
            print(f"Started crawl job: {crawl_job.job_id}")

            # Get crawl results
            # This automatically handles pagination and returns all pages
            try:
                pages = supadata.web.get_crawl_results(job_id=crawl_job.job_id)
                for page in pages:
                    print(f"Crawled page: {page.url}")
                    print(f"Page title: {page.name}")
                    print(f"Content: {page.content}")
            except SupadataError as e:
                print(f"Crawl job failed: {e}")
        - lang: Shell
          label: cURL - Start Crawl
          source: |-
            curl -X POST "https://api.supadata.ai/v1/web/crawl" \
              -H "x-api-key: YOUR_API_KEY" \
              -H "Content-Type: application/json" \
              -d '{
                "url": "https://example.com",
                "limit": 10
              }'
        - lang: Shell
          label: cURL - Get Crawl Results
          source: |-
            curl -X GET "https://api.supadata.ai/v1/web/crawl/JOB_ID" \
              -H "x-api-key: YOUR_API_KEY" \
              -H "Content-Type: application/json"
components:
  schemas:
    JobId:
      type: object
      properties:
        jobId:
          type: string
          description: The ID of the job
          example: 123e4567-e89b-12d3-a456-426614174000
      required:
        - jobId
    Error:
      type: object
      properties:
        error:
          type: string
          enum:
            - invalid-request
            - internal-error
            - forbidden
            - unauthorized
            - upgrade-required
            - transcript-unavailable
            - not-found
            - limit-exceeded
          description: Error code identifying the type of error
          example: invalid-request
        message:
          type: string
          description: Human readable error message
          example: Invalid Request
        details:
          type: string
          description: Detailed error description
          example: The request is invalid or malformed
        documentationUrl:
          type: string
          description: URL to error documentation
          example: https://supadata.ai/documentation/errors#invalid-request
      required:
        - error
        - message
        - details
      description: Standard error response format
  securitySchemes:
    apiKeyAuth:
      type: apiKey
      in: header
      name: x-api-key

````