Crawl

import {
  Supadata,
  JobId
} from '@supadata/js';

// Initialize the client
const supadata = new Supadata({
  apiKey: 'YOUR_API_KEY',
});

// Crawl website
const crawl: JobId = await supadata.web.crawl({
  url: 'https://supadata.ai',
  limit: 10,
});

console.log(`Started crawl job: ${crawl.jobId}`);

from supadata import Supadata

# Initialize the client
supadata = Supadata(api_key="YOUR_API_KEY")

# Start a crawl job
crawl_job = supadata.web.crawl(
    url="https://supadata.ai",
    limit=100  # Optional: limit the number of pages to crawl
)
print(f"Started crawl job: {crawl_job.job_id}")

curl -X POST 'https://api.supadata.ai/v1/web/crawl' \
-H 'x-api-key: YOUR_API_KEY' \
-H 'Content-Type: application/json' \
-d '{"url": "https://supadata.ai", "limit": 100}'

import { Supadata } from "@supadata/js";

const supadata = new Supadata("YOUR_API_KEY");

// Get crawl job results
// This automatically handles pagination and returns all pages
const crawlResult = await supadata.web.getCrawlResults(jobId);

if (crawlResult.status === "completed") {
  console.log("Crawl job completed successfully!");
  console.log(`Total pages crawled: ${crawlResult.pages.length}`);
  
  // Process each page
  crawlResult.pages.forEach((page, index) => {
    console.log(`Page ${index + 1}: ${page.name}`);
    console.log(`URL: ${page.url}`);
    console.log(`Description: ${page.description}`);
    console.log(`Content preview: ${page.content.substring(0, 100)}...`);
    console.log("---");
  });
} else if (crawlResult.status === "failed") {
  console.error("Crawl job failed:", crawlResult.error);
} else {
  console.log("Job status:", crawlResult.status);
}

from supadata import Supadata

supadata = Supadata("YOUR_API_KEY")

# Get crawl results
# This automatically handles pagination and returns all pages
crawl_result = supadata.web.get_crawl_results(job_id=job_id)

if crawl_result.status == "completed":
    print("Crawl job completed successfully!")
    print(f"Total pages crawled: {len(crawl_result.pages)}")
    
    # Process each page
    for i, page in enumerate(crawl_result.pages):
        print(f"Page {i + 1}: {page.name}")
        print(f"URL: {page.url}")
        print(f"Description: {page.description}")
        print(f"Content preview: {page.content[:100]}...")
        print("---")
elif crawl_result.status == "failed":
    print(f"Crawl job failed: {crawl_result.error}")
else:
    print(f"Job status: {crawl_result.status}")

curl -X GET 'https://api.supadata.ai/v1/web/crawl/123e4567-e89b-12d3-a456-426614174000' \
  -H 'x-api-key: YOUR_API_KEY'