import { Supadata } from '@supadata/js';
const supadata = new Supadata({
apiKey: 'YOUR_API_KEY',
});
// Start a crawl job
const crawl = await supadata.web.crawl({
url: 'https://example.com',
limit: 10,
});
// Get crawl job results
// This automatically handles pagination and returns all pages
const crawlResults = await supadata.web.getCrawlResults(crawl.jobId);
console.log(crawlResults);
{
"status": "completed",
"pages": [
{
"url": "https://supadata.ai",
"content": "# Supadata\n## What is Supadata?\nSupadata is an API platform for data extraction for LLM training.",
"countCharacters": 12300,
"name": "Supadata: Web & YouTube to text API for developers",
"ogUrl": "https://supadata.ai/opengraph-image.png"
}
],
"next": "https://api.supadata.ai/v1/web/crawl/123e4567-e89b-12d3-a456-426614174000?skip=10"
}{
"error": "invalid-request",
"message": "Invalid Request",
"details": "The request is invalid or malformed",
"documentationUrl": "https://docs.supadata.ai/errors#invalid-request"
}{
"error": "invalid-request",
"message": "Invalid Request",
"details": "The request is invalid or malformed",
"documentationUrl": "https://docs.supadata.ai/errors#invalid-request"
}Web Endpoints
Crawl Status
Get the status and results of a crawl by job ID.
GET
/
web
/
crawl
/
{jobId}
import { Supadata } from '@supadata/js';
const supadata = new Supadata({
apiKey: 'YOUR_API_KEY',
});
// Start a crawl job
const crawl = await supadata.web.crawl({
url: 'https://example.com',
limit: 10,
});
// Get crawl job results
// This automatically handles pagination and returns all pages
const crawlResults = await supadata.web.getCrawlResults(crawl.jobId);
console.log(crawlResults);
{
"status": "completed",
"pages": [
{
"url": "https://supadata.ai",
"content": "# Supadata\n## What is Supadata?\nSupadata is an API platform for data extraction for LLM training.",
"countCharacters": 12300,
"name": "Supadata: Web & YouTube to text API for developers",
"ogUrl": "https://supadata.ai/opengraph-image.png"
}
],
"next": "https://api.supadata.ai/v1/web/crawl/123e4567-e89b-12d3-a456-426614174000?skip=10"
}{
"error": "invalid-request",
"message": "Invalid Request",
"details": "The request is invalid or malformed",
"documentationUrl": "https://docs.supadata.ai/errors#invalid-request"
}{
"error": "invalid-request",
"message": "Invalid Request",
"details": "The request is invalid or malformed",
"documentationUrl": "https://docs.supadata.ai/errors#invalid-request"
}Authorizations
Path Parameters
The crawl job ID
Query Parameters
Required range:
x >= 0Response
Successfully retrieved crawl status
The status of the crawl job
Available options:
scraping, completed, failed, cancelled Example:
"completed"
List of pages that were crawled (only present when status is completed)
Show child attributes
Show child attributes
URL for fetching the next page of results (the same endpoint with a skip offset). Only present when more pages are available.
Example:
"https://api.supadata.ai/v1/web/crawl/123e4567-e89b-12d3-a456-426614174000?skip=10"
Was this page helpful?