Manage keys and run conversions over HTTP

import time
import requests

API_KEY = "your_api_key_here"
BASE_URL = "https://sitetomarkdown.com/api/v1"
SEED_URL = "https://docs.example.com/"

headers = {"Authorization": f"Bearer {API_KEY}"}
started = False

# Step 1: check your available balance before creating a crawl.
balance_response = requests.get(f"{BASE_URL}/balance", headers=headers, timeout=30)
balance_response.raise_for_status()
print(balance_response.json())

# Step 2: create a conversion and capture the returned tracking ID.
create_response = requests.post(
        f"{BASE_URL}/conversions",
        headers={**headers, "Content-Type": "application/json"},
        json={"url": SEED_URL},
        timeout=30,
)
create_response.raise_for_status()
tracking_id = create_response.json()["tracking_id"]
print(f"Tracking ID: {tracking_id}")

# Step 3: poll until the conversion is ready to download or ready to start.
while True:
        status_response = requests.get(
                f"{BASE_URL}/conversions/{tracking_id}",
                headers=headers,
                timeout=30,
        )
        status_response.raise_for_status()
        payload = status_response.json()
        status = payload["status"]
        print(f"Current status: {status}")

        if status == "ready_to_crawl" and not started:
                # Step 4: start the paid crawl once discovery is complete.
                start_response = requests.post(
                        f"{BASE_URL}/conversions/{tracking_id}/start",
                        headers=headers,
                        timeout=30,
                )

                if (
                    start_response.status_code == 409
                    and start_response.json().get("error", {}).get("code") == "start_not_required"
                ):
                    print("Start not required; continuing to poll for download availability.")
                else:
                    start_response.raise_for_status()
                    print(start_response.json())
                    started = True

        elif status in {"download_available", "completed"}:
                break

        elif status == "error":
                raise RuntimeError("Conversion failed before a downloadable artifact was produced.")

        time.sleep(5)

# Step 5: download the consolidated Markdown artifact.
with requests.get(
        f"{BASE_URL}/conversions/{tracking_id}/download",
        headers=headers,
        timeout=60,
        stream=True,
) as download_response:
        download_response.raise_for_status()
        with open("docs.md", "wb") as output_file:
                for chunk in download_response.iter_content(chunk_size=8192):
                        if chunk:
                                output_file.write(chunk)

import { writeFile } from "node:fs/promises";

const API_KEY = "your_api_key_here";
const BASE_URL = "https://sitetomarkdown.com/api/v1";
const SEED_URL = "https://docs.example.com/";

const headers = {
    Authorization: `Bearer ${API_KEY}`,
};

let started = false;

async function getJson(url: string, init?: RequestInit) {
    const response = await fetch(url, init);
    const contentType = response.headers.get("content-type") || "";
    const payload = contentType.includes("application/json")
        ? await response.json()
        : null;

    if (!response.ok) {
        const error = payload?.error;
        const suffix = error?.code ? ` (${error.code})` : "";
        throw new Error(
            `${response.status} ${response.statusText}${suffix}: ${error?.message || "Request failed."}`,
        );
    }

    return payload;
}

const balance = await getJson(`${BASE_URL}/balance`, {
    headers,
});
console.log(balance);

const created = await getJson(`${BASE_URL}/conversions`, {
    method: "POST",
    headers: {
        ...headers,
        "Content-Type": "application/json",
    },
    body: JSON.stringify({ url: SEED_URL }),
});

const trackingId = created.tracking_id;
console.log(`Tracking ID: ${trackingId}`);

while (true) {
    const statusPayload = await getJson(`${BASE_URL}/conversions/${trackingId}`, {
        headers,
    });
    const status = statusPayload.status;
    console.log(`Current status: ${status}`);

    if (status === "ready_to_crawl" && !started) {
        const startResponse = await fetch(`${BASE_URL}/conversions/${trackingId}/start`, {
            method: "POST",
            headers,
        });
        const startContentType = startResponse.headers.get("content-type") || "";
        const startPayload = startContentType.includes("application/json")
            ? await startResponse.json()
            : null;

        if (startResponse.ok) {
            console.log(startPayload);
            started = true;
        } else if (startResponse.status === 409 && startPayload?.error?.code === "start_not_required") {
            console.log("Start not required; continuing to poll for download availability.");
        } else {
            throw new Error(
                `${startResponse.status} ${startResponse.statusText} (${startPayload?.error?.code || "unknown_error"}): ${startPayload?.error?.message || "Request failed."}`,
            );
        }
    } else if (status === "download_available" || status === "completed") {
        break;
    } else if (status === "error") {
        throw new Error("Conversion failed before a downloadable artifact was produced.");
    }

    await new Promise((resolve) => setTimeout(resolve, 5000));
}

const downloadResponse = await fetch(`${BASE_URL}/conversions/${trackingId}/download`, {
    headers,
});

if (!downloadResponse.ok) {
    throw new Error(`${downloadResponse.status} ${downloadResponse.statusText}`);
}

const markdown = await downloadResponse.text();
await writeFile("docs.md", markdown, "utf8");
console.log("Saved docs.md");