How to backfill historical data into Supermemory

import { Supermemory } from "supermemory";

type SourceDocument = {
  id: string;
  content: string;
  createdAt: string;
};

const supermemory = new Supermemory({ apiKey: process.env.SUPERMEMORY_API_KEY });
const batchSize = 100;

async function backfillHistoricalData(sourceDocuments: SourceDocument[]) {
  const documents = sourceDocuments
    .map((document) => ({
      content: document.content,
      id: document.id,
      date: new Date(document.createdAt).toISOString()
    }))
    .sort((a, b) => a.date.localeCompare(b.date));

  for (let offset = 0; offset < documents.length; offset += batchSize) {
    const result = await supermemory.documents.batchAdd("historical_import", {
      documents: documents.slice(offset, offset + batchSize),
    });

    if (result.failed > 0) {
      throw new Error(`${result.failed} documents failed to ingest`);
    }
  }
}

from datetime import datetime, timezone
from supermemory import Supermemory

client = Supermemory()
batch_size = 100

def to_utc(value: str) -> str:
    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
    if parsed.tzinfo is None:
        raise ValueError("created_at must include a timezone")
    return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")

def backfill_historical_data(source_documents: list[dict[str, str]]) -> None:
    documents = sorted(
        [
            {
                "content": document["content"],
                "id": document["id"],
                "date": to_utc(document["created_at"]),
            }
            for document in source_documents
        ],
        key=lambda document: document["date"],
    )

    for offset in range(0, len(documents), batch_size):
        result = client.documents.batch_add(
            "historical_import",
            documents=documents[offset : offset + batch_size],
        )

        if result.failed > 0:
            raise RuntimeError(f"{result.failed} documents failed to ingest")

curl -X POST "https://api.supermemory.ai/ns/historical_import/document/batch" \
  -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "documents": [
      { "content": "first message", "id": "msg_001", "date": "2023-01-05T09:00:00Z" },
      { "content": "second message", "id": "msg_002", "date": "2023-01-06T14:30:00Z" }
    ]
  }'

async function waitUntilDone(ids: string[]) {
  while (true) {
    const documents = await Promise.all(
      ids.map((id) => supermemory.documents.get("historical_import", id))
    );

    if (documents.some((document) => document.system.status === "failed")) {
      throw new Error("A document failed to process");
    }

    if (documents.every((document) => document.system.status === "done")) {
      return;
    }
    await new Promise((resolve) => setTimeout(resolve, 10_000));
  }
}

import time

def wait_until_done(ids: list[str]) -> None:
    while True:
        documents = [
            client.documents.get("historical_import", document_id) for document_id in ids
        ]

        if any(document.system.status == "failed" for document in documents):
            raise RuntimeError("A document failed to process")

        if all(document.system.status == "done" for document in documents):
            return

        time.sleep(10)

curl "https://api.supermemory.ai/ns/historical_import/document/msg_001" \
  -H "Authorization: Bearer $SUPERMEMORY_API_KEY"
# repeat until "system": { "status": "done" }