Pagination

curl -sS "https://api.main-team.org/v1/student?page=2&limit=100" \
  -H "Authorization: Bearer $TOKEN"

curl -sS "https://api.main-team.org/v1/student?page=2&limit=2" \
  -H "Authorization: Bearer $TOKEN"

{
  "success": true,
  "message": "Students fetched successfully.",
  "data": [
    {
      "_id": "652f1c9b8e4b2a0012a3c4d5",
      "username": "XXK1042",
      "firstName": "Jane",
      "lastName": "Doe",
      "grade": { "_id": "5f1a2b3c4d5e6f7a8b9c0d1e", "name": "10" }
    },
    {
      "_id": "652f1c9b8e4b2a0012a3c4d6",
      "username": "XXB1043",
      "firstName": "Arben",
      "lastName": "Hoxha",
      "grade": { "_id": "5f1a2b3c4d5e6f7a8b9c0d1f", "name": "11" }
    }
  ],
  "pagination": { "page": 2, "limit": 2, "total": 7, "totalPages": 4 }
}

page=1
while : ; do
  resp=$(curl -sS --fail-with-body \
    -H "Authorization: Bearer $TOKEN" \
    "https://api.main-team.org/v1/student?page=$page&limit=100") || { echo "$resp" >&2; exit 1; }

  echo "$resp" | jq -c '.data[]' >> students.ndjson

  total_pages=$(echo "$resp" | jq '.pagination.totalPages')
  count=$(echo "$resp" | jq '.data | length')
  if [ "$count" -eq 0 ] || [ "$page" -ge "$total_pages" ]; then break; fi
  page=$((page + 1))
done

const BASE = process.env.MTO_API_BASE ?? 'https://api.main-team.org/v1';
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

export async function* everyRecord(token, path) {
  for (let page = 1; ; page += 1) {
    const sep = path.includes('?') ? '&' : '?';
    const res = await fetch(`${BASE}${path}${sep}page=${page}&limit=100`, {
      headers: { Authorization: `Bearer ${token}` },
    });

    if (res.status === 429) {
      await sleep(Number(res.headers.get('retry-after') ?? 60) * 1000);
      page -= 1; // ask for the same page again
      continue;
    }
    const body = await res.json();
    if (!res.ok) {
      throw new Error(`${res.status} ${body.error?.code}: ${body.error?.message} (${body.error?.request_id})`);
    }

    yield* body.data;

    const { totalPages } = body.pagination;
    if (body.data.length === 0 || page >= totalPages) return;

    // Out of budget for this operation: wait for the window to reset.
    if (res.headers.get('x-ratelimit-remaining') === '0') {
      await sleep(Number(res.headers.get('x-ratelimit-reset') ?? 60) * 1000);
    }
  }
}

// Usage: every student, then every application on one organization.
for await (const student of everyRecord(token, '/student')) {
  console.log(student._id, student.username);
}
for await (const application of everyRecord(token, `/${organizationId}/application`)) {
  console.log(application._id, application.user?.mainId);
}

<?php
function everyRecord(string $token, string $path): Generator
{
    $base = getenv('MTO_API_BASE') ?: 'https://api.main-team.org/v1';
    $page = 1;

    while (true) {
        $headers = [];
        $sep = str_contains($path, '?') ? '&' : '?';
        $ch = curl_init("{$base}{$path}{$sep}page={$page}&limit=100");
        curl_setopt_array($ch, [
            CURLOPT_HTTPHEADER => ["Authorization: Bearer {$token}"],
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT => 30,
            CURLOPT_HEADERFUNCTION => function ($ch, $line) use (&$headers) {
                $parts = explode(':', $line, 2);
                if (count($parts) === 2) {
                    $headers[strtolower(trim($parts[0]))] = trim($parts[1]);
                }
                return strlen($line);
            },
        ]);
        $raw = curl_exec($ch);
        $status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        curl_close($ch);

        if ($status === 429) {
            sleep((int) ($headers['retry-after'] ?? 60));
            continue; // same page again
        }
        $body = json_decode((string) $raw, true);
        if ($status >= 400) {
            $e = $body['error'] ?? [];
            throw new RuntimeException("{$status} " . ($e['code'] ?? '') . ': ' . ($e['message'] ?? '') . ' (' . ($e['request_id'] ?? '-') . ')');
        }

        yield from $body['data'];

        if (count($body['data']) === 0 || $page >= $body['pagination']['totalPages']) {
            return;
        }
        if (($headers['x-ratelimit-remaining'] ?? null) === '0') {
            sleep((int) ($headers['x-ratelimit-reset'] ?? 60));
        }
        $page++;
    }
}

// Usage
foreach (everyRecord($token, '/student') as $student) {
    echo $student['_id'], ' ', $student['username'], PHP_EOL;
}