Snippets

Copy-and-paste helpers for the Developer API in cURL, JavaScript, Python and Go. The auth header, keyset paging, and retries that follow Retry-After.

Each helper reads the key from HALO_API_KEY. Run them on a server, never in a browser.

The auth header

Send the key on every request, in Authorization, and never in the URL.

curl -sS https://api.usehalo.net/v1/developer/whoami \
  -H "Authorization: Bearer $HALO_API_KEY"

const response = await fetch("https://api.usehalo.net/v1/developer/whoami", {
  headers: { Authorization: `Bearer ${process.env.HALO_API_KEY}` },
});
console.log(await response.json());

import os

import requests

session = requests.Session()
session.headers["Authorization"] = f"Bearer {os.environ['HALO_API_KEY']}"
print(session.get("https://api.usehalo.net/v1/developer/whoami", timeout=30).json())

package main

import (
    "fmt"
    "io"
    "log"
    "net/http"
    "os"
)

func main() {
    req, err := http.NewRequest(http.MethodGet, "https://api.usehalo.net/v1/developer/whoami", nil)
    if err != nil {
        log.Fatal(err)
    }
    req.Header.Set("Authorization", "Bearer "+os.Getenv("HALO_API_KEY"))
    res, err := http.DefaultClient.Do(req)
    if err != nil {
        log.Fatal(err)
    }
    defer res.Body.Close()
    body, err := io.ReadAll(res.Body)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(string(body))
}

Retry

Retry only what a retry can fix.

# Retries on 408, 429, 500, 502, 503 and 504, and honors Retry-After.
# It goes by status, so on a query it would also wait out row_budget_exceeded: an hour.
curl -sS --retry 4 https://api.usehalo.net/v1/developer/whoami \
  -H "Authorization: Bearer $HALO_API_KEY"

const API = "https://api.usehalo.net/v1/developer";
const RETRY = new Set(["rate_limited", "query_busy", "source_error", "request_timeout", "internal"]);

export class HaloError extends Error {
  constructor(status, problem) {
    super(`${problem.code ?? status}: ${problem.detail ?? "no detail"}`);
    this.status = status;
    this.code = problem.code;
    this.requestId = problem.requestId;
  }
}

export async function halo(path, body, attempts = 5) {
  for (let attempt = 1; ; attempt++) {
    const response = await fetch(`${API}${path}`, {
      method: body ? "POST" : "GET",
      headers: {
        Authorization: `Bearer ${process.env.HALO_API_KEY}`,
        ...(body ? { "Content-Type": "application/json" } : {}),
      },
      body: body ? JSON.stringify(body) : undefined,
    });
    if (response.ok) return response.json();

    const problem = await response.json().catch(() => ({}));
    if (!RETRY.has(problem.code) || attempt === attempts) throw new HaloError(response.status, problem);

    const retryAfter = Number(response.headers.get("Retry-After"));
    const seconds = retryAfter > 0 ? retryAfter : Math.min(2 ** attempt, 30) + Math.random();
    await new Promise((resolve) => setTimeout(resolve, seconds * 1000));
  }
}

import os
import random
import time

import requests

API = "https://api.usehalo.net/v1/developer"
RETRY = {"rate_limited", "query_busy", "source_error", "request_timeout", "internal"}

session = requests.Session()
session.headers["Authorization"] = f"Bearer {os.environ['HALO_API_KEY']}"


class HaloError(Exception):
    def __init__(self, status, problem):
        super().__init__(f"{problem.get('code', status)}: {problem.get('detail', 'no detail')}")
        self.status = status
        self.code = problem.get("code")
        self.request_id = problem.get("requestId")


def halo(path, body=None, attempts=5):
    method = "POST" if body is not None else "GET"
    for attempt in range(1, attempts + 1):
        response = session.request(method, f"{API}{path}", json=body, timeout=330)
        if response.ok:
            return response.json()
        try:
            problem = response.json()
        except ValueError:
            problem = {}
        if problem.get("code") not in RETRY or attempt == attempts:
            raise HaloError(response.status_code, problem)
        retry_after = response.headers.get("Retry-After")
        time.sleep(float(retry_after) if retry_after else min(2**attempt, 30) + random.random())

package halo

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "math/rand/v2"
    "net/http"
    "os"
    "slices"
    "strconv"
    "time"
)

const api = "https://api.usehalo.net/v1/developer"

var retry = []string{"rate_limited", "query_busy", "source_error", "request_timeout", "internal"}

// Error is a problem response.
type Error struct {
    Status    int    `json:"status"`
    Code      string `json:"code"`
    Detail    string `json:"detail"`
    RequestID string `json:"requestId"`
}

func (e *Error) Error() string {
    return fmt.Sprintf("%s: %s (request %s)", e.Code, e.Detail, e.RequestID)
}

// Call makes a request, and decodes the answer into out. A nil body is a GET.
func Call(ctx context.Context, path string, body, out any) error {
    var raw []byte
    method := http.MethodGet
    if body != nil {
        method = http.MethodPost
        var err error
        if raw, err = json.Marshal(body); err != nil {
            return err
        }
    }
    const attempts = 5
    for attempt := 1; ; attempt++ {
        req, err := http.NewRequestWithContext(ctx, method, api+path, bytes.NewReader(raw))
        if err != nil {
            return err
        }
        req.Header.Set("Authorization", "Bearer "+os.Getenv("HALO_API_KEY"))
        if body != nil {
            req.Header.Set("Content-Type", "application/json")
        }
        res, err := http.DefaultClient.Do(req)
        if err != nil {
            return err
        }
        data, err := io.ReadAll(res.Body)
        _ = res.Body.Close()
        if err != nil {
            return err
        }
        if res.StatusCode < 300 {
            return json.Unmarshal(data, out)
        }

        problem := &Error{Status: res.StatusCode}
        _ = json.Unmarshal(data, problem)
        if !slices.Contains(retry, problem.Code) || attempt == attempts {
            return problem
        }
        wait := min(time.Duration(1<<attempt)*time.Second, 30*time.Second) + time.Duration(rand.Int64N(int64(time.Second)))
        if seconds, err := strconv.Atoi(res.Header.Get("Retry-After")); err == nil && seconds > 0 {
            wait = time.Duration(seconds) * time.Second
        }
        select {
        case <-time.After(wait):
        case <-ctx.Done():
            return ctx.Err()
        }
    }
}

Page through every row

Send keyset: true on every window, and pass back offset and cursor from the window before. See Keyset paging.

query='{
  "dimensions": [{ "field": "order_date" }, { "field": "region" }],
  "measures": [{ "field": "revenue" }],
  "limit": 1000,
  "keyset": true
}'
offset=0
after=""
while :; do
  body=$(jq -c --argjson offset "$offset" --arg after "$after" \
    '. + { offset: $offset } + (if $after == "" then {} else { cursor: $after } end)' <<<"$query")
  page=$(curl -sS --fail-with-body -X POST "https://api.usehalo.net/v1/developer/datasets/$DATASET_ID/query" \
    -H "Authorization: Bearer $HALO_API_KEY" \
    -H "Content-Type: application/json" \
    -d "$body") || { echo "$page" >&2; exit 1; }
  jq -c '.rows[]' <<<"$page"
  [ "$(jq -r '.stats.hasMore' <<<"$page")" = "true" ] || break
  offset=$((offset + $(jq -r '.stats.rowCount' <<<"$page")))
  after=$(jq -r '.stats.nextCursor // ""' <<<"$page")
done

These use the halo helpers above.

export async function* allRows(datasetId, query) {
  let offset = 0;
  let after;
  while (true) {
    const page = await halo(`/datasets/${datasetId}/query`, {
      ...query,
      limit: 1000,
      keyset: true,
      offset,
      ...(after ? { cursor: after } : {}),
    });
    const keys = page.columns.map((column) => column.key);
    for (const row of page.rows) {
      yield Object.fromEntries(keys.map((key, i) => [key, row[i]]));
    }
    if (!page.stats.hasMore) return;
    offset += page.stats.rowCount;
    after = page.stats.nextCursor;
  }
}

for await (const row of allRows(process.env.HALO_DATASET_ID, {
  dimensions: [{ field: "order_date" }, { field: "region" }],
  measures: [{ field: "revenue" }],
})) {
  console.log(row);
}

def all_rows(dataset_id, query):
    offset, after = 0, None
    while True:
        window = {**query, "limit": 1000, "keyset": True, "offset": offset}
        if after:
            window["cursor"] = after
        page = halo(f"/datasets/{dataset_id}/query", window)
        keys = [column["key"] for column in page["columns"]]
        for row in page["rows"]:
            yield dict(zip(keys, row))
        if not page["stats"]["hasMore"]:
            return
        offset += page["stats"]["rowCount"]
        after = page["stats"].get("nextCursor")


for row in all_rows(os.environ["HALO_DATASET_ID"], {
    "dimensions": [{"field": "order_date"}, {"field": "region"}],
    "measures": [{"field": "revenue"}],
}):
    print(row)

// Page is one window of a query's answer.
type Page struct {
    Columns []struct {
        Key string `json:"key"`
    } `json:"columns"`
    Rows  [][]any `json:"rows"`
    Stats struct {
        RowCount int    `json:"rowCount"`
        HasMore  bool   `json:"hasMore"`
        Cursor   string `json:"nextCursor"`
    } `json:"stats"`
}

// AllRows calls yield for each window of a query, until there are no more.
// query is the body, as a map, without limit, keyset, offset or cursor.
func AllRows(ctx context.Context, datasetID string, query map[string]any, yield func(Page) error) error {
    offset, after := 0, ""
    for {
        window := map[string]any{"limit": 1000, "keyset": true, "offset": offset}
        for k, v := range query {
            window[k] = v
        }
        if after != "" {
            window["cursor"] = after
        }
        var page Page
        if err := Call(ctx, "/datasets/"+datasetID+"/query", window, &page); err != nil {
            return err
        }
        if err := yield(page); err != nil {
            return err
        }
        if !page.Stats.HasMore {
            return nil
        }
        offset += page.Stats.RowCount
        after = page.Stats.Cursor
    }
}

Read limits before a big job

curl -sS https://api.usehalo.net/v1/developer/whoami \
  -H "Authorization: Bearer $HALO_API_KEY" | jq '.limits.rowBudget'

left is what you can still read in the rolling 24 hours. See Rate limits.