Bigram Browser

What words appear next to a given word in speech to and from children? Type a word, pick a collection, and see the words that most often occur immediately before and after it, split by corpus and speaker role. Counts are case-insensitive and come from adjacent word pairs within utterances.

Plot = import("https://esm.sh/@observablehq/plot@0.6.17")
d3 = require("d3@7")

collections = FileAttachment("data/collections.json").json()
corpora_all = FileAttachment("data/corpora.json").json()
collection_names = collections.map((d) => d.name)

// slices are column-oriented JSON ({col: [values]}); expand to row objects
unpack = (cols) => {
  const keys = Object.keys(cols);
  const n = keys.length ? cols[keys[0]].length : 0;
  return Array.from({ length: n }, (_, i) =>
    Object.fromEntries(keys.map((k) => [k, cols[k][i]])));
}

collection_slug = (name) => collections.find((d) => d.name === name)?.slug

download_csv = (rows, filename) => {
  if (!rows || !rows.length) return html`<span></span>`;
  const cols = Object.keys(rows[0]);
  const esc = (v) => {
    const s = String(v ?? "");
    return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
  };
  const csv = [cols.join(","), ...rows.map((r) => cols.map((c) => esc(r[c])).join(","))].join("\n");
  const url = URL.createObjectURL(new Blob([csv], { type: "text/csv" }));
  return html`<a class="btn btn-sm btn-outline-primary" href=${url} download=${filename}>Download CSV</a>`;
}

// input styling: pill labels, bordered rounded selects (scoped to the ojs
// namespace class so it never leaks into other page elements)
ojs_ns = Inputs.text().classList[0]

html`<style>
  .${ojs_ns} div label {
    background-color: #eaf3fc;
    padding: 0.2rem 0.5rem;
    border-radius: 0.4rem;
    margin-bottom: 0.25rem;
    width: auto;
  }

  .${ojs_ns} select,
  .${ojs_ns} input[type="text"],
  .${ojs_ns} input[type="file"] {
    background-color: #fff;
    border: 1px solid #d8dee4;
    border-radius: 0.4rem;
    padding: 0.2rem 0.4rem;
  }

  .${ojs_ns} input[type="checkbox"] {
    accent-color: #3399f3;
    margin-bottom: 0;
  }
</style>`

duckdb = import("https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm")

db = {
  const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
  const worker = await duckdb.createWorker(bundle.mainWorker);
  const database = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
  await database.instantiate(bundle.mainModule, bundle.pthreadWorker);
  return database;
}
conn = db.connect()
registered = new Set()

sql = async (q) => (await (await conn).query(q)).toArray().map((r) => ({ ...r.toJSON() }))
quote_list = (xs) => xs.map((x) => `'${String(x).replace(/'/g, "''")}'`).join(", ")

// fetch and register a parquet file's bytes once (virtual name = its site
// path, so freq and bigram shards never collide); buffers cached across
// queries. Returns the name to use in read_parquet().
register_parquet = async (dir, fname) => {
  const vname = `${dir}/${fname}`;
  if (!registered.has(vname)) {
    const buf = new Uint8Array(
      await fetch(vname).then((r) => {
        if (!r.ok) throw new Error(`failed to fetch ${vname}`);
        return r.arrayBuffer();
      }));
    await (await db).registerFileBuffer(vname, buf);
    registered.add(vname);
  }
  return vname;
}

normGloss = (s) => s.normalize("NFC").toLowerCase()

fnv1a = (s) => {
  let h = 0x811c9dc5;
  for (const b of new TextEncoder().encode(s)) {
    h ^= b;
    h = Math.imul(h, 0x01000193) >>> 0;
  }
  return h >>> 0;
}

bg_register_shard = (s) => register_parquet("slices/bigrams", `shard-${s}.parquet`)

bg_hash_check = FileAttachment("slices/bigrams/hash_check.json").json()

bg_hash_ok = {
  for (const v of bg_hash_check.vectors) {
    const lower = normGloss(v.gloss);
    const shard = fnv1a(lower) % bg_hash_check.n_shards;
    if (lower !== v.gloss_lower || shard !== v.shard) {
      throw new Error(
        `gloss hash mismatch for "${v.gloss}": JS says (${lower}, shard ${shard}), ` +
        `build says (${v.gloss_lower}, shard ${v.shard}) — ` +
        `etl/write_bigram_shards.py and bigrams.qmd have drifted`);
    }
  }
  return true;
}

viewof bg_word_raw = Inputs.text({
  label: "Word",
  value: "dog",
  submit: true
})
viewof bg_collection = Inputs.select(collection_names, {
  label: "Collection",
  value: "Eng-NA"
})

bg_corpus_options = [...new Set(
  corpora_all
    .filter((d) => d.collection_name === bg_collection)
    .map((d) => d.name)
)].sort()

viewof bg_corpora = Inputs.select(bg_corpus_options, {
  label: "Corpora",
  multiple: 6,
  value: bg_corpus_options
})

// roles that actually occur with this word/collection, biggest first
bg_role_options = d3.groupSort(bg_counts,
  (v) => -d3.sum(v, (d) => d.n), (d) => d.speaker_role)

viewof bg_roles = Inputs.select(bg_role_options, {
  label: "Speaker roles",
  multiple: 5,
  value: bg_role_options.includes("Target_Child")
    ? ["Target_Child"] : bg_role_options.slice(0, 1)
})
viewof bg_topn = Inputs.range([10, 50], {
  label: "Top N words", step: 5, value: 20
})
viewof bg_min = Inputs.range([2, 50], {
  label: "Minimum count", step: 1, value: 2
})

Loading data — the first load can take a few seconds…

// call once the first visualization has rendered
clear_loading = () => document.getElementById("cdb-loading")?.remove()

Plot

bg_word = normGloss(bg_word_raw.trim())

// lazy shard load + one DuckDB query per (word, collection); corpus / role /
// top-N / min-count filters below run reactively in JS on this result
bg_counts = {
  if (!bg_hash_ok || !bg_word) return [];
  const file = await bg_register_shard(fnv1a(bg_word) % bg_hash_check.n_shards);
  const rows = await sql(`
    SELECT pos, word, corpus_name, speaker_role, SUM(n)::INT AS n
    FROM read_parquet('${file}')
    WHERE anchor = '${bg_word.replace(/'/g, "''")}'
      AND collection_name = '${bg_collection.replace(/'/g, "''")}'
    GROUP BY 1, 2, 3, 4`);
  return rows.map((d) => ({ ...d, n: Number(d.n) }));
}

BG_SEP = "\u0001"

// sum over the selected corpora and roles, then apply the min-count filter
bg_filtered = {
  const corpus_set = new Set(bg_corpora);
  const role_set = new Set(bg_roles);
  const agg = new Map();
  for (const d of bg_counts) {
    if (!corpus_set.has(d.corpus_name) || !role_set.has(d.speaker_role)) continue;
    const k = d.pos + BG_SEP + d.word;
    agg.set(k, (agg.get(k) ?? 0) + d.n);
  }
  const out = [];
  for (const [k, n] of agg) {
    if (n < bg_min) continue;
    const [pos, word] = k.split(BG_SEP);
    out.push({ word: bg_word, pos, neighbor: word, n });
  }
  return out.sort((a, b) =>
    d3.ascending(a.pos, b.pos) || d3.descending(a.n, b.n) ||
    d3.ascending(a.neighbor, b.neighbor));
}

bg_before = bg_filtered.filter((d) => d.pos === "before").slice(0, bg_topn)
bg_after = bg_filtered.filter((d) => d.pos === "after").slice(0, bg_topn)

{
  clear_loading();
  if (!bg_word) {
    return html`<em>Type a word and press Enter.</em>`;
  }
  if (!bg_counts.length) {
    return html`<div style="color:#8a6d3b;background:#fcf8e3;border:1px solid #faebcc;
        border-radius:4px;padding:0.6rem 0.9rem">
        No occurrences of “${bg_word}” in the ${bg_collection} collection
        (words are matched on the lowercased gloss, and pairs occurring only
        once are not included). Try another word or collection.</div>`;
  }
  if (!bg_before.length && !bg_after.length) {
    return html`<em>No neighboring words pass the current corpus, speaker-role,
      and minimum-count filters — try selecting more corpora or roles, or
      lowering the minimum count.</em>`;
  }

  const panel = (rows, title) => {
    if (!rows.length) {
      return html`<div style="width:400px;font-size:13px"><strong>${title}</strong>
        <br><em>no pairs pass the current filters</em></div>`;
    }
    return Plot.plot({
      title,
      style: { fontFamily: "var(--sans-serif)", fontSize: "13px" },
      width: 410,
      height: 70 + rows.length * 19,
      marginLeft: 110,
      marginRight: 40,
      x: { label: "count", grid: true },
      y: { label: null },
      marks: [
        Plot.barX(rows, {
          x: "n", y: "neighbor", fill: "#3399f3",
          sort: { y: "-x" }, tip: true
        }),
        Plot.textX(rows, {
          x: "n", y: "neighbor", text: (d) => d.n.toLocaleString(),
          dx: 4, textAnchor: "start", fill: "#555"
        }),
        Plot.ruleX([0])
      ]
    });
  };

  return html`<div style="display:flex;flex-wrap:wrap;gap:1.5rem;align-items:flex-start">
    ${panel(bg_before, `words before “${bg_word}”`)}
    ${panel(bg_after, `words after “${bg_word}”`)}
  </div>`;
}

Bars show, for the selected corpora and speaker roles, the words most often adjacent to the queried word within an utterance, in descending order of total count.

Table

Inputs.table(bg_filtered, { maxWidth: 900 })

download_csv(bg_filtered,
  `childes_bigrams_${collection_slug(bg_collection)}_${bg_word}.csv`)

NoteAbout these counts

Pairs are adjacent words within a single utterance in the childes-db 2026.1 token table, matched on the NFC-normalized, lowercased gloss; pairs touching an unintelligible or untranscribed gloss (xxx, yyy, www) are excluded. Counts are precomputed at the corpus level, so there is no per-child filtering here — use childesr for child-level analyses of utterance context. Word pairs occurring only once in a (collection, corpus, speaker role) cell are pruned from the precomputed tables, so the minimum observable count is 2. The first query initializes an in-browser database (~10 s); later queries fetch only the ~1 MB shard containing the queried word.