Population Viewer

How much speech does each transcript contain, and how is it distributed over age? Each point is one transcript (summing over the selected speaker roles), colored by corpus — a quick way to see what data exist for a collection.

Plot = import("https://esm.sh/@observablehq/plot@0.6.17")
d3 = require("d3@7")

collections = FileAttachment("data/collections.json").json()
corpora_all = FileAttachment("data/corpora.json").json()
collection_names = collections.map((d) => d.name)

// slices are column-oriented JSON ({col: [values]}); expand to row objects
unpack = (cols) => {
  const keys = Object.keys(cols);
  const n = keys.length ? cols[keys[0]].length : 0;
  return Array.from({ length: n }, (_, i) =>
    Object.fromEntries(keys.map((k) => [k, cols[k][i]])));
}

collection_slug = (name) => collections.find((d) => d.name === name)?.slug

loess_points = (data, { x = "age", y = "value", w = null, span = 0.75, grid = 60 } = {}) => {
  const pts = data
    .map((d) => ({ x: +d[x], y: +d[y], w: w ? +d[w] : 1 }))
    .filter((d) => Number.isFinite(d.x) && Number.isFinite(d.y))
    .sort((a, b) => a.x - b.x);
  if (pts.length < 3) return pts.map((d) => ({ [x]: d.x, [y]: d.y }));

  const k = Math.max(2, Math.ceil(span * pts.length));
  const x0 = pts[0].x, x1 = pts[pts.length - 1].x;
  const out = [];
  for (let i = 0; i <= grid; i++) {
    const xt = x0 + (x1 - x0) * (i / grid);
    const dists = pts.map((p) => Math.abs(p.x - xt)).sort((a, b) => a - b);
    const dmax = dists[Math.min(k, dists.length) - 1] || 1e-9;
    let sw = 0, swx = 0, swy = 0, swxx = 0, swxy = 0;
    for (const p of pts) {
      const t = Math.abs(p.x - xt) / dmax;
      if (t >= 1) continue;
      const wt = Math.pow(1 - t * t * t, 3) * p.w;
      sw += wt; swx += wt * p.x; swy += wt * p.y;
      swxx += wt * p.x * p.x; swxy += wt * p.x * p.y;
    }
    if (sw <= 0) continue;
    const denom = sw * swxx - swx * swx;
    const yt = Math.abs(denom) < 1e-12
      ? swy / sw
      : (swy * swxx - swx * swxy) / denom + ((sw * swxy - swx * swy) / denom) * xt;
    out.push({ [x]: xt, [y]: yt });
  }
  return out;
}

// smooth each series (grouped by `series`) and tag points with the group
loess_series = (data, series, opts = {}) => {
  const groups = new Map();
  for (const d of data) {
    const key = d[series];
    if (!groups.has(key)) groups.set(key, []);
    groups.get(key).push(d);
  }
  return [...groups.entries()].flatMap(([key, rows]) =>
    loess_points(rows, opts).map((p) => ({ ...p, [series]: key })));
}

download_csv = (rows, filename) => {
  if (!rows || !rows.length) return html`<span></span>`;
  const cols = Object.keys(rows[0]);
  const esc = (v) => {
    const s = String(v ?? "");
    return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
  };
  const csv = [cols.join(","), ...rows.map((r) => cols.map((c) => esc(r[c])).join(","))].join("\n");
  const url = URL.createObjectURL(new Blob([csv], { type: "text/csv" }));
  return html`<a class="btn btn-sm btn-outline-primary" href=${url} download=${filename}>Download CSV</a>`;
}

// input styling: pill labels, bordered rounded selects (scoped to the ojs
// namespace class so it never leaks into other page elements)
ojs_ns = Inputs.text().classList[0]

html`<style>
  .${ojs_ns} div label {
    background-color: #eaf3fc;
    padding: 0.2rem 0.5rem;
    border-radius: 0.4rem;
    margin-bottom: 0.25rem;
    width: auto;
  }

  .${ojs_ns} select,
  .${ojs_ns} input[type="text"],
  .${ojs_ns} input[type="file"] {
    background-color: #fff;
    border: 1px solid #d8dee4;
    border-radius: 0.4rem;
    padding: 0.2rem 0.4rem;
  }

  .${ojs_ns} input[type="checkbox"] {
    accent-color: #3399f3;
    margin-bottom: 0;
  }
</style>`

function interval(range = [], options = {}) {
  const [min = 0, max = 1] = range;
  const {
    step = .001,
    label = null,
    value = [min, max],
    format = ([start, end]) => `${start} … ${end}`,
    color,
    width,
    theme,
    __ns__ = randomScope(),
  } = options;

  const css = `
#${__ns__} {
  //font: 13px/1.2 var(--sans-serif);
  display: flex;
  align-items: baseline;
  flex-wrap: wrap;
  max-width: 100%;
  width: auto;
  flex-direction: column;
}
@media only screen and (min-width: 30em) {
  #${__ns__} {
    flex-wrap: nowrap;
    width: 360px;
  }
}
#${__ns__} .label {
  //width: 120px;
  //padding: 5px 0 4px 0;
  //padding: 5px 0 0 0;
  //margin-right: 6.5px;
  flex-shrink: 0;
}
#${__ns__} .form {
  display: flex;
  width: 100%;
}
#${__ns__} .range {
  flex-shrink: 1;
  width: 100%;
}
#${__ns__} .range-slider {
  width: 100%;
  margin-bottom: .3em;
  margin-top: .3em;
}
  `;
  
  const $range = rangeInput({min, max, value: [value[0], value[1]], step, color, width, theme});
  const $output = ihtml`<output>`;
  const $view = ihtml`<div id=${__ns__}>
${label == null ? '' : ihtml`<div class="label">${label}`}
<div class=form>
  <div class=range>
    ${$range}<div class=range-output>${$output}</div>
  </div>
</div>
${ihtml`<style>${css}`}
  `;

  const update = () => {
    const content = format([$range.value[0], $range.value[1]]);
    if(typeof content === 'string') $output.value = content;
    else {
      while($output.lastChild) $output.lastChild.remove();
      $output.appendChild(content);
    }
  };
  $range.oninput = update;
  update();
  
  return Object.defineProperty($view, 'value', {
    get: () => $range.value,
    set: ([a, b]) => {
      $range.value = [a, b];
      update();
    },
  });
}

function rangeInput(options = {}) {
  const {
    min = 0,
    max = 100,
    step = 'any',
    value: defaultValue = [min, max],
    color,
    width,
    theme = theme_Flat,
  } = options;
  
  const controls = {};
  const scope = randomScope();
  const clamp = (a, b, v) => v < a ? a : v > b ? b : v;

  // Will be used to sanitize values while avoiding floating point issues.
  const input = ihtml`<input type=range ${{min, max, step}}>`;
  
  const dom = ihtml`<div class=${`${scope} range-slider`} style=${{
    color,
    width: cssLength(width),
  }}>
  ${controls.track = ihtml`<div class="range-track">
    ${controls.zone = ihtml`<div class="range-track-zone">
      ${controls.range = ihtml`<div class="range-select" tabindex=0>
        ${controls.min = ihtml`<div class="thumb thumb-min" tabindex=0>`}
        ${controls.max = ihtml`<div class="thumb thumb-max" tabindex=0>`}
      `}
    `}
  `}
  ${ihtml`<style>${theme.replace(/:scope\b/g, '.'+scope)}`}
</div>`;

  let value = [], changed = false;
  Object.defineProperty(dom, 'value', {
    get: () => [...value],
    set: ([a, b]) => {
      value = sanitize(a, b);
      updateRange();
    },
  });

  const sanitize = (a, b) => {
    a = isNaN(a) ? min : ((input.value = a), input.valueAsNumber);
    b = isNaN(b) ? max : ((input.value = b), input.valueAsNumber);
    return [Math.min(a, b), Math.max(a, b)];
  }
  
  const updateRange = () => {
    const ratio = v => (v - min) / (max - min);
    dom.style.setProperty('--range-min', `${ratio(value[0]) * 100}%`);
    dom.style.setProperty('--range-max', `${ratio(value[1]) * 100}%`);
  };

  const dispatch = name => {
    dom.dispatchEvent(new Event(name, {bubbles: true}));
  };
  const setValue = (vmin, vmax) => {
    const [pmin, pmax] = value;
    value = sanitize(vmin, vmax);
    updateRange();
    // Only dispatch if values have changed.
    if(pmin === value[0] && pmax === value[1]) return;
    dispatch('input');
    changed = true;
  };
  
  setValue(...defaultValue);
  
  // Mousemove handlers.
  const handlers = new Map([
    [controls.min, (dt, ov) => {
      const v = clamp(min, ov[1], ov[0] + dt * (max - min));
      setValue(v, ov[1]);
    }],
    [controls.max, (dt, ov) => {
      const v = clamp(ov[0], max, ov[1] + dt * (max - min));
      setValue(ov[0], v);
    }],
    [controls.range, (dt, ov) => {
      const d = ov[1] - ov[0];
      const v = clamp(min, max - d, ov[0] + dt * (max - min));
      setValue(v, v + d);
    }],
  ]);
  
  // Returns client offset object.
  const pointer = e => e.touches ? e.touches[0] : e;
  // Note: Chrome defaults "passive" for touch events to true.
  const on  = (e, fn) => e.split(' ').map(e => document.addEventListener(e, fn, {passive: false}));
  const off = (e, fn) => e.split(' ').map(e => document.removeEventListener(e, fn, {passive: false}));
  
  let initialX, initialV, target, dragging = false;
  function handleDrag(e) {
    // Gracefully handle exit and reentry of the viewport.
    if(!e.buttons && !e.touches) {
      handleDragStop();
      return;
    }
    dragging = true;
    const w = controls.zone.getBoundingClientRect().width;
    e.preventDefault();
    handlers.get(target)((pointer(e).clientX - initialX) / w, initialV);
  }
  
  
  function handleDragStop(e) {
    off('mousemove touchmove', handleDrag);
    off('mouseup touchend', handleDragStop);
    if(changed) dispatch('change');
  }
  
  invalidation.then(handleDragStop);
  
  dom.ontouchstart = dom.onmousedown = e => {
    dragging = false;
    changed = false;
    if(!handlers.has(e.target)) return;
    on('mousemove touchmove', handleDrag);
    on('mouseup touchend', handleDragStop);
    e.preventDefault();
    e.stopPropagation();
    
    target = e.target;
    initialX = pointer(e).clientX;
    initialV = value.slice();
  };
  
  controls.track.onclick = e => {
    if(dragging) return;
    changed = false;
    const r = controls.zone.getBoundingClientRect();
    const t = clamp(0, 1, (pointer(e).clientX - r.left) / r.width);
    const v = min + t * (max - min);
    const [vmin, vmax] = value, d = vmax - vmin;
    if(v < vmin) setValue(v, v + d);
    else if(v > vmax) setValue(v - d, v);
    if(changed) dispatch('change');
  };
  
  return dom;
}

function randomScope(prefix = 'scope-') {
  return prefix + (performance.now() + Math.random()).toString(32).replace('.', '-');
}

cssLength = v => v == null ? null : typeof v === 'number' ? `${v}px` : `${v}`

// NOTE: the upstream notebook re-binds the page-wide `html` to htl.html.
// In Quarto's OJS runtime `htl` is not a builtin, so that line hangs the
// whole page silently; and Quarto's stdlib `html` is the old non-htl
// implementation, which drops the attribute interpolations this component
// relies on (${{min, max, step}}, class=${...}). We therefore import htl
// explicitly and use it here under the component-scoped name `ihtml`.
htl = import("https://esm.sh/htl@0.3.1")

ihtml = htl.html

theme_Flat = `
/* Options */
:scope {
  color: #3b99fc;
  width: 240px;
}

:scope {
  position: relative;
  display: inline-block;
  --thumb-size: 15px;
  --thumb-radius: calc(var(--thumb-size) / 2);
  //padding: var(--thumb-radius) 0;
  margin: 2px;
  vertical-align: middle;
}
:scope .range-track {
  box-sizing: border-box;
  position: relative;
  height: 7px;
  background-color: hsl(0, 0%, 80%);
  overflow: visible;
  border-radius: 4px;
  padding: 0 var(--thumb-radius);
}
:scope .range-track-zone {
  box-sizing: border-box;
  position: relative;
}
:scope .range-select {
  box-sizing: border-box;
  position: relative;
  left: var(--range-min);
  width: calc(var(--range-max) - var(--range-min));
  cursor: ew-resize;
  background: currentColor;
  height: 7px;
  border: inherit;
}
/* Expands the hotspot area. */
:scope .range-select:before {
  content: "";
  position: absolute;
  width: 100%;
  height: var(--thumb-size);
  left: 0;
  top: calc(2px - var(--thumb-radius));
}
:scope .range-select:focus,
:scope .thumb:focus {
  outline: none;
}
:scope .thumb {
  box-sizing: border-box;
  position: absolute;
  width: var(--thumb-size);
  height: var(--thumb-size);

  background: #fcfcfc;
  top: -4px;
  border-radius: 100%;
  border: 1px solid hsl(0,0%,55%);
  cursor: default;
  margin: 0;
}
:scope .thumb:active {
  box-shadow: inset 0 var(--thumb-size) #0002;
}
:scope .thumb-min {
  left: calc(-1px - var(--thumb-radius));
}
:scope .thumb-max {
  right: calc(-1px - var(--thumb-radius));
}
`

pop_measure_options = new Map([
  ["Utterances", "num_utterances"],
  ["Tokens", "num_tokens"]
])

viewof pop_collection = Inputs.select(collection_names, {
  label: "Collection",
  value: "Eng-NA"
})

pop_slice = d3.json(`slices/speaker_stats/${collection_slug(pop_collection)}.json`)
  .then(unpack)

pop_corpus_options = [...new Set(
  corpora_all
    .filter((d) => d.collection_name === pop_collection)
    .map((d) => d.name)
)].sort()

pop_role_options = d3.groupSort(pop_slice, (v) => -v.length, (d) => d.speaker_role)

pop_default_roles = pop_role_options.includes("Target_Child")
  ? ["Target_Child"] : pop_role_options.slice(0, 1)

viewof pop_corpora = Inputs.select(pop_corpus_options, {
  label: "Corpora",
  multiple: 8,
  value: pop_corpus_options
})
viewof pop_roles = Inputs.select(pop_role_options, {
  label: "Speaker roles",
  multiple: 6,
  value: pop_default_roles
})
viewof pop_measure = Inputs.select(pop_measure_options, { label: "Measure" })
viewof pop_ages = interval([0, 18], { step: 0.5, value: [0, 18], label: "Ages (years)" })

Loading data — the first load can take a few seconds…

// call once the first visualization has rendered
clear_loading = () => document.getElementById("cdb-loading")?.remove()

Plot

pop_rows = {
  const corpus_set = new Set(pop_corpora);
  const role_set = new Set(pop_roles);
  const lo = pop_ages[0] * 12;
  const hi = pop_ages[1] * 12;
  const kept = pop_slice.filter((d) =>
    corpus_set.has(d.corpus_name) &&
    role_set.has(d.speaker_role) &&
    d.age != null && d.age >= lo && d.age <= hi);
  return d3.groups(kept, (d) => d.transcript_id)
    .map(([id, cell]) => ({
      transcript_id: id,
      corpus_name: cell[0].corpus_name,
      target_child_id: cell[0].target_child_id,
      target_child_name: cell[0].target_child_name,
      target_child_sex: cell[0].target_child_sex,
      age: cell[0].age,
      value: d3.sum(cell, (d) => d[pop_measure])
    }))
    .sort((a, b) =>
      d3.ascending(a.corpus_name, b.corpus_name) || d3.ascending(a.age, b.age));
}

pop_plot_corpora = [...new Set(pop_rows.map((d) => d.corpus_name))].sort()

{
  clear_loading();
  if (!pop_rows.length) {
    return html`<em>No data for this selection — try widening the age range
      or adding corpora / speaker roles.</em>`;
  }
  const smooth = loess_points(pop_rows, { x: "age", y: "value" });
  return Plot.plot({
    style: { fontFamily: "var(--sans-serif)", fontSize: "13px" },
    width: 850,
    height: 500,
    inset: 8,
    grid: true,
    x: { label: "Target child age (months)", tickFormat: "d", line: true },
    y: { label: `${measure_name(pop_measure)} per transcript`, line: true },
    color: {
      // a legend for a handful of corpora; tooltips carry the corpus otherwise
      legend: pop_plot_corpora.length <= 12,
      domain: pop_plot_corpora
    },
    marks: [
      Plot.dot(pop_rows, {
        x: "age", y: "value", fill: "corpus_name", r: 2.5, fillOpacity: 0.5,
        channels: {
          corpus: "corpus_name",
          child: (d) => d.target_child_name ?? String(d.target_child_id)
        },
        tip: true
      }),
      Plot.lineY(smooth, { x: "age", y: "value", stroke: "#333", strokeWidth: 2.5 })
    ]
  });
}

measure_name = (key) =>
  [...pop_measure_options].find(([, v]) => v === key)?.[0] ?? key

The black curve is an (unweighted) LOESS smooth over all plotted transcripts.

Table

Inputs.table(pop_rows, { maxWidth: 900 })

download_csv(pop_rows,
  `childes_population_${collection_slug(pop_collection)}_${pop_measure}.csv`)

Note

Counts come from the transcript_by_speaker table of childes-db 2026.1. Transcripts without a recorded target-child age are not shown. Ages are converted from days to months as in childesr.