Derived Measures

Explore how measures of linguistic productivity and lexical diversity change with age, for both children and their conversational partners. Pick a collection, then filter by corpus and speaker role — transcript-level statistics are binned and averaged in your browser.

Plot = import("https://esm.sh/@observablehq/plot@0.6.17")
d3 = require("d3@7")

collections = FileAttachment("data/collections.json").json()
corpora_all = FileAttachment("data/corpora.json").json()
collection_names = collections.map((d) => d.name)

// slices are column-oriented JSON ({col: [values]}); expand to row objects
unpack = (cols) => {
  const keys = Object.keys(cols);
  const n = keys.length ? cols[keys[0]].length : 0;
  return Array.from({ length: n }, (_, i) =>
    Object.fromEntries(keys.map((k) => [k, cols[k][i]])));
}

collection_slug = (name) => collections.find((d) => d.name === name)?.slug

loess_points = (data, { x = "age", y = "value", w = null, span = 0.75, grid = 60 } = {}) => {
  const pts = data
    .map((d) => ({ x: +d[x], y: +d[y], w: w ? +d[w] : 1 }))
    .filter((d) => Number.isFinite(d.x) && Number.isFinite(d.y))
    .sort((a, b) => a.x - b.x);
  if (pts.length < 3) return pts.map((d) => ({ [x]: d.x, [y]: d.y }));

  const k = Math.max(2, Math.ceil(span * pts.length));
  const x0 = pts[0].x, x1 = pts[pts.length - 1].x;
  const out = [];
  for (let i = 0; i <= grid; i++) {
    const xt = x0 + (x1 - x0) * (i / grid);
    const dists = pts.map((p) => Math.abs(p.x - xt)).sort((a, b) => a - b);
    const dmax = dists[Math.min(k, dists.length) - 1] || 1e-9;
    let sw = 0, swx = 0, swy = 0, swxx = 0, swxy = 0;
    for (const p of pts) {
      const t = Math.abs(p.x - xt) / dmax;
      if (t >= 1) continue;
      const wt = Math.pow(1 - t * t * t, 3) * p.w;
      sw += wt; swx += wt * p.x; swy += wt * p.y;
      swxx += wt * p.x * p.x; swxy += wt * p.x * p.y;
    }
    if (sw <= 0) continue;
    const denom = sw * swxx - swx * swx;
    const yt = Math.abs(denom) < 1e-12
      ? swy / sw
      : (swy * swxx - swx * swxy) / denom + ((sw * swxy - swx * swy) / denom) * xt;
    out.push({ [x]: xt, [y]: yt });
  }
  return out;
}

// smooth each series (grouped by `series`) and tag points with the group
loess_series = (data, series, opts = {}) => {
  const groups = new Map();
  for (const d of data) {
    const key = d[series];
    if (!groups.has(key)) groups.set(key, []);
    groups.get(key).push(d);
  }
  return [...groups.entries()].flatMap(([key, rows]) =>
    loess_points(rows, opts).map((p) => ({ ...p, [series]: key })));
}

download_csv = (rows, filename) => {
  if (!rows || !rows.length) return html`<span></span>`;
  const cols = Object.keys(rows[0]);
  const esc = (v) => {
    const s = String(v ?? "");
    return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
  };
  const csv = [cols.join(","), ...rows.map((r) => cols.map((c) => esc(r[c])).join(","))].join("\n");
  const url = URL.createObjectURL(new Blob([csv], { type: "text/csv" }));
  return html`<a class="btn btn-sm btn-outline-primary" href=${url} download=${filename}>Download CSV</a>`;
}

// input styling: pill labels, bordered rounded selects (scoped to the ojs
// namespace class so it never leaks into other page elements)
ojs_ns = Inputs.text().classList[0]

html`<style>
  .${ojs_ns} div label {
    background-color: #eaf3fc;
    padding: 0.2rem 0.5rem;
    border-radius: 0.4rem;
    margin-bottom: 0.25rem;
    width: auto;
  }

  .${ojs_ns} select,
  .${ojs_ns} input[type="text"],
  .${ojs_ns} input[type="file"] {
    background-color: #fff;
    border: 1px solid #d8dee4;
    border-radius: 0.4rem;
    padding: 0.2rem 0.4rem;
  }

  .${ojs_ns} input[type="checkbox"] {
    accent-color: #3399f3;
    margin-bottom: 0;
  }
</style>`

function interval(range = [], options = {}) {
  const [min = 0, max = 1] = range;
  const {
    step = .001,
    label = null,
    value = [min, max],
    format = ([start, end]) => `${start} … ${end}`,
    color,
    width,
    theme,
    __ns__ = randomScope(),
  } = options;

  const css = `
#${__ns__} {
  //font: 13px/1.2 var(--sans-serif);
  display: flex;
  align-items: baseline;
  flex-wrap: wrap;
  max-width: 100%;
  width: auto;
  flex-direction: column;
}
@media only screen and (min-width: 30em) {
  #${__ns__} {
    flex-wrap: nowrap;
    width: 360px;
  }
}
#${__ns__} .label {
  //width: 120px;
  //padding: 5px 0 4px 0;
  //padding: 5px 0 0 0;
  //margin-right: 6.5px;
  flex-shrink: 0;
}
#${__ns__} .form {
  display: flex;
  width: 100%;
}
#${__ns__} .range {
  flex-shrink: 1;
  width: 100%;
}
#${__ns__} .range-slider {
  width: 100%;
  margin-bottom: .3em;
  margin-top: .3em;
}
  `;
  
  const $range = rangeInput({min, max, value: [value[0], value[1]], step, color, width, theme});
  const $output = ihtml`<output>`;
  const $view = ihtml`<div id=${__ns__}>
${label == null ? '' : ihtml`<div class="label">${label}`}
<div class=form>
  <div class=range>
    ${$range}<div class=range-output>${$output}</div>
  </div>
</div>
${ihtml`<style>${css}`}
  `;

  const update = () => {
    const content = format([$range.value[0], $range.value[1]]);
    if(typeof content === 'string') $output.value = content;
    else {
      while($output.lastChild) $output.lastChild.remove();
      $output.appendChild(content);
    }
  };
  $range.oninput = update;
  update();
  
  return Object.defineProperty($view, 'value', {
    get: () => $range.value,
    set: ([a, b]) => {
      $range.value = [a, b];
      update();
    },
  });
}

function rangeInput(options = {}) {
  const {
    min = 0,
    max = 100,
    step = 'any',
    value: defaultValue = [min, max],
    color,
    width,
    theme = theme_Flat,
  } = options;
  
  const controls = {};
  const scope = randomScope();
  const clamp = (a, b, v) => v < a ? a : v > b ? b : v;

  // Will be used to sanitize values while avoiding floating point issues.
  const input = ihtml`<input type=range ${{min, max, step}}>`;
  
  const dom = ihtml`<div class=${`${scope} range-slider`} style=${{
    color,
    width: cssLength(width),
  }}>
  ${controls.track = ihtml`<div class="range-track">
    ${controls.zone = ihtml`<div class="range-track-zone">
      ${controls.range = ihtml`<div class="range-select" tabindex=0>
        ${controls.min = ihtml`<div class="thumb thumb-min" tabindex=0>`}
        ${controls.max = ihtml`<div class="thumb thumb-max" tabindex=0>`}
      `}
    `}
  `}
  ${ihtml`<style>${theme.replace(/:scope\b/g, '.'+scope)}`}
</div>`;

  let value = [], changed = false;
  Object.defineProperty(dom, 'value', {
    get: () => [...value],
    set: ([a, b]) => {
      value = sanitize(a, b);
      updateRange();
    },
  });

  const sanitize = (a, b) => {
    a = isNaN(a) ? min : ((input.value = a), input.valueAsNumber);
    b = isNaN(b) ? max : ((input.value = b), input.valueAsNumber);
    return [Math.min(a, b), Math.max(a, b)];
  }
  
  const updateRange = () => {
    const ratio = v => (v - min) / (max - min);
    dom.style.setProperty('--range-min', `${ratio(value[0]) * 100}%`);
    dom.style.setProperty('--range-max', `${ratio(value[1]) * 100}%`);
  };

  const dispatch = name => {
    dom.dispatchEvent(new Event(name, {bubbles: true}));
  };
  const setValue = (vmin, vmax) => {
    const [pmin, pmax] = value;
    value = sanitize(vmin, vmax);
    updateRange();
    // Only dispatch if values have changed.
    if(pmin === value[0] && pmax === value[1]) return;
    dispatch('input');
    changed = true;
  };
  
  setValue(...defaultValue);
  
  // Mousemove handlers.
  const handlers = new Map([
    [controls.min, (dt, ov) => {
      const v = clamp(min, ov[1], ov[0] + dt * (max - min));
      setValue(v, ov[1]);
    }],
    [controls.max, (dt, ov) => {
      const v = clamp(ov[0], max, ov[1] + dt * (max - min));
      setValue(ov[0], v);
    }],
    [controls.range, (dt, ov) => {
      const d = ov[1] - ov[0];
      const v = clamp(min, max - d, ov[0] + dt * (max - min));
      setValue(v, v + d);
    }],
  ]);
  
  // Returns client offset object.
  const pointer = e => e.touches ? e.touches[0] : e;
  // Note: Chrome defaults "passive" for touch events to true.
  const on  = (e, fn) => e.split(' ').map(e => document.addEventListener(e, fn, {passive: false}));
  const off = (e, fn) => e.split(' ').map(e => document.removeEventListener(e, fn, {passive: false}));
  
  let initialX, initialV, target, dragging = false;
  function handleDrag(e) {
    // Gracefully handle exit and reentry of the viewport.
    if(!e.buttons && !e.touches) {
      handleDragStop();
      return;
    }
    dragging = true;
    const w = controls.zone.getBoundingClientRect().width;
    e.preventDefault();
    handlers.get(target)((pointer(e).clientX - initialX) / w, initialV);
  }
  
  
  function handleDragStop(e) {
    off('mousemove touchmove', handleDrag);
    off('mouseup touchend', handleDragStop);
    if(changed) dispatch('change');
  }
  
  invalidation.then(handleDragStop);
  
  dom.ontouchstart = dom.onmousedown = e => {
    dragging = false;
    changed = false;
    if(!handlers.has(e.target)) return;
    on('mousemove touchmove', handleDrag);
    on('mouseup touchend', handleDragStop);
    e.preventDefault();
    e.stopPropagation();
    
    target = e.target;
    initialX = pointer(e).clientX;
    initialV = value.slice();
  };
  
  controls.track.onclick = e => {
    if(dragging) return;
    changed = false;
    const r = controls.zone.getBoundingClientRect();
    const t = clamp(0, 1, (pointer(e).clientX - r.left) / r.width);
    const v = min + t * (max - min);
    const [vmin, vmax] = value, d = vmax - vmin;
    if(v < vmin) setValue(v, v + d);
    else if(v > vmax) setValue(v - d, v);
    if(changed) dispatch('change');
  };
  
  return dom;
}

function randomScope(prefix = 'scope-') {
  return prefix + (performance.now() + Math.random()).toString(32).replace('.', '-');
}

cssLength = v => v == null ? null : typeof v === 'number' ? `${v}px` : `${v}`

// NOTE: the upstream notebook re-binds the page-wide `html` to htl.html.
// In Quarto's OJS runtime `htl` is not a builtin, so that line hangs the
// whole page silently; and Quarto's stdlib `html` is the old non-htl
// implementation, which drops the attribute interpolations this component
// relies on (${{min, max, step}}, class=${...}). We therefore import htl
// explicitly and use it here under the component-scoped name `ihtml`.
htl = import("https://esm.sh/htl@0.3.1")

ihtml = htl.html

theme_Flat = `
/* Options */
:scope {
  color: #3b99fc;
  width: 240px;
}

:scope {
  position: relative;
  display: inline-block;
  --thumb-size: 15px;
  --thumb-radius: calc(var(--thumb-size) / 2);
  //padding: var(--thumb-radius) 0;
  margin: 2px;
  vertical-align: middle;
}
:scope .range-track {
  box-sizing: border-box;
  position: relative;
  height: 7px;
  background-color: hsl(0, 0%, 80%);
  overflow: visible;
  border-radius: 4px;
  padding: 0 var(--thumb-radius);
}
:scope .range-track-zone {
  box-sizing: border-box;
  position: relative;
}
:scope .range-select {
  box-sizing: border-box;
  position: relative;
  left: var(--range-min);
  width: calc(var(--range-max) - var(--range-min));
  cursor: ew-resize;
  background: currentColor;
  height: 7px;
  border: inherit;
}
/* Expands the hotspot area. */
:scope .range-select:before {
  content: "";
  position: absolute;
  width: 100%;
  height: var(--thumb-size);
  left: 0;
  top: calc(2px - var(--thumb-radius));
}
:scope .range-select:focus,
:scope .thumb:focus {
  outline: none;
}
:scope .thumb {
  box-sizing: border-box;
  position: absolute;
  width: var(--thumb-size);
  height: var(--thumb-size);

  background: #fcfcfc;
  top: -4px;
  border-radius: 100%;
  border: 1px solid hsl(0,0%,55%);
  cursor: default;
  margin: 0;
}
:scope .thumb:active {
  box-shadow: inset 0 var(--thumb-size) #0002;
}
:scope .thumb-min {
  left: calc(-1px - var(--thumb-radius));
}
:scope .thumb-max {
  right: calc(-1px - var(--thumb-radius));
}
`

measure_options = new Map([
  ["MLU-w (mean length of utterance in words)", "mlu_w"],
  ["MLU-m (mean length of utterance in morphemes)", "mlu_m"],
  ["TTR (type-token ratio)", "ttr"],
  ["MTLD (measure of textual lexical diversity)", "mtld"],
  ["HD-D (lexical diversity via hypergeometric distribution)", "hdd"]
])

measure_label = (key) =>
  [...measure_options].find(([, v]) => v === key)?.[0] ?? key

viewof dm_collection = Inputs.select(collection_names, {
  label: "Collection",
  value: "Eng-NA"
})

dm_slice = d3.json(`slices/speaker_stats/${collection_slug(dm_collection)}.json`)
  .then(unpack)

dm_corpus_options = [...new Set(
  corpora_all
    .filter((d) => d.collection_name === dm_collection)
    .map((d) => d.name)
)].sort()

// roles present in this collection, most common first
dm_role_options = d3.groupSort(dm_slice, (v) => -v.length, (d) => d.speaker_role)

dm_default_roles = {
  const present = ["Target_Child", "Mother"].filter((r) => dm_role_options.includes(r));
  return present.length ? present : dm_role_options.slice(0, 1);
}

viewof dm_corpora = Inputs.select(dm_corpus_options, {
  label: "Corpora",
  multiple: 8,
  value: dm_corpus_options
})
viewof dm_roles = Inputs.select(dm_role_options, {
  label: "Speaker roles",
  multiple: 6,
  value: dm_default_roles
})
viewof dm_measure = Inputs.select(measure_options, { label: "Measure" })
viewof dm_ages = interval([0, 18], { step: 0.5, value: [0, 18], label: "Ages (years)" })
viewof dm_binwidth = Inputs.range([1, 12], { label: "Bin width (months)", step: 1, value: 2 })

Loading data — the first load can take a few seconds…

// call once the first visualization has rendered
clear_loading = () => document.getElementById("cdb-loading")?.remove()

Plot

dm_filtered = {
  const corpus_set = new Set(dm_corpora);
  const role_set = new Set(dm_roles);
  const lo = dm_ages[0] * 12;
  const hi = dm_ages[1] * 12;
  return dm_slice
    .filter((d) =>
      corpus_set.has(d.corpus_name) &&
      role_set.has(d.speaker_role) &&
      d.age != null && d.age >= lo && d.age <= hi)
    .map((d) => ({
      ...d,
      value: dm_measure === "ttr"
        ? (d.num_tokens > 0 ? d.num_types / d.num_tokens : null)
        : d[dm_measure]
    }))
    // MTLD and HD-D are undefined for speakers with < 50 tokens; drop nulls
    .filter((d) => d.value != null && Number.isFinite(d.value));
}

dm_binned = {
  const bw = dm_binwidth;
  const out = [];
  for (const [role, byRole] of d3.groups(dm_filtered, (d) => d.speaker_role)) {
    for (const [bin, cell] of d3.groups(byRole, (d) => Math.floor(d.age / bw))) {
      out.push({
        speaker_role: role,
        measure: dm_measure,
        age: bin * bw + bw / 2,
        mean: d3.mean(cell, (d) => d.value),
        n_transcripts: cell.length,
        n_utterances: d3.sum(cell, (d) => d.num_utterances)
      });
    }
  }
  return out.sort((a, b) =>
    d3.ascending(a.speaker_role, b.speaker_role) || d3.ascending(a.age, b.age));
}

dm_plot_roles = [...new Set(dm_binned.map((d) => d.speaker_role))]

{
  clear_loading();
  if (!dm_binned.length) {
    return html`<em>No data for this selection — try widening the age range
      or adding corpora / speaker roles.</em>`;
  }
  const smooth = loess_series(dm_binned, "speaker_role",
    { x: "age", y: "mean", w: "n_utterances" });
  return Plot.plot({
    style: { fontFamily: "var(--sans-serif)", fontSize: "13px" },
    width: 850,
    height: 500,
    inset: 8,
    grid: true,
    x: { label: "Target child age (months)", tickFormat: "d", line: true },
    y: { label: measure_label(dm_measure), line: true },
    color: { legend: true, domain: dm_plot_roles },
    r: { range: [2, 10], label: "Utterances" },
    marks: [
      Plot.lineY(smooth, {
        x: "age", y: "mean", stroke: "speaker_role", strokeWidth: 2.5
      }),
      Plot.dot(dm_binned, {
        x: "age", y: "mean", r: "n_utterances", fill: "speaker_role",
        fillOpacity: 0.6,
        channels: { n_transcripts: "n_transcripts", n_utterances: "n_utterances" },
        tip: true
      })
    ]
  });
}

Points are age-bin means; point area is proportional to the total number of utterances in the bin. Curves are LOESS smooths weighted by utterance count.

Table

Inputs.table(dm_binned, { maxWidth: 900 })

download_csv(dm_binned,
  `childes_derived_${collection_slug(dm_collection)}_${dm_measure}.csv`)

Note

Measures are computed per speaker per transcript in childes-db 2026.1: MLU in words and morphemes, type-token ratio (types / tokens), and the lexical-diversity measures MTLD and HD-D (only defined for speakers who produced at least 50 tokens in a transcript). Ages are converted from days to months as in childesr.