Replicability

Is the field running studies that can find the effects it studies? Bergmann, Tsuji, Piccinini, Lewis, Braginsky, Frank, & Cristia (2018) used MetaLab to diagnose replicability across language acquisition research: typical sample sizes, statistical power, method sensitivity, and small-study effects. This page recomputes those analyses live from the current data release for every dataset — the 2018 paper’s snapshot covered 12 meta-analyses; the numbers below update as MetaLab grows.

Loading data…

// call once the first visualization has rendered
clear_loading = () => document.getElementById("ml-loading")?.remove()

pnorm = (x) => {
  const t = 1 / (1 + 0.3275911 * Math.abs(x) / Math.SQRT2);
  const y = 1 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t -
    0.284496736) * t + 0.254829592) * t * Math.exp(-(x * x) / 2);
  return x >= 0 ? 0.5 * (1 + y) : 0.5 * (1 - y);
}

qnorm975 = 1.959963984540054

// Lanczos log-gamma (self-recursion wrapped: OJS cells cannot reference
// their own name)
lgamma = {
  const g = [676.5203681218851, -1259.1392167224028, 771.32342877765313,
    -176.61502916214059, 12.507343278686905, -0.13857109526572012,
    9.9843695780195716e-6, 1.5056327351493116e-7];
  const f = (z) => {
    if (z < 0.5) return Math.log(Math.PI / Math.sin(Math.PI * z)) - f(1 - z);
    z -= 1;
    let x = 0.99999999999980993;
    for (let i = 0; i < 8; i++) x += g[i] / (z + i + 1);
    const t = z + 7.5;
    return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
  };
  return f;
}

// regularized incomplete beta I_x(a, b) by continued fraction (Lentz)
ibeta = {
  const f = (x, a, b) => {
    if (x <= 0) return 0;
    if (x >= 1) return 1;
    if (x > (a + 1) / (a + b + 2)) return 1 - f(1 - x, b, a);
    const lbeta = lgamma(a) + lgamma(b) - lgamma(a + b);
    const front = Math.exp(a * Math.log(x) + b * Math.log(1 - x) - lbeta) / a;
    let h = 1, c = 1, d = 0;
    for (let i = 0; i <= 300; i++) {
      const m = Math.floor(i / 2);
      let numerator;
      if (i === 0) numerator = 1;
      else if (i % 2 === 0) numerator = (m * (b - m) * x) / ((a + 2 * m - 1) * (a + 2 * m));
      else numerator = -((a + m) * (a + b + m) * x) / ((a + 2 * m) * (a + 2 * m + 1));
      d = 1 + numerator * d;
      if (Math.abs(d) < 1e-30) d = 1e-30;
      d = 1 / d;
      c = 1 + numerator / c;
      if (Math.abs(c) < 1e-30) c = 1e-30;
      h *= c * d;
      if (Math.abs(1 - c * d) < 1e-9) break;
    }
    return front * (h - 1);
  };
  return f;
}

// two-sided p for a t statistic with df degrees of freedom
t_pvalue = (t, df) => {
  const x = df / (df + t * t);
  return ibeta(x, df / 2, 0.5);
}

// upper-tail p for an F statistic
f_pvalue = (F, df1, df2) => {
  if (F <= 0) return 1;
  return ibeta(df2 / (df2 + df1 * F), df2 / 2, df1 / 2);
}

// standard normal draws (Box-Muller)
rnorm = (n, mean = 0, sd = 1) => {
  const out = new Array(n);
  for (let i = 0; i < n; i += 2) {
    const u = Math.random() || 1e-12, v = Math.random();
    const r = Math.sqrt(-2 * Math.log(u));
    out[i] = mean + sd * r * Math.cos(2 * Math.PI * v);
    if (i + 1 < n) out[i + 1] = mean + sd * r * Math.sin(2 * Math.PI * v);
  }
  return out;
}

mean_ = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length
sd_ = (xs) => {
  const m = mean_(xs);
  return Math.sqrt(xs.reduce((a, b) => a + (b - m) * (b - m), 0) / (xs.length - 1));
}

// paired t test -> two-sided p
paired_t_p = (x, y) => {
  const d = x.map((v, i) => v - y[i]);
  const t = mean_(d) / (sd_(d) / Math.sqrt(d.length));
  return t_pvalue(t, d.length - 1);
}

// power of a two-sided one-sample z test with effect h at size n
// (the exact formula behind pwr::pwr.p.test, as the legacy app used)
power_z = (h, n) =>
  pnorm(Math.sqrt(n) * Math.abs(h) - qnorm975) +
  pnorm(-Math.sqrt(n) * Math.abs(h) - qnorm975)

// smallest n achieving target power (legacy "N for 80% power")
n_for_power = (h, target = 0.8, nmax = 100000) => {
  if (Math.abs(h) < 1e-9) return Infinity;
  let lo = 2, hi = 4;
  while (power_z(h, hi) < target && hi < nmax) { lo = hi; hi *= 2; }
  if (hi >= nmax) return Infinity;
  for (let i = 0; i < 60; i++) {
    const mid = (lo + hi) / 2;
    if (power_z(h, mid) < target) lo = mid; else hi = mid;
  }
  return hi;
}

// critical t (quantile) via bisection on t_pvalue
qt975 = (df) => {
  let lo = 1, hi = 300;
  for (let i = 0; i < 80; i++) {
    const mid = (lo + hi) / 2;
    if (t_pvalue(mid, df) > 0.05) lo = mid; else hi = mid;
  }
  return (lo + hi) / 2;
}

// input styling adapted from levante-datapage: pill labels, borderless
// rounded selects (scoped to the ojs namespace class)
ojs_ns = Inputs.text().classList[0]

html`<style>
  .${ojs_ns} div label {
    background-color: #eef2f6;
    padding: 0.2rem 0.5rem;
    border-radius: 0.4rem;
    margin-bottom: 0.25rem;
    width: auto;
  }

  .${ojs_ns} select,
  .${ojs_ns} input[type="text"],
  .${ojs_ns} input[type="file"] {
    background-color: #fff;
    border: 1px solid #d8dee4;
    border-radius: 0.4rem;
    padding: 0.2rem 0.4rem;
  }

  .${ojs_ns} input[type="checkbox"] {
    accent-color: #2780e3;
    margin-bottom: 0;
  }
</style>`

html`<style>
  .${ojs_ns} button {
    background-color: #fff;
    color: #212529;
    border: 1px solid #c6ced6;
    border-radius: 0.4rem;
    padding: 0.25rem 0.9rem;
    cursor: pointer;
  }

  .${ojs_ns} button:hover {
    background-color: #eef2f6;
  }
</style>`

registry = d3.json("slices/datasets.json")

// field-wide page: load every dataset's effect sizes and model file
all_data = Promise.all(registry.map(async (r) => {
  const rows = await d3.json(`slices/es/${r.short_name}.json`);
  const models = await d3.json(`slices/models/${r.short_name}.json`)
    .catch(() => null);
  return { meta: r, rows, models };
}))

solar_blue = "#268bd2"

Power of typical studies, dataset by dataset

For each dataset: the multilevel meta-analytic effect size (Cohen’s d), the median per-study sample size, and the power of that median-sized study to detect that effect (two-sided α = .05, within-subjects normal approximation). The 2018 paper’s headline — a median power of 44% — reflected a field standard of n ≈ 15–20 regardless of effect size.

power_rows = {
  clear_loading();
  return all_data.map(({ meta, rows, models }) => {
    const ns = rows.map((r) => +r.n).filter(Number.isFinite).sort(d3.ascending);
    const median_n = d3.median(ns);
    const combo = models?.["All data|d|"];
    const d_hat = combo?.converged ? combo.coefs[0].est : null;
    const power = d_hat === null ? null : power_z(d_hat, median_n);
    return {
      dataset: meta.name,
      es_n: rows.length,
      papers: meta.num_papers,
      median_n,
      d: d_hat,
      power
    };
  });
}

median_power = d3.median(power_rows.filter((d) => d.power !== null), (d) => d.power)

html`<div style="display:flex; gap:0.75rem; flex-wrap:wrap; margin-bottom:0.75rem;">
  <div class="value-box"><div class="vb-value">${(100 * median_power).toFixed(0)}%</div>
    <div class="vb-label">Median power across datasets<br>(at each dataset's median n and d̂)</div></div>
  <div class="value-box"><div class="vb-value">${d3.median(power_rows, (d) => d.median_n)}</div>
    <div class="vb-label">Median sample size</div></div>
</div>`

Plot

power_scatter = Plot.plot({
  width: 780, height: 460,
  marginLeft: 55,
  x: { label: "Meta-analytic effect size (d)" },
  y: { label: "Power at median n", domain: [0, 1] },
  marks: [
    Plot.ruleY([0.8], { strokeDasharray: "5,4", stroke: "#666" }),
    Plot.dot(power_rows.filter((d) => d.power !== null), {
      x: "d", y: "power", r: (d) => Math.sqrt(d.es_n) / 2,
      fill: solar_blue, fillOpacity: 0.6,
      title: (d) => `${d.dataset}\nd = ${d.d.toFixed(2)}, median n = ${d.median_n}\npower = ${(100 * d.power).toFixed(0)}%`,
      tip: true
    }),
    Plot.text(power_rows.filter((d) => d.power !== null && (d.power > 0.8 || d.d > 1)), {
      x: "d", y: "power", text: (d) => d.dataset.slice(0, 18),
      dy: -10, fontSize: 9, fill: "#555"
    })
  ]
})

Table

Inputs.table(d3.sort(power_rows, (d) => -(d.power ?? -1)).map((d) => ({
  dataset: d.dataset, "effect sizes": d.es_n, papers: d.papers,
  "median n": d.median_n,
  "d (multilevel)": d.d === null ? null : +d.d.toFixed(2),
  "power at median n": d.power === null ? null : +d.power.toFixed(2)
})), { rows: 32 })

Do methods differ in the effects they yield?

Effect sizes by experimental method, pooled across datasets (methods with at least 10 effect sizes). The 2018 paper found conditioned head-turn yields d > 1 while several passive-looking methods hover near d ≈ 0.2 — method choice is a first-order design decision.

method_box = {
  const rows = all_data.flatMap(({ meta, rows }) =>
    rows.map((r) => ({ method: r.method, g: +r.g_calc }))
      .filter((r) => r.method && Number.isFinite(r.g)));
  const counts = d3.rollup(rows, (v) => v.length, (d) => d.method);
  const keep = [...counts].filter(([, n]) => n >= 10).map(([m]) => m);
  const dat = rows.filter((r) => keep.includes(r.method));
  const order = d3.sort(keep, (m) => d3.median(dat.filter((r) => r.method === m), (r) => r.g));
  return Plot.plot({
    width: 780, height: order.length * 34 + 80,
    marginLeft: 230,
    x: { label: "Effect size (Hedges' g)", domain: [-2, 4] },
    y: { domain: order, label: null },
    marks: [
      Plot.ruleX([0], { strokeDasharray: "3,3", stroke: "#999" }),
      Plot.boxX(dat, { x: "g", y: "method", fill: "#e3ecf5", stroke: "#268bd2" })
    ]
  });
}

Small-study effects: sample size vs effect size

If studies with smaller samples report systematically larger effects — because small studies only reach significance (and publication) with big effects — the literature overestimates. Kendall’s τ between n and effect size per dataset (the 2018 paper’s Table 5 found significant negative correlations in 4 of 12 meta-analyses).

kendall_tau = (xs, ys) => {
  // tau-b with normal approximation p-value
  const n = xs.length;
  let concordant = 0, discordant = 0, tx = 0, ty = 0;
  for (let i = 0; i < n; i++)
    for (let j = i + 1; j < n; j++) {
      const dx = Math.sign(xs[i] - xs[j]), dy = Math.sign(ys[i] - ys[j]);
      if (dx === 0 && dy === 0) { tx++; ty++; }
      else if (dx === 0) tx++;
      else if (dy === 0) ty++;
      else if (dx === dy) concordant++;
      else discordant++;
    }
  const n0 = n * (n - 1) / 2;
  const tau = (concordant - discordant) /
    Math.sqrt((n0 - tx) * (n0 - ty) || 1);
  const z = 3 * (concordant - discordant) /
    Math.sqrt(n * (n - 1) * (2 * n + 5) / 2);
  const p = 2 * (1 - pnorm(Math.abs(z)));
  return { tau, p };
}

tau_rows = all_data.map(({ meta, rows }) => {
  const pts = rows
    .map((r) => ({ n: +r.n, g: +r.g_calc }))
    .filter((r) => Number.isFinite(r.n) && Number.isFinite(r.g));
  if (pts.length < 10) return null;
  const { tau, p } = kendall_tau(pts.map((r) => r.n), pts.map((r) => r.g));
  return { dataset: meta.name, k: pts.length, tau, p, significant: p < 0.05 && tau < 0 };
}).filter(Boolean)

tau_plot = {
  const rows = d3.sort(tau_rows, (d) => d.tau);
  return Plot.plot({
    width: 780, height: rows.length * 22 + 60,
    marginLeft: 240,
    x: { label: "Kendall's τ (sample size vs effect size)" },
    y: { domain: rows.map((d) => d.dataset), label: null },
    marks: [
      Plot.ruleX([0], { strokeDasharray: "3,3", stroke: "#999" }),
      Plot.dot(rows, { y: "dataset", x: "tau",
        fill: (d) => d.significant ? "#dc322f" : "#333", r: 4,
        title: (d) => `${d.dataset}\nτ = ${d.tau.toFixed(2)}, p = ${d.p.toFixed(3)} (k = ${d.k})`,
        tip: true })
    ]
  });
}

html`<p style="font-size:0.9rem; color:#444;">Red: significantly negative
(bias-consistent) at p < .05 — currently
${tau_rows.filter((d) => d.significant).length} of ${tau_rows.length}
datasets. A negative τ can also arise from accurate prospective power
analysis for subtler effects; the pervasively low power above argues
against that explanation.</p>`

---

Analyses follow Bergmann et al. (2018) (scripts), recomputed in the browser from the current MetaLab release. Power uses the same normal approximation as the power analysis tool; the multilevel effect-size estimates match the visualization page.