Dataset Validator

Check a candidate MetaLab dataset against the field specification before contributing it. Everything runs in your browser — your data is not uploaded anywhere. Upload a CSV export of your coding spreadsheet (in Google Sheets: File → Download → Comma Separated Values), or paste CSV text.

// input styling adapted from levante-datapage: pill labels, borderless
// rounded selects (scoped to the ojs namespace class)
ojs_ns = Inputs.text().classList[0]

html`<style>
  .${ojs_ns} div label {
    background-color: #eef2f6;
    padding: 0.2rem 0.5rem;
    border-radius: 0.4rem;
    margin-bottom: 0.25rem;
    width: auto;
  }

  .${ojs_ns} select,
  .${ojs_ns} input[type="text"],
  .${ojs_ns} input[type="file"] {
    background-color: #fff;
    border: 1px solid #d8dee4;
    border-radius: 0.4rem;
    padding: 0.2rem 0.4rem;
  }

  .${ojs_ns} input[type="checkbox"] {
    accent-color: #2780e3;
    margin-bottom: 0;
  }
</style>`

html`<style>
  .${ojs_ns} button {
    background-color: #fff;
    color: #212529;
    border: 1px solid #c6ced6;
    border-radius: 0.4rem;
    padding: 0.25rem 0.9rem;
    cursor: pointer;
  }

  .${ojs_ns} button:hover {
    background-color: #eef2f6;
  }
</style>`

spec = d3.json("slices/spec.json")

// validation rules ported from metalabr validate_metalab_dataset_field.R:
// - required fields must be present as columns
// - short_cite additionally: no missing values, max 60 characters
// - options fields: all non-missing values within the allowed options
//   (nullable fields excuse missing values)
// - numeric fields: numeric or entirely missing
// - r and corr additionally: within [-1, 1]
validate_rows = (rows) => {
  if (!rows || !rows.length) return [];
  const cols = new Set(Object.keys(rows[0]));
  const isMissing = (v) => v === null || v === undefined || String(v).trim() === "" ||
    String(v).trim().toUpperCase() === "NA";
  const results = [];
  for (const f of spec) {
    if (!f.required) continue;
    let ok = true, notes = [];
    if (!cols.has(f.field)) {
      ok = false; notes.push("column missing");
    } else {
      const vals = rows.map((r) => r[f.field]);
      const present = vals.filter((v) => !isMissing(v));
      if (f.field === "short_cite") {
        const nMissing = vals.length - present.length;
        if (nMissing > 0) { ok = false; notes.push(`${nMissing} missing value(s)`); }
        const tooLong = present.filter((v) => String(v).length > 60).length;
        if (tooLong > 0) { ok = false; notes.push(`${tooLong} value(s) over 60 characters`); }
      }
      if (f.type === "options") {
        const allowed = new Set(
          (Array.isArray(f.options) ? f.options : Object.keys(f.options ?? {}))
            .map((o) => typeof o === "object" ? Object.keys(o)[0] : String(o)));
        const bad = [...new Set(present.map(String))].filter((v) => !allowed.has(v));
        if (bad.length > 0) {
          ok = false;
          notes.push(`invalid value(s): ${bad.slice(0, 5).join(", ")}${bad.length > 5 ? ", …" : ""}`);
        }
      }
      if (f.type === "numeric") {
        const nonNum = present.filter((v) => !Number.isFinite(+String(v).replace(/,/g, ""))).length;
        if (nonNum > 0) { ok = false; notes.push(`${nonNum} non-numeric value(s)`); }
        if (["r", "corr"].includes(f.field)) {
          const bad = present
            .map((v, idx) => ({ v: +v, row: idx + 2 }))
            .filter((d) => Number.isFinite(d.v) && (d.v < -1 || d.v > 1));
          if (bad.length > 0) {
            ok = false;
            notes.push(`out of [-1, 1] at sheet row(s) ${bad.slice(0, 5).map((d) => d.row).join(", ")}`);
          }
        }
      }
    }
    results.push({ field: f.field, valid: ok, notes: notes.join("; ") });
  }
  return results;
}

viewof csv_file = Inputs.file({ label: "Upload CSV", accept: ".csv" })

viewof csv_text = Inputs.textarea({
  label: "…or paste CSV text",
  placeholder: "study_ID,long_cite,short_cite,…",
  rows: 6
})

Validation runs against the required fields of the current spec (metadata/spec.yaml). Non-required fields are not checked.

candidate_rows = {
  if (csv_file) {
    const text = await csv_file.text();
    return d3.csvParse(text);
  }
  if (csv_text && csv_text.trim().length > 0) return d3.csvParse(csv_text);
  return null;
}

validation = candidate_rows ? validate_rows(candidate_rows) : null

validation === null
  ? html`<p style="color:#777;"><em>Upload or paste a dataset to see results.</em></p>`
  : html`<div style="display:flex; gap:0.75rem; margin-bottom: 0.75rem;">
      <div class="value-box"><div class="vb-value">${candidate_rows.length}</div>
        <div class="vb-label">Rows</div></div>
      <div class="value-box" style="background:${validation.every((d) => d.valid) ? "#2e7d32" : "#c62828"}">
        <div class="vb-value">${validation.filter((d) => !d.valid).length}</div>
        <div class="vb-label">Failing fields</div></div>
    </div>`

validation === null ? html`` : Inputs.table(
  validation.map((d) => ({ Field: d.field, Valid: d.valid ? "✅" : "❌", Notes: d.notes })),
  { rows: 60, width: { Field: 220, Valid: 60 } })

Field reference

Inputs.table(spec.map((f) => ({
  field: f.field,
  type: f.type ?? "",
  required: f.required ? "yes" : "",
  description: f.description ?? "",
  options: Array.isArray(f.options)
    ? f.options.map((o) => typeof o === "object" ? Object.keys(o)[0] : o).join(", ")
    : (f.options ? Object.keys(f.options).join(", ") : "")
})), { rows: 20 })