llm-wiki/tests/browser/graph-renderer-trial-shared.ts

import type { LargeGraphFixtureId, LargeGraphFixtureMetadata } from "../../packages/graph-engine/test/large-graph-fixtures";

export const FULL_TRIAL_SHAPES: LargeGraphFixtureId[] = [
  "real-snapshot-proxy",
  "nodes-1000-sparse",
  "nodes-1000-dense",
  "nodes-1000-many-communities",
  "nodes-5000-sparse",
  "nodes-5000-dense",
  "nodes-10000-aggregation",
  "nodes-10000-high-edge",
  "oversized-community",
  "many-small-communities",
  "many-search-hits",
  "many-pin-nodes"
];

export const REQUIRED_TRIAL_ACTIONS = [
  "initial_render",
  "wheel_zoom",
  "drag",
  "search_highlight",
  "point_select",
  "container_select",
  "spotlight_animation",
  "drawer_open",
  "enter_community",
  "return_global",
  "repeated_search_community_drawer_cycles"
] as const;

export type TrialAction = typeof REQUIRED_TRIAL_ACTIONS[number];

// Actions where fps + frame p95 are mandatory-to-measure (wheel/drag).
export const FRAME_SAMPLED_ACTIONS = new Set<string>(["wheel_zoom", "drag", "spotlight_animation"]);
// Actions where wall-clock duration has an upper bound per the hard-gate table.
export const DURATION_GATED_ACTIONS = new Set<string>([
  "initial_render",
  "search_highlight",
  "drawer_open",
  "return_global"
]);
// The memory-gated repeated-cycle action.
export const MEMORY_GATED_ACTION = "repeated_search_community_drawer_cycles";

export const TRIAL_SCHEMA_VERSION = "1.0.0";

// Hard-gate table from the plan. fps and frame p95 are uniform across scales;
// duration and memory ceilings scale with node count.
export const FPS_FLOOR = 45;
export const FRAME_P95_CEILING_MS = 22.3;

export interface TrialRecordLike {
  graph_shape: string;
  action: string;
  pass: boolean;
  nodes?: number | null;
  fps?: number | null;
  frame_p95_ms?: number | null;
  duration_ms?: number | null;
  memory_growth_mb?: number | null;
  failure_class: string | null;
  failure_detail?: string | null;
  schema_version?: string;
  production_path?: boolean;
  thresholds?: Record<string, number> | null;
  browser?: string | null;
  build_commit?: string | null;
  run_started_at?: string | null;
  run_finished_at?: string | null;
}

export function parseRequestedShapes(value: string | undefined): LargeGraphFixtureId[] {
  return (value || FULL_TRIAL_SHAPES.join(","))
    .split(",")
    .map((item) => item.trim())
    .filter(Boolean) as LargeGraphFixtureId[];
}

export async function waitForAnimationFrames(page: PageLike, count = 2): Promise<void> {
  await page.evaluate((frameCount: number) => new Promise<void>((resolve) => {
    let remaining = Math.max(1, frameCount);
    const tick = () => {
      remaining -= 1;
      if (remaining <= 0) {
        resolve();
        return;
      }
      requestAnimationFrame(tick);
    };
    requestAnimationFrame(tick);
  }), count);
}

export async function waitForAfterDrawing(page: PageLike): Promise<void> {
  await page.evaluate(() => new Promise<void>((resolve, reject) => {
    const trial = (window as any).__visTrial;
    if (!trial?.network?.once) {
      requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
      return;
    }
    let finished = false;
    const timeout = window.setTimeout(() => {
      if (finished) return;
      finished = true;
      reject(new Error("vis-network afterDrawing did not fire after action"));
    }, 2500);
    const finish = () => requestAnimationFrame(() => {
      if (finished) return;
      finished = true;
      window.clearTimeout(timeout);
      resolve();
    });
    trial.network.once("afterDrawing", finish);
    trial.network.redraw();
  }));
}

export function memoryGrowthLimitMb(metadata: LargeGraphFixtureMetadata): number {
  if (metadata.nodes >= 10000) return 100;
  if (metadata.nodes >= 5000) return 75;
  return 50;
}

export function memoryGrowthFailureClass(memoryGrowthMb: number | null, metadata: LargeGraphFixtureMetadata): string | null {
  if (memoryGrowthMb == null) return null;
  return memoryGrowthMb <= memoryGrowthLimitMb(metadata) ? null : "memory_growth_above_floor";
}

export function memoryGrowthFailureDetail(memoryGrowthMb: number | null, metadata: LargeGraphFixtureMetadata): string | null {
  if (memoryGrowthMb == null) return "memory_growth_unavailable";
  const limit = memoryGrowthLimitMb(metadata);
  return memoryGrowthMb <= limit ? null : `memory_growth_mb=${memoryGrowthMb}; limit_mb=${limit}`;
}

export function initialRenderLimitMs(metadata: LargeGraphFixtureMetadata): number {
  if (metadata.nodes >= 10000) return 2000;
  if (metadata.nodes >= 5000) return 1200;
  return 500;
}

export function searchHighlightLimitMs(metadata: LargeGraphFixtureMetadata): number {
  if (metadata.nodes >= 10000) return 700;
  if (metadata.nodes >= 5000) return 400;
  return 200;
}

export function drawerOpenLimitMs(metadata: LargeGraphFixtureMetadata): number {
  if (metadata.nodes >= 10000) return 500;
  if (metadata.nodes >= 5000) return 400;
  return 200;
}

export function returnGlobalLimitMs(metadata: LargeGraphFixtureMetadata): number {
  if (metadata.nodes >= 10000) return 800;
  if (metadata.nodes >= 5000) return 500;
  return 250;
}

export function durationLimitMs(metadata: LargeGraphFixtureMetadata, action: string): number | null {
  switch (action) {
    case "initial_render":
      return initialRenderLimitMs(metadata);
    case "search_highlight":
      return searchHighlightLimitMs(metadata);
    case "drawer_open":
      return drawerOpenLimitMs(metadata);
    case "return_global":
      return returnGlobalLimitMs(metadata);
    default:
      return null;
  }
}

// Build the per-action thresholds object the schema requires. It records the
// fps, frame p95, duration and memory ceilings actually applied so reviewers
// can audit each record without re-deriving them from node count.
export function actionThresholds(metadata: LargeGraphFixtureMetadata, action: string): Record<string, number> {
  const thresholds: Record<string, number> = {
    fps_floor: FPS_FLOOR,
    frame_p95_ms_ceiling: FRAME_P95_CEILING_MS
  };
  const duration = durationLimitMs(metadata, action);
  if (duration != null) thresholds.duration_ms_ceiling = duration;
  thresholds.memory_growth_mb_ceiling = memoryGrowthLimitMb(metadata);
  return thresholds;
}

// esbuild (used by tsx with keepNames:true) wraps named functions in a __name
// helper. When such a function is serialized into the browser via page.evaluate
// the helper is undefined there. Shim it to a no-op on each page so the
// serialized arrow functions resolve. Applied through addInitScript-equivalent.
export const NAME_HELPER_INIT_SCRIPT = "window.__name = typeof window.__name === 'function' ? window.__name : function (fn) { return fn; };";

// Judge a frame-sampled action (wheel/drag). Returns a failure_class string when
// the mandatory fps / frame-p95 metrics breach the hard gate, or null when clean.
export function frameSampleFailureClass(
  record: { fps?: number | null; frame_p95_ms?: number | null }
): string | null {
  if (record.fps == null) return "fps_missing";
  if (record.frame_p95_ms == null) return "frame_p95_missing";
  if (record.fps < FPS_FLOOR) return "fps_below_floor";
  if (record.frame_p95_ms > FRAME_P95_CEILING_MS) return "frame_p95_above_ceiling";
  return null;
}

// Judge a duration-gated action. Null metrics fail; over-budget durations fail.
export function durationFailureClass(
  record: { duration_ms?: number | null },
  metadata: LargeGraphFixtureMetadata,
  action: string
): string | null {
  if (record.duration_ms == null) return "duration_missing";
  const limit = durationLimitMs(metadata, action);
  if (limit != null && record.duration_ms > limit) return "duration_above_ceiling";
  return null;
}

export function validateTrialResults(input: {
  renderer: string;
  requestedShapes: readonly string[];
  requiredActions?: readonly string[];
  // Default true: enforce the hard-gate schema (schema_version, thresholds,
  // browser, build_commit, run timestamps) and frame/duration mandatory metrics.
  // Sibling trials that have not yet been upgraded opt out with requireSchema:false.
  requireSchema?: boolean;
  records: readonly TrialRecordLike[];
  errors: readonly string[];
  resultPath: string;
}): void {
  const requiredActions = input.requiredActions ?? REQUIRED_TRIAL_ACTIONS;
  const requireSchema = input.requireSchema !== false;
  const failures: string[] = [];

  for (const error of input.errors) {
    failures.push(`error: ${error}`);
  }

  for (const shape of input.requestedShapes) {
    const shapeRecords = input.records.filter((record) => record.graph_shape === shape);
    if (!shapeRecords.length) {
      failures.push(`${shape}: no records`);
      continue;
    }
    for (const action of requiredActions) {
      if (!shapeRecords.some((record) => record.action === action)) {
        failures.push(`${shape}: missing action ${action}`);
      }
    }
  }

  if (requireSchema) for (const record of input.records) {
    const shapeAction = `${record.graph_shape}/${record.action}`;
    // Schema completeness: every record must carry the documented fields.
    if (!record.schema_version) failures.push(`${shapeAction}: missing schema_version`);
    if (typeof record.production_path !== "boolean") failures.push(`${shapeAction}: missing production_path`);
    if (!record.thresholds) failures.push(`${shapeAction}: missing thresholds`);
    if (!record.browser) failures.push(`${shapeAction}: missing browser`);
    if (!record.build_commit) failures.push(`${shapeAction}: missing build_commit`);
    if (!record.run_started_at) failures.push(`${shapeAction}: missing run_started_at`);
    if (!record.run_finished_at) failures.push(`${shapeAction}: missing run_finished_at`);

    // Mandatory-metric gates: a null or not-run value on a mandatory metric
    // blocks pass (the artifact is treated as incomplete), per the hard gate.
    if (FRAME_SAMPLED_ACTIONS.has(record.action)) {
      const frameFailure = frameSampleFailureClass(record);
      if (frameFailure) failures.push(`${shapeAction}: ${frameFailure}; fps=${record.fps ?? "null"}; frame_p95_ms=${record.frame_p95_ms ?? "null"}`);
    }
    if (DURATION_GATED_ACTIONS.has(record.action)) {
      const metadata = { nodes: typeof record.nodes === "number" ? record.nodes : 0 };
      const durationFailure = durationFailureClass(record, metadata, record.action);
      if (durationFailure) failures.push(`${shapeAction}: ${durationFailure}; duration_ms=${record.duration_ms ?? "null"}`);
    }
    if (record.action === MEMORY_GATED_ACTION && record.memory_growth_mb == null) {
      failures.push(`${shapeAction}: memory_growth_missing`);
    }
  }

  for (const record of input.records) {
    if (record.pass === false || record.failure_class) {
      failures.push(`${record.graph_shape}/${record.action}: pass=${record.pass}; failure=${record.failure_class ?? "none"}; detail=${record.failure_detail ?? "none"}`);
    }
  }

  if (failures.length) {
    throw new Error(`${input.renderer} trial failed validation (${failures.length} issue(s)). result=${input.resultPath}\n${failures.slice(0, 40).join("\n")}`);
  }
}


interface PageLike {
  evaluate<T>(fn: Function | string, arg?: unknown): Promise<T>;
}