Tool Comparison (ObservableJS Experimental version)

tools = FileAttachment("../data/tools.json").json()

columns = FileAttachment("../data/tool-comparison/columns.json").json()

viewof answers = Inputs.form({
  useCase: Inputs.select(
    ["RAG", "Agents", "Chatbot", "Classification", "I don't know yet"],
    {label: "What are you evaluating?"}
  ),
  experience: Inputs.select(
    ["New to evals", "Have used evals before", "Building eval infrastructure"],
    {label: "Your eval experience?"}
  ),
  teamSize: Inputs.select(
    ["Just me", "Small team (2-5)", "Larger team/org"],
    {label: "Team size?"}
  )
})

recommendation = {
  const {useCase, experience} = answers;
  if (experience === "New to evals") {
    if (useCase === "RAG") {
      return {
        frameworks: ["Ragas", "Promptfoo"],
        reason: "Ragas has RAG-specific metrics out of the box. Promptfoo is great for learning eval basics."
      };
    }
    if (useCase === "Agents") {
      return {
        frameworks: ["DeepEval", "Promptfoo"],
        reason: "Start with simple evaluation harnesses and build up from there."
      };
    }
  }
  return {
    frameworks: ["Promptfoo", "DeepEval"],
    reason: "Both have good docs and active communities for getting started."
  };
}

html`<div class="recommendation">
  <h3>Recommended: ${recommendation.frameworks.join(" or ")}</h3>
  <p>${recommendation.reason}</p>
</div>`

colors = ({
  yes: "#d6f5d6",
  no: "#f8d7da",
  unknown: "#eeeeee",
  yesText: "#145214",
  noText: "#6b1b1b",
  unknownText: "#666666"
})

style = html`
<style>
.observable-table {
  width: 100%;
  border-collapse: collapse;
  font-size: 0.95rem;
}
.observable-table th,
.observable-table td {
  border: 1px solid rgba(0,0,0,0.12);
  padding: 6px 8px;
  text-align: center;
}
.observable-table th:first-child,
.observable-table td:first-child {
  text-align: left;
  position: sticky;
  left: 0;
  background: var(--bs-body-bg, #fff);
  z-index: 1;
}
.observable-table .yes {
  background: ${colors.yes};
  color: ${colors.yesText};
  font-weight: 700;
}
.observable-table .no {
  background: ${colors.no};
  color: ${colors.noText};
  font-weight: 700;
}
.observable-table .unknown {
  background: ${colors.unknown};
  color: ${colors.unknownText};
  font-weight: 700;
}
.observable-table thead th {
  writing-mode: vertical-rl;
  transform: rotate(180deg);
  white-space: nowrap;
  height: 140px;
  vertical-align: bottom;
  padding: 6px 4px;
}
</style>
`

buildTable = (cols, dataRows) => {
  const table = html`<table class="observable-table"></table>`;
  const thead = html`<thead></thead>`;
  const headerRow = html`<tr></tr>`;
  for (const col of cols) {
    headerRow.append(html`<th>${col}</th>`);
  }
  thead.append(headerRow);

  const tbody = html`<tbody></tbody>`;
  for (const row of dataRows) {
    const tr = html`<tr></tr>`;
    for (const col of cols) {
      const val = row[col] ?? "?";
      let cls = "";
      if (val === "Y") cls = "yes";
      else if (val === "N") cls = "no";
      else cls = "unknown";
      tr.append(html`<td class=${cls}>${val}</td>`);
    }
    tbody.append(tr);
  }

  table.append(thead);
  table.append(tbody);
  return table
}

contextCols = columns.columns.context
coreCols = columns.columns.core
fullCols = columns.columns.full

container = html`<div></div>`
container.append(style);

container.append(html`<h2>Context</h2>`);

container.append(buildTable(contextCols, tools.rows));

container.append(html`<h2>Core capabilities</h2>`);

container.append(buildTable(coreCols, tools.rows));

container.append(html`<h2>Everything all at once (Experimental)</h2>`);

container.append(buildTable(fullCols, tools.rows));

container