{
  "schema": "canli.dataset-summary.v1",
  "dataset": "Canli FilingFacts",
  "version": "0",
  "declared": "2026-09-26",
  "license": {
    "name": "CC BY 4.0",
    "url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "items": 1882,
  "companies": 402,
  "by_template": {
    "change": 402,
    "lookup": 402,
    "net_assets": 338,
    "ratio": 402,
    "unanswerable": 338
  },
  "facts_cited": 3024,
  "period_ends": {
    "first": "1988-01-14",
    "last": "2026-06-30"
  },
  "filed": {
    "first": "2009-11-20",
    "last": "2026-09-11"
  },
  "baseline": {
    "closed_book": {
      "model": "gpt-5.4-mini",
      "items": 150,
      "accuracy": 0.19333333333333333,
      "mean_tokens": 187,
      "by_template": {
        "change": 0,
        "lookup": 0,
        "net_assets": 0,
        "ratio": 0.03333333333333333,
        "unanswerable": 0.9333333333333333
      },
      "behaviour": {
        "answerable": 120,
        "answerable_accuracy": 0.008333333333333333,
        "answerable_abstention": 0.36666666666666664,
        "numbers_given": 76,
        "numbers_wrong": 0.9868421052631579,
        "unanswerable": 30,
        "unanswerable_abstention": 0.9333333333333333,
        "unanswerable_invented_number": 0
      }
    },
    "with_mcp": {
      "model": "gpt-5.4-mini",
      "items": 150,
      "accuracy": 0.68,
      "mean_tokens": 3833,
      "by_template": {
        "change": 0.4,
        "lookup": 0.8,
        "net_assets": 0.7333333333333333,
        "ratio": 0.7,
        "unanswerable": 0.7666666666666667
      },
      "behaviour": {
        "answerable": 120,
        "answerable_accuracy": 0.6583333333333333,
        "answerable_abstention": 0.14166666666666666,
        "numbers_given": 96,
        "numbers_wrong": 0.17708333333333334,
        "unanswerable": 30,
        "unanswerable_abstention": 0.7666666666666667,
        "unanswerable_invented_number": 0
      }
    }
  },
  "gold_packet_items": 50,
  "human_verified_items": 0,
  "files": {
    "items": {
      "path": "/datasets/filing-facts/v0/filing-facts-v0.jsonl",
      "bytes": 1663084,
      "kilobytes": 1624,
      "sha256": "c01b9e3e660ff341017532718e6bc8f7fe96d1ffa68d86236c53e73ae3149d7d"
    },
    "closed": {
      "path": "/datasets/filing-facts/v0/ff-eval-closed.json",
      "bytes": 32458,
      "kilobytes": 32,
      "sha256": "450bff2b316adf5e8f4562d4cacdceca143c6b10ed657ecefa02af24e4c95720"
    },
    "mcp": {
      "path": "/datasets/filing-facts/v0/ff-eval-mcp.json",
      "bytes": 32324,
      "kilobytes": 32,
      "sha256": "50a88c0c383ac492a8c1c2d489a1c5830c478a71ae7bab7660fd2878a73cb6c6"
    },
    "gold": {
      "path": "/datasets/filing-facts/v0/gold-packet-v0.json",
      "bytes": 28263,
      "kilobytes": 28,
      "sha256": "91c6f960e34509653800a4cda81bd0632dd58c889ee78eb6461a985765500413"
    }
  },
  "jsonld": {
    "@context": "https://schema.org",
    "@type": "Dataset",
    "name": "Canli FilingFacts v0: financial reasoning items verified from SEC XBRL data",
    "alternateName": "FilingFacts v0",
    "description": "1,882 questions about 402 companies' SEC annual filings, each answer computed from the company's XBRL facts, re-derived by an independent checker, and cited to the filing accession it depends on. Includes unanswerable items and a closed-book versus tool-using baseline.",
    "url": "https://canlicapital.com/research/filing-facts-v0",
    "identifier": "https://canlicapital.com/research/filing-facts-v0",
    "version": "0",
    "datePublished": "2026-09-26",
    "license": "https://creativecommons.org/licenses/by/4.0/",
    "isAccessibleForFree": true,
    "creator": {
      "@type": "Person",
      "@id": "https://canlicapital.com/#arhan-canli",
      "name": "Arhan Canli",
      "url": "https://canlicapital.com/founder",
      "sameAs": [
        "https://github.com/arhancanli",
        "https://orcid.org/0009-0004-4138-7907"
      ]
    },
    "publisher": {
      "@type": "Organization",
      "@id": "https://canlicapital.com/#organization",
      "name": "Canli Capital",
      "url": "https://canlicapital.com/"
    },
    "keywords": [
      "financial question answering",
      "SEC XBRL",
      "10-K",
      "LLM evaluation",
      "hallucination",
      "financial reasoning benchmark"
    ],
    "temporalCoverage": "1988-01-14/2026-06-30",
    "measurementTechnique": "Generated from SEC XBRL company facts; every answer re-derived by an independent checker",
    "isBasedOn": "https://www.sec.gov/search-filings/edgar-application-programming-interfaces",
    "variableMeasured": [
      "question",
      "answer",
      "facts",
      "template",
      "hops"
    ],
    "distribution": [
      {
        "@type": "DataDownload",
        "name": "Items (JSON Lines)",
        "contentUrl": "https://canlicapital.com/datasets/filing-facts/v0/filing-facts-v0.jsonl",
        "encodingFormat": "application/jsonl",
        "contentSize": "1663084 bytes"
      },
      {
        "@type": "DataDownload",
        "name": "Gold packet for annotators",
        "contentUrl": "https://canlicapital.com/datasets/filing-facts/v0/gold-packet-v0.json",
        "encodingFormat": "application/json"
      }
    ]
  }
}