{
  "schema": "memcat-public-release-evaluation-v1",
  "generatedAt": "2026-10-05T21:34:05.302Z",
  "index": {
    "version": "banking77-minilm-q8-32-v1",
    "encoderId": "minilm-l6-v2@751bff37182d3f1213fa05d7196b954e230abad9:q8:mean:l2:max256:transformers4.3.0",
    "sha256": "970cd75ccfdfbadaecd4b9df2fa07bcaf8bad225013cd154c612294dbe3d7ec1",
    "bytes": 10069091,
    "gzipBytes": 3670725,
    "thresholds": {
      "minScore": 0.7,
      "minMargin": 0.08
    },
    "prototypeCount": 2537
  },
  "environment": {
    "node": "v22.22.3",
    "cpu": "Apple M4 Max",
    "platform": "darwin",
    "arch": "arm64",
    "memoryBytes": 137438953472,
    "backend": "ONNX CPU; individual sequential queries"
  },
  "data": {
    "train": 7963,
    "development": 965,
    "calibration": 1071,
    "test": 3080,
    "labels": 77,
    "testSha256": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d",
    "clincSha256": "36923c3705a59e08fe9c3883d8bc2dd966ef93e22cb78ac41171782a698d56e0",
    "overlap": [
      {
        "testId": "banking77-test-00555",
        "earlierId": "banking77-train-01723",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-00977",
        "earlierId": "banking77-train-03104",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-00978",
        "earlierId": "banking77-train-03117",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-01433",
        "earlierId": "banking77-train-04477",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-01475",
        "earlierId": "banking77-train-04577",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-02150",
        "earlierId": "banking77-train-06985",
        "sameLabel": true
      },
      {
        "testId": "banking77-test-03071",
        "earlierId": "banking77-train-09922",
        "sameLabel": true
      }
    ],
    "normalizedTestDuplicates": [
      "banking77-test-01462"
    ],
    "removedTestRows": 0,
    "attribution": "BANKING77: Casanueva et al. / PolyAI, CC BY 4.0. CLINC150: Larson et al. / Clinc, CC BY 3.0. See /models/NOTICE.md."
  },
  "summary": {
    "selective": {
      "total": 3080,
      "accepted": 2146,
      "correct": 2092,
      "wrong": 54,
      "precision": 0.9748369058713886,
      "coverage": 0.6967532467532468,
      "interval95": {
        "lower": 0.9673137616586543,
        "upper": 0.9806631189797693
      },
      "passed": true
    },
    "nonOverlap": {
      "total": 3073,
      "accepted": 2139,
      "correct": 2085,
      "wrong": 54,
      "precision": 0.9747545582047685,
      "coverage": 0.6960624796615685,
      "interval95": {
        "lower": 0.9672072636070559,
        "upper": 0.9805996735976004
      },
      "passed": true
    },
    "forced": {
      "correct": 2718,
      "total": 3080,
      "accuracy": 0.8824675324675325,
      "macroF1": 0.8821107377733081,
      "perClass": [
        {
          "label": "card_arrival",
          "tp": 30,
          "fp": 5,
          "fn": 10,
          "f1": 0.8
        },
        {
          "label": "card_linking",
          "tp": 39,
          "fp": 3,
          "fn": 1,
          "f1": 0.9512195121951219
        },
        {
          "label": "exchange_rate",
          "tp": 39,
          "fp": 5,
          "fn": 1,
          "f1": 0.9285714285714286
        },
        {
          "label": "card_payment_wrong_exchange_rate",
          "tp": 38,
          "fp": 2,
          "fn": 2,
          "f1": 0.95
        },
        {
          "label": "extra_charge_on_statement",
          "tp": 35,
          "fp": 4,
          "fn": 5,
          "f1": 0.8860759493670886
        },
        {
          "label": "pending_cash_withdrawal",
          "tp": 33,
          "fp": 0,
          "fn": 7,
          "f1": 0.9041095890410958
        },
        {
          "label": "fiat_currency_support",
          "tp": 34,
          "fp": 13,
          "fn": 6,
          "f1": 0.7816091954022989
        },
        {
          "label": "card_delivery_estimate",
          "tp": 34,
          "fp": 7,
          "fn": 6,
          "f1": 0.8395061728395061
        },
        {
          "label": "automatic_top_up",
          "tp": 36,
          "fp": 2,
          "fn": 4,
          "f1": 0.9230769230769231
        },
        {
          "label": "card_not_working",
          "tp": 38,
          "fp": 8,
          "fn": 2,
          "f1": 0.8837209302325582
        },
        {
          "label": "exchange_via_app",
          "tp": 28,
          "fp": 5,
          "fn": 12,
          "f1": 0.7671232876712328
        },
        {
          "label": "lost_or_stolen_card",
          "tp": 40,
          "fp": 11,
          "fn": 0,
          "f1": 0.8791208791208791
        },
        {
          "label": "age_limit",
          "tp": 40,
          "fp": 0,
          "fn": 0,
          "f1": 1
        },
        {
          "label": "pin_blocked",
          "tp": 32,
          "fp": 2,
          "fn": 8,
          "f1": 0.8648648648648649
        },
        {
          "label": "contactless_not_working",
          "tp": 38,
          "fp": 0,
          "fn": 2,
          "f1": 0.9743589743589743
        },
        {
          "label": "top_up_by_bank_transfer_charge",
          "tp": 35,
          "fp": 3,
          "fn": 5,
          "f1": 0.8974358974358975
        },
        {
          "label": "pending_top_up",
          "tp": 33,
          "fp": 5,
          "fn": 7,
          "f1": 0.8461538461538461
        },
        {
          "label": "cancel_transfer",
          "tp": 38,
          "fp": 1,
          "fn": 2,
          "f1": 0.9620253164556962
        },
        {
          "label": "top_up_limits",
          "tp": 39,
          "fp": 3,
          "fn": 1,
          "f1": 0.9512195121951219
        },
        {
          "label": "wrong_amount_of_cash_received",
          "tp": 33,
          "fp": 6,
          "fn": 7,
          "f1": 0.8354430379746836
        },
        {
          "label": "card_payment_fee_charged",
          "tp": 31,
          "fp": 5,
          "fn": 9,
          "f1": 0.8157894736842105
        },
        {
          "label": "transfer_not_received_by_recipient",
          "tp": 31,
          "fp": 9,
          "fn": 9,
          "f1": 0.775
        },
        {
          "label": "supported_cards_and_currencies",
          "tp": 39,
          "fp": 9,
          "fn": 1,
          "f1": 0.8863636363636364
        },
        {
          "label": "getting_virtual_card",
          "tp": 39,
          "fp": 8,
          "fn": 1,
          "f1": 0.896551724137931
        },
        {
          "label": "card_acceptance",
          "tp": 38,
          "fp": 3,
          "fn": 2,
          "f1": 0.9382716049382716
        },
        {
          "label": "top_up_reverted",
          "tp": 34,
          "fp": 8,
          "fn": 6,
          "f1": 0.8292682926829268
        },
        {
          "label": "balance_not_updated_after_cheque_or_cash_deposit",
          "tp": 37,
          "fp": 2,
          "fn": 3,
          "f1": 0.9367088607594937
        },
        {
          "label": "card_payment_not_recognised",
          "tp": 29,
          "fp": 7,
          "fn": 11,
          "f1": 0.7631578947368421
        },
        {
          "label": "edit_personal_details",
          "tp": 40,
          "fp": 0,
          "fn": 0,
          "f1": 1
        },
        {
          "label": "why_verify_identity",
          "tp": 32,
          "fp": 5,
          "fn": 8,
          "f1": 0.8311688311688312
        },
        {
          "label": "unable_to_verify_identity",
          "tp": 39,
          "fp": 2,
          "fn": 1,
          "f1": 0.9629629629629629
        },
        {
          "label": "get_physical_card",
          "tp": 36,
          "fp": 6,
          "fn": 4,
          "f1": 0.8780487804878049
        },
        {
          "label": "visa_or_mastercard",
          "tp": 37,
          "fp": 1,
          "fn": 3,
          "f1": 0.9487179487179487
        },
        {
          "label": "topping_up_by_card",
          "tp": 33,
          "fp": 7,
          "fn": 7,
          "f1": 0.825
        },
        {
          "label": "disposable_card_limits",
          "tp": 36,
          "fp": 5,
          "fn": 4,
          "f1": 0.8888888888888888
        },
        {
          "label": "compromised_card",
          "tp": 29,
          "fp": 5,
          "fn": 11,
          "f1": 0.7837837837837838
        },
        {
          "label": "atm_support",
          "tp": 38,
          "fp": 2,
          "fn": 2,
          "f1": 0.95
        },
        {
          "label": "direct_debit_payment_not_recognised",
          "tp": 27,
          "fp": 4,
          "fn": 13,
          "f1": 0.7605633802816901
        },
        {
          "label": "passcode_forgotten",
          "tp": 40,
          "fp": 0,
          "fn": 0,
          "f1": 1
        },
        {
          "label": "declined_cash_withdrawal",
          "tp": 35,
          "fp": 9,
          "fn": 5,
          "f1": 0.8333333333333334
        },
        {
          "label": "pending_card_payment",
          "tp": 38,
          "fp": 7,
          "fn": 2,
          "f1": 0.8941176470588236
        },
        {
          "label": "lost_or_stolen_phone",
          "tp": 37,
          "fp": 0,
          "fn": 3,
          "f1": 0.961038961038961
        },
        {
          "label": "request_refund",
          "tp": 38,
          "fp": 4,
          "fn": 2,
          "f1": 0.926829268292683
        },
        {
          "label": "declined_transfer",
          "tp": 35,
          "fp": 10,
          "fn": 5,
          "f1": 0.8235294117647058
        },
        {
          "label": "Refund_not_showing_up",
          "tp": 36,
          "fp": 2,
          "fn": 4,
          "f1": 0.9230769230769231
        },
        {
          "label": "declined_card_payment",
          "tp": 29,
          "fp": 8,
          "fn": 11,
          "f1": 0.7532467532467533
        },
        {
          "label": "pending_transfer",
          "tp": 21,
          "fp": 4,
          "fn": 19,
          "f1": 0.6461538461538462
        },
        {
          "label": "terminate_account",
          "tp": 40,
          "fp": 3,
          "fn": 0,
          "f1": 0.963855421686747
        },
        {
          "label": "card_swallowed",
          "tp": 37,
          "fp": 3,
          "fn": 3,
          "f1": 0.925
        },
        {
          "label": "transaction_charged_twice",
          "tp": 39,
          "fp": 8,
          "fn": 1,
          "f1": 0.896551724137931
        },
        {
          "label": "verify_source_of_funds",
          "tp": 40,
          "fp": 4,
          "fn": 0,
          "f1": 0.9523809523809523
        },
        {
          "label": "transfer_timing",
          "tp": 32,
          "fp": 4,
          "fn": 8,
          "f1": 0.8421052631578947
        },
        {
          "label": "reverted_card_payment?",
          "tp": 34,
          "fp": 9,
          "fn": 6,
          "f1": 0.8192771084337349
        },
        {
          "label": "change_pin",
          "tp": 38,
          "fp": 5,
          "fn": 2,
          "f1": 0.9156626506024096
        },
        {
          "label": "beneficiary_not_allowed",
          "tp": 31,
          "fp": 6,
          "fn": 9,
          "f1": 0.8051948051948052
        },
        {
          "label": "transfer_fee_charged",
          "tp": 36,
          "fp": 11,
          "fn": 4,
          "f1": 0.8275862068965517
        },
        {
          "label": "receiving_money",
          "tp": 36,
          "fp": 0,
          "fn": 4,
          "f1": 0.9473684210526315
        },
        {
          "label": "failed_transfer",
          "tp": 32,
          "fp": 7,
          "fn": 8,
          "f1": 0.810126582278481
        },
        {
          "label": "transfer_into_account",
          "tp": 34,
          "fp": 5,
          "fn": 6,
          "f1": 0.8607594936708861
        },
        {
          "label": "verify_top_up",
          "tp": 40,
          "fp": 2,
          "fn": 0,
          "f1": 0.975609756097561
        },
        {
          "label": "getting_spare_card",
          "tp": 38,
          "fp": 2,
          "fn": 2,
          "f1": 0.95
        },
        {
          "label": "top_up_by_cash_or_cheque",
          "tp": 36,
          "fp": 3,
          "fn": 4,
          "f1": 0.9113924050632911
        },
        {
          "label": "order_physical_card",
          "tp": 34,
          "fp": 8,
          "fn": 6,
          "f1": 0.8292682926829268
        },
        {
          "label": "virtual_card_not_working",
          "tp": 37,
          "fp": 0,
          "fn": 3,
          "f1": 0.961038961038961
        },
        {
          "label": "wrong_exchange_rate_for_cash_withdrawal",
          "tp": 33,
          "fp": 0,
          "fn": 7,
          "f1": 0.9041095890410958
        },
        {
          "label": "get_disposable_virtual_card",
          "tp": 33,
          "fp": 2,
          "fn": 7,
          "f1": 0.88
        },
        {
          "label": "top_up_failed",
          "tp": 31,
          "fp": 7,
          "fn": 9,
          "f1": 0.7948717948717948
        },
        {
          "label": "balance_not_updated_after_bank_transfer",
          "tp": 33,
          "fp": 26,
          "fn": 7,
          "f1": 0.6666666666666666
        },
        {
          "label": "cash_withdrawal_not_recognised",
          "tp": 30,
          "fp": 7,
          "fn": 10,
          "f1": 0.7792207792207793
        },
        {
          "label": "exchange_charge",
          "tp": 38,
          "fp": 1,
          "fn": 2,
          "f1": 0.9620253164556962
        },
        {
          "label": "top_up_by_card_charge",
          "tp": 35,
          "fp": 2,
          "fn": 5,
          "f1": 0.9090909090909091
        },
        {
          "label": "activate_my_card",
          "tp": 36,
          "fp": 0,
          "fn": 4,
          "f1": 0.9473684210526315
        },
        {
          "label": "cash_withdrawal_charge",
          "tp": 39,
          "fp": 9,
          "fn": 1,
          "f1": 0.8863636363636364
        },
        {
          "label": "card_about_to_expire",
          "tp": 39,
          "fp": 0,
          "fn": 1,
          "f1": 0.9873417721518988
        },
        {
          "label": "apple_pay_or_google_pay",
          "tp": 39,
          "fp": 1,
          "fn": 1,
          "f1": 0.975
        },
        {
          "label": "verify_my_identity",
          "tp": 35,
          "fp": 8,
          "fn": 5,
          "f1": 0.8433734939759037
        },
        {
          "label": "country_support",
          "tp": 37,
          "fp": 2,
          "fn": 3,
          "f1": 0.9367088607594937
        }
      ]
    },
    "latency": {
      "loadFromLocalDiskMs": 371.7293330000002,
      "warmupCalls": 3,
      "warmup": [
        {
          "candidate": "order_physical_card",
          "runnerUp": "country_support",
          "score": 0.9983358894112423,
          "margin": 0.2741106092145251,
          "matchedExample": {
            "category": "order_physical_card",
            "index": 0
          },
          "scoreKind": "cosine_similarity",
          "version": "banking77-minilm-q8-32-v1",
          "encoderId": "minilm-l6-v2@751bff37182d3f1213fa05d7196b954e230abad9:q8:mean:l2:max256:transformers4.3.0",
          "elapsedMs": 6.864915999999539,
          "status": "classified",
          "category": "order_physical_card"
        },
        {
          "candidate": "get_disposable_virtual_card",
          "runnerUp": "getting_virtual_card",
          "score": 0.99872508817134,
          "margin": 0.1866917369386225,
          "matchedExample": {
            "category": "get_disposable_virtual_card",
            "index": 0
          },
          "scoreKind": "cosine_similarity",
          "version": "banking77-minilm-q8-32-v1",
          "encoderId": "minilm-l6-v2@751bff37182d3f1213fa05d7196b954e230abad9:q8:mean:l2:max256:transformers4.3.0",
          "elapsedMs": 2.6679999999996653,
          "status": "classified",
          "category": "get_disposable_virtual_card"
        },
        {
          "candidate": "card_not_working",
          "runnerUp": "declined_card_payment",
          "score": 0.9985718050777592,
          "margin": 0.2530763627814272,
          "matchedExample": {
            "category": "card_not_working",
            "index": 0
          },
          "scoreKind": "cosine_similarity",
          "version": "banking77-minilm-q8-32-v1",
          "encoderId": "minilm-l6-v2@751bff37182d3f1213fa05d7196b954e230abad9:q8:mean:l2:max256:transformers4.3.0",
          "elapsedMs": 2.0229159999998956,
          "status": "classified",
          "category": "card_not_working"
        }
      ],
      "p50Ms": 1.7842080000009446,
      "p95Ms": 2.702166999999463,
      "p99Ms": 3.717290999999932,
      "sequentialMessagesPerSecond": 523.841462102749,
      "scope": "SDK evaluation callback: token validation, local tokenization, q8 embedding, cosine ranking and rejection. Excludes model load, setup and rendering. Per-message samples include no input-result cache."
    },
    "outOfDomain": {
      "total": 240,
      "falseAccepted": 0,
      "rate": 0,
      "interval95": {
        "lower": 0,
        "upper": 0.015753919941558815
      },
      "passed": true,
      "scope": "Only eight declared nonbanking CLINC intents. Does not establish near-domain unknown rejection."
    },
    "stress": {
      "total": 14,
      "failed": 3,
      "failures": [
        "mixed-02",
        "negation-02",
        "typo-01"
      ],
      "passed": false
    },
    "economics": {
      "providerClassificationCalls": 0,
      "providerClassificationCostUsd": 0,
      "totalComputeCostUsd": null,
      "liveLlmBaseline": null,
      "claim": "Local execution made no hosted classification requests. End-to-end cost savings versus an LLM remain unmeasured."
    }
  },
  "baselines": [
    {
      "name": "normalized-exact-training-lookup",
      "forced": 0.0012987012987012987,
      "selective": {
        "total": 3080,
        "accepted": 0,
        "correct": 0,
        "wrong": 0,
        "precision": null,
        "coverage": 0,
        "interval95": {
          "lower": null,
          "upper": null
        },
        "passed": false
      }
    },
    {
      "name": "label-word-overlap",
      "forced": 0.3665584415584416,
      "selective": {
        "total": 3080,
        "accepted": 0,
        "correct": 0,
        "wrong": 0,
        "precision": null,
        "coverage": 0,
        "interval95": {
          "lower": null,
          "upper": null
        },
        "passed": false
      }
    },
    {
      "name": "tfidf-word-unigram-bigram-nearest-centroid",
      "forced": 0.8116883116883117,
      "selective": {
        "total": 3080,
        "accepted": 1431,
        "correct": 1387,
        "wrong": 44,
        "precision": 0.9692522711390635,
        "coverage": 0.4646103896103896,
        "interval95": {
          "lower": 0.9589755808369481,
          "upper": 0.9770163308435386
        },
        "passed": false
      }
    }
  ],
  "limitations": [
    "Public benchmark may have appeared in pretrained encoder training; no contamination-free claim.",
    "English single-intent banking queries only; not representative live traffic.",
    "Cosine scores are similarities, not calibrated correctness probabilities.",
    "Three warmups and local-disk model load; first network download and browser timings measured separately.",
    "No live paid model baseline, customer savings, independent adoption or retention evidence.",
    "Authored stress results expose ambiguity limitations; a benchmark pass does not authorize consequential actions."
  ],
  "fullReportSha256": "8d29ebcbfa71e7000b095c983fe74b5cc4e133cd4c8aaacf19844ead5e115c41"
}