{ "descriptions": { "stages": { "0": "no usable open option exists (and nothing is mature anywhere)", "1": "fully-open options exist but are weak on both axes", "2": "no mature fully-open product; the best fully-open option is promising but limited", "3": "no mature fully-open product, but the best fully-open option is strong", "4": "at least one mature fully-open product, but not yet enough for depth", "5": "enough mature fully-open products to be redundant/resilient" }, "gaps": { "void": "no usable open option at all.", "capability": "the best fully-open option isn't capable enough to be useful.", "adoption": "a capable fully-open option exists but is under-adopted.", "maturity": "open options exist and at least one may be mature, but the ecosystem lacks the depth/redundancy of a mature ecosystem (too few mature fully-open products).", "openness": "capable, adopted options exist, but the mature ones are not fully open (open-ish or closed). This is the orthogonal flag; it can co-occur with the others." }, "categories": { "base_pretrained": "Foundation models trained from scratch.", "finetuned_chat": "Instruction-tuned chat and reasoning assistants.", "inference_code": "Engines and runtimes that serve model inference.", "finetuning_code": "Libraries and platforms for fine-tuning models.", "evaluation_code": "Harnesses that run and grade model evaluations.", "benchmark_eval_data": "Benchmark datasets used to evaluate models.", "training_synthetic_datasets": "Corpora for pre-training and post-training models.", "orchestration_agents": "Frameworks and agents that plan and execute tasks.", "ui_api": "Chat UIs and API gateways in front of models.", "telemetry_observability": "Tracing and observability for LLM apps.", "agent_tools_protocols": "Tools, protocols, and retrieval for agents.", "ml_frameworks": "Foundational libraries the rest of the stack is built on.", "deployment": "Sandboxes, runtimes, and serverless model hosting.", "edge_hardware": "Boards and chips that run model inference at the edge.", "safeguards": "Tools whose job is to detect, filter, or constrain unsafe model inputs and outputs, or to stress-test AI systems for harm: guardrail models (safety/content classifiers, jailbreak detectors), guardrail libraries, and AI red-teaming / safety-evaluation tooling." } }, "layer_order": [ "model_components", "product_ux", "infrastructure" ], "categories": { "base_pretrained": { "label": "Base / pretrained models", "arc": "Model components", "layer": "model_components", "stage": { "num": 3, "name": "Viable Alternatives" }, "gaps": [ "maturity", "openness" ], "products": [ { "product": "Llama 3.1", "org": "Meta", "type": "model", "description": "Meta's open-weights foundation model family in 8B, 70B, and 405B parameter variants that established the open source ecosystem baseline. Trained on 15T+ tokens with grouped query attention and a 128K context window under the Llama Community License. Superseded by Llama 4 (April 2025) and Llama 5 (April 2026), but still the most widely adopted historical baseline; the 8B variant alone retains roughly 5M monthly Hugging Face downloads and remains the default seed for community fine-tunes.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF, gated/terms acceptance);data:closed;code:partial(inference utils via llama-models, no training code);license:Llama-3.1-Community-License(non-OSI: 700M MAU commercial cap + acceptable-use policy)", "confidence": "high", "note": "Weights open but Llama 3.1 Community License is non-OSI with a 700M MAU cap (confirmed in the actual Llama 3.1 license, Section 2); marketed as 'open' so flagged open_washed. Corrected source: the record originally cited the Llama 2 license text as a proxy; replaced with the actual Llama 3.1 Community License.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-3.1-8B", "shows": "base pretrained 8B (page states base, not instruct), Llama 3.1 Community License, downloadable weights behind terms acceptance", "accessed": "2026-06-04" }, { "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE", "shows": "Llama 3.1 Community License Section 2, 700M MAU commercial threshold requiring a separate Meta license (verified, replaces the prior Llama 2 proxy)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Foundational base family. CORRECTED: HF page for the 3.1 8B base shows ~1.29M downloads/month (not the 'tens of thousands' the record claimed); the single base SKU alone is in the millions/mo. Family-level reach (Llama ~476M cumulative HF downloads per ATOM, second only to Qwen) well into 1-10M+ band. Calibrated to Llama 4 anchor (A4).", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-3.1-8B", "shows": "live HF page for 3.1 8B base, 1.29M+ downloads last month", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/meta-llama", "shows": "Meta Llama org lists 3.1 (8B-405B) among active families; latest is Llama 4", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": "Llama 3.1 405B: AA Intelligence Index 17, ranked #29/43, 'below average' vs current peers (peer average 23); mid-2024 non-reasoning model surpassed by 2026 frontier.", "confidence": "medium", "note": "Capable for its 2024 generation but clearly mid-tier in 2026; AA Index 17 / rank #29/43 confirmed on primary AA source. Matches Llama 4 anchor C3.", "sources": [ { "url": "https://artificialanalysis.ai/models/llama-3-1-instruct-405b", "shows": "AA Intelligence Index 17, rank #29/43, below-average vs peers (avg 23)", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Llama 3.1 base/pretrained family (8B/70B/405B), released 23 Jul 2024, 128K context, ~15T training tokens. Base SKU (meta-llama/Llama-3.1-8B) verified live on Meta's HF org June 2026; superseded by Llama 3.2/3.3/4 but still listed and widely used. VERIFIED: Meta HF org shows Llama 4 (Scout/Maverick) as latest; 3.1 family (8B-405B) still listed." }, { "product": "DeepSeek-V3 Base", "org": "DeepSeek", "type": "model", "description": "Legacy 671B-parameter mixture-of-experts model (37B active per token) trained on 14.8T tokens with FP8 mixed precision at a reported ~$5.5M training cost. Shifted the narrative on cost-efficient frontier model training when released in December 2024, and remained the open-weights reference point through most of 2025. Superseded by DeepSeek-V4-Base (April 2026); the hosted `deepseek-chat` and `deepseek-reasoner` aliases deprecate 2026-07-24, but V3 weights remain widely fine-tuned in the community.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference code; no training pipeline/data);license:code MIT + model DeepSeek-Model-License(commercial OK with use-based AUP restrictions)", "confidence": "high", "note": "Weights open and commercially usable; model license is a custom non-OSI agreement (commercial use explicitly permitted per the card), so flagged near_open_source_non_osi_license. Verified: code MIT + custom model agreement.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Base", "shows": "671B/37B MoE base model, MIT code license + custom Model Agreement License, commercial use supported, 6,510 downloads/mo", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/blob/main/LICENSE-MODEL", "shows": "DeepSeek Model License, commercial use permitted with use-based restrictions", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Base checkpoint sees modest direct downloads (~6.5k/mo, confirmed), but it is the foundation of the heavily-served DeepSeek-V3 chat line and many derivatives; DeepSeek family is a primary driver of China open-model downloads. Placed at 4 on family/derivative reach, conservative vs the V3.2 chat anchor (A5).", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-Base", "shows": "live base repo with 6,510 downloads/mo + derivative interest", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": "DeepSeek-V3 (Dec 2024): AA Intelligence Index 16, ranked #31/43, 'below average' vs current frontier; strong open model at launch but surpassed by V3.1/V3.2/V4.", "confidence": "medium", "note": "Mid-tier in 2026; the base lacks instruct alignment so direct chat benchmarks reflect the chat sibling. Sits below the V3.2 anchor (C4).", "sources": [ { "url": "https://artificialanalysis.ai/models/deepseek-v3", "shows": "AA Intelligence Index 16, rank #31/43, below-average; newer V3.1 recommended", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "DeepSeek-V3-Base, 671B total / 37B active MoE (685B incl MTP module), 128K context, trained on 14.8T tokens. Released 27 Dec 2024 (arXiv 2412.19437). Base/pretrained SKU. VERIFIED live on HF June 2026: 671B/37B MoE base, MIT code + custom Model License, commercial use supported." }, { "product": "DeepSeek-V4-Pro-Base", "org": "DeepSeek", "type": "model", "description": "CORRECTED NAME: record was titled 'DeepSeek-V4-Base' which does not exist as a repo. DeepSeek's HF org ships base SKUs as DeepSeek-V4-Pro-Base (1.6T total MoE) and DeepSeek-V4-Flash-Base (292B), updated Apr 27 2026 (V4 family released ~24 Apr 2026). Scored on DeepSeek-V4-Pro-Base as the flagship base. VERIFIED live June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(downloadable on HF, both Pro-Base and Flash-Base);data:closed;code:partial(inference/serving; no training pipeline);license:MIT(permissive, no use restrictions)", "confidence": "high", "note": "MIT-licensed open weights (confirmed via the repo's LICENSE blob, note the HF page itself shows 'No model card', so license relied on the LICENSE file directly), data/training pipeline closed. Matches the DeepSeek-V4-Pro anchor (O3 open_weights). Confidence raised to high: independently re-verified the live repo (1.6T, 24,567 dl/mo) AND the MIT LICENSE blob (first line \"MIT License\", DeepSeek 2023) this pass; model card empty but license unambiguous.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-Base", "shows": "1.6T base model, 24,567 downloads/mo, live repo, but 'No model card'", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-Base/blob/main/LICENSE", "shows": "MIT License text, dated 2023 DeepSeek (LICENSE blob exists, 1.08 kB)", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Base released ~6 weeks before scoring; ~24.5k base-checkpoint downloads/mo (confirmed) and broad serving of the V4-Pro chat line across providers. Calibrated to the DeepSeek-V4-Pro anchor (A4), strong early traction, too recent to confirm >10M independently.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-Base", "shows": "live base repo 24,567 downloads/mo", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/deepseek-ai", "shows": "DeepSeek-V4 collection: V4-Pro (862B), V4-Flash (158B), V4-Pro-Base (1.6T), V4-Flash-Base (292B); confirms bare 'DeepSeek-V4-Base' does not exist", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench/MMLU-Pro/GPQA", "value": "DeepSeek-V4-Pro (chat sibling of this base): SWE-bench Verified 80.6%, MMLU-Pro 87.5, GPQA Diamond 90.1, LiveCodeBench 93.5, strongest open source model at release (per frozen anchor).", "confidence": "medium", "note": "Frontier-tier for the category; benchmarks reflect the post-trained V4-Pro built on this base. Matches the DeepSeek-V4-Pro anchor (C5). Note: these figures are from the frozen anchor, not re-verified against a primary AA/vendor card in this pass (V4-Pro-Base card is empty).", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-Base", "shows": "1.6T V4-Pro base, the pretrained foundation of frontier V4-Pro", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "CORRECTED NAME: record was titled 'DeepSeek-V4-Base' which does not exist as a repo. DeepSeek's HF org ships base SKUs as DeepSeek-V4-Pro-Base (1.6T total MoE) and DeepSeek-V4-Flash-Base (292B), updated Apr 27 2026 (V4 family released ~24 Apr 2026). Scored on DeepSeek-V4-Pro-Base as the flagship base. VERIFIED live June 2026." }, { "product": "Qwen 2.5", "org": "Alibaba Cloud", "type": "model", "description": "Alibaba's Apache-2.0 open-weights family (0.5B-72B dense plus a 57B-A14B MoE) trained on 18T+ tokens, with specialist code and math variants. Now the previous-generation Qwen line, superseded by Qwen3 (April 2025) and Qwen3.5 / Qwen3.7-Max (2026), but still the most-downloaded non-Western open model: the 2.5 lineup recorded 750M+ Hugging Face downloads in 2025 and over 113K Qwen-derivative models now exist on the Hub, anchored largely by 2.5's 7B variant.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference; no training pipeline);license:mixed, 0.5B-14B Apache-2.0, but flagship 72B & 3B under Qwen-License-Agreement(non-OSI, 100M MAU commercial cap)", "confidence": "high", "note": "Flagship 72B base carries the Qwen License with a 100M MAU cap (CONFIRMED in license Section 4: >100M MAU commercial requires separate authorization), non-OSI, so the headline SKU is restricted; smaller sizes are Apache. Mixed-tier across the family.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-72B/blob/main/LICENSE", "shows": "Qwen License Agreement Section 4, 100M MAU commercial cap requiring separate authorization", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/Qwen/Qwen2.5-72B", "shows": "base 72B model (not instruct), qwen license tag, 68,655 downloads/mo", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Qwen family ~942M cumulative HF downloads (>50% of all global open source downloads per ATOM); Qwen2.5 was the dominant fine-tune base through 2025. The 72B base alone shows 68,655 downloads/mo (confirmed). Family reach unambiguously >10M; calibrated to Qwen 3.6 anchor (A5).", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-72B", "shows": "live 72B base repo, 68,655 downloads/mo", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": "Qwen2.5 72B: AA Intelligence Index 16, ranked #10/39, 'well above average' among similar-size open non-reasoning models, but a 2024 generation surpassed by Qwen3/Qwen 3.6.", "confidence": "medium", "note": "Strong for its generation, mid-tier in 2026; below the Qwen 3.6 anchor (C5).", "sources": [ { "url": "https://artificialanalysis.ai/models/qwen2-5-72b-instruct", "shows": "AA Intelligence Index 16, rank #10/39, above-average for size", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Qwen2.5 base/pretrained family (0.5B/1.5B/3B/7B/14B/32B/72B), released Sept 2024, 131K context, up to ~18T training tokens. Scored on flagship Qwen2.5-72B base SKU. VERIFIED live on HF June 2026: page states 'base 72B Qwen2.5 model', qwen license, 68,655 downloads/mo. Superseded by Qwen3/Qwen3.6." }, { "product": "Qwen3", "org": "Alibaba Cloud", "type": "model", "description": "Alibaba's current open-weights flagship line, launched April 28, 2025 under Apache 2.0. Includes six dense models (0.6B / 1.7B / 4B / 8B / 14B / 32B) and two MoE variants (30B-A3B and 235B-A22B), trained on 36T tokens. Introduced hybrid reasoning, a single model that exposes both 'Thinking' (chain-of-thought) and 'Non-Thinking' (fast-response) modes via a runtime switch. Extended in 2025-2026 with Qwen3-Next-80B-A3B (ultra-sparse MoE with hybrid attention, September 2025), Qwen3.5 / Qwen3.5-Plus (February 16, 2026, 397B-A17B Apache 2.0), and Qwen3.7-Max (May 20-21, 2026, Alibaba's proprietary 'agent-era' flagship, #1 on Artificial Analysis Intelligence Index at 57, capable of 35-hour autonomous runs and 1000+ tool calls). Qwen3.7-Max is closed-weights, signalling a hybrid open/closed strategy.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference; no training pipeline/data);license:Apache-2.0(OSI) across base variants e.g. 30B-A3B-Base", "confidence": "high", "note": "Apache-2.0 base weights with no MAU cap (cleaner than Qwen2.5's flagship license, confirmed); data and training code closed -> open_weights, matching the Qwen 3.6 anchor (O3).", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Base", "shows": "Qwen3 base MoE (30.5B/3.3B), Apache-2.0, 42,966 downloads/mo", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Current-generation flagship base of the dominant Qwen family (~942M cumulative HF downloads per ATOM). Qwen3 235B flagship line shows 480,710 downloads/mo (confirmed); base 30B-A3B ~43k/mo (confirmed). Unambiguously >10M; calibrated to Qwen 3.6 anchor (A5).", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B", "shows": "480,710 downloads/mo for the Qwen3 235B flagship line", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Base", "shows": "Qwen3 base 42,966 downloads/mo", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": "Qwen3 235B-A22B is a hybrid thinking/non-thinking MoE that was near-frontier among open models on release (mid-2025); strong reasoning + agentic coding, but surpassed by Qwen 3.6 (C5) and DeepSeek-V4 by mid-2026.", "confidence": "medium", "note": "Near-frontier for its 2025 generation, now just below the current open frontier; placed at C4 (one tier under the Qwen 3.6 anchor C5).", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B", "shows": "Qwen3 235B-A22B hybrid thinking model, tech report arXiv 2505.09388", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "Qwen3 base/pretrained family, released 2025 (tech report arXiv 2505.09388, May 2025), ~36T training tokens / 119 languages. Base SKUs incl Qwen3-30B-A3B-Base (30.5B total/3.3B active MoE) and Qwen3-235B-A22B-Base. VERIFIED live on HF June 2026: 30B-A3B-Base is Apache-2.0, 42,966 downloads/mo. Superseded by Qwen3.6." }, { "product": "Mistral Large 2", "org": "Mistral AI", "type": "model", "description": "Mistral AI's 123B-parameter dense model (July 2024, 128K context) under the Mistral Research License, Europe's flagship open-weights frontier model at release, fluent across nine languages and strong on code/math. Retired and replaced by Mistral Large 3 on 30 March 2025; relevant historically but no longer the active SKU. Mistral overall reported ~450K customers and $400M+ ARR by mid-2025.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:closed;license:Mistral-Research-License(MRL, non-OSI: research/non-commercial only; commercial use requires a paid Mistral agreement)", "confidence": "high", "note": "Weights downloadable but under the Mistral Research License, non-commercial/research-only (CONFIRMED: card explicitly restricts to personal/scientific/academic non-profit use; commercial requires contacting Mistral). Notable contrast with Mistral Large 3's Apache-2.0. No base-only weights published; public SKU is instruct.", "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407", "shows": "123B instruct model, Mistral AI Research License (research/non-commercial only), 7,455 downloads/mo, no separate base release", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "Modest direct HF downloads (~7.5k/mo, confirmed); the research-only license caps commercial deployment, limiting reach vs permissively-licensed peers. Placed at 3.", "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407", "shows": "live repo 7,455 downloads/mo", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": "Mistral Large 2 (Nov '24): AA Intelligence Index 15, ranked #11/39, 'above average' for similar-size open non-reasoning models; vendor card reports MMLU 84.0, HumanEval 92, GSM8K 93.", "confidence": "medium", "note": "Solid 2024-generation model, mid-tier in 2026; below the Mistral Large 3 anchor (C4).", "sources": [ { "url": "https://artificialanalysis.ai/models/mistral-large-2", "shows": "AA Intelligence Index 15, rank #11/39, above-average for size", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407", "shows": "vendor card: MMLU 84.0, HumanEval 92, GSM8K 93", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Mistral Large 2 (Large-Instruct-2407, July 2024; revised 2411), 123B dense, 128K context, 80+ languages. Mistral ships only the instruct SKU publicly; no separately released base/pretrained weights (CONFIRMED: HF page lists only the instruct variant). Verified live on HF June 2026; superseded by Mistral Large 3." }, { "product": "Gemma 2", "org": "Google DeepMind", "type": "model", "description": "Google's previous-generation open-weights model family (2B/9B/27B) derived from Gemini research, using knowledge distillation from larger models during training. Released July 2024 under the Gemma license; now superseded by Gemma 3 (March 2025) which added multimodal vision capability. Gemma 2 9B remains widely used as a community fine-tuning base, with ~1.5M monthly Hugging Face downloads on the base 9B variant.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(gated, terms acceptance required);data:closed;code:closed;license:Gemma-License(non-OSI, Prohibited-Use-Policy)", "confidence": "high", "note": "Weights open but gated under the non-OSI Gemma License with a prohibited-use policy (CONFIRMED: repo requires accepting conditions to access files); marketed as 'open' so flagged open_washed. Matches the Gemma 3 anchor (O2 restricted).", "sources": [ { "url": "https://huggingface.co/google/gemma-2-27b", "shows": "27B base (PT, not instruct), Gemma license, gated access requiring terms acceptance, 5,893 downloads/mo, 13T tokens", "accessed": "2026-06-04" }, { "url": "https://blog.google/technology/developers/google-gemma-2/", "shows": "Google release post: Gemma license, 9B/27B, 27 Jun 2024", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Among the most-downloaded open model families; the 27B-IT shows ~133.6k downloads/mo and the family runs widely via HF, Ollama, Kaggle, Google AI tooling (27 finetunes, 14 quantizations, 10 adapters listed on the base card). Family-level reach in the tens-of-millions; calibrated to the Gemma 3 anchor (A5).", "sources": [ { "url": "https://huggingface.co/google/gemma-2-27b-it", "shows": "~133.6k downloads/mo for Gemma 2 27B-IT", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/google/gemma-2-27b", "shows": "27B base 5,893 downloads/mo, 27 finetunes / 14 quantizations / 10 adapters, broad multi-platform distribution", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU", "value": "Gemma 2 27B-IT: MMLU 75.2 (5-shot), HumanEval 51.8, GSM8K 74.0, MATH 42.3, BIG-Bench 74.9; 'best-in-class for size' at launch, competitive with 2x-larger 2024 models.", "confidence": "medium", "note": "Strong for size in 2024, mid-tier non-reasoning model in 2026; matches the Gemma 3 anchor (C3).", "sources": [ { "url": "https://huggingface.co/google/gemma-2-27b-it", "shows": "model card benchmarks: MMLU 75.2, HumanEval 51.8, GSM8K 74.0, MATH 42.3", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Gemma 2 family (9B, 27B), released 27 Jun 2024. Both pretrained (PT) and instruction-tuned (IT) weights released; scored on the 27B base (google/gemma-2-27b). 13T tokens (27B). VERIFIED live on HF June 2026: 27B base (PT), Gemma license, gated, 5,893 downloads/mo. Succeeded by Gemma 3." }, { "product": "Yi-1.5", "org": "01.AI", "type": "model", "description": "Kai-Fu Lee's 01.AI foundation model family in 6B, 9B, and 34B variants, with continued pretraining on 500B tokens over Yi-1.0. Strong performance on Chinese and English benchmarks. Apache 2.0 licensed. One of the top Chinese-origin open-weights models, though increasingly competing with Qwen for mindshare in the Chinese open source community.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed(pretraining corpus not released);code:partial(inference/usage examples; no training pipeline);license:Apache-2.0(OSI, permissive, no use restrictions)", "confidence": "high", "note": "Permissive Apache-2.0 weights but training data and full training code are not released, so open_weights not open_source. Apache-2.0 confirmed on HF base cards 2026-06-04.", "sources": [ { "url": "https://huggingface.co/01-ai/Yi-1.5-34B", "shows": "Apache-2.0 license, 34B base model (base vs chat variants listed), downloadable weights", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/01-ai/Yi-1.5-9B", "shows": "Apache-2.0 license, 9B base model, base/chat variants listed", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "HF monthly downloads are modest in 2026 (Yi-1.5-34B 10,925/mo, Yi-1.5-9B 16,052/mo, both confirmed 2026-06-04) as a 2024-era family superseded by Qwen/Llama/DeepSeek generations. Cumulative reach across variants + ModelScope/wisemodel mirrors lands in the early-adopter band; relevance has faded.", "sources": [ { "url": "https://huggingface.co/01-ai/Yi-1.5-34B", "shows": "10,925 downloads last month", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/01-ai/Yi-1.5-9B", "shows": "16,052 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU/GSM8K", "value": "Yi-1.5-34B base: MMLU 76.3, C-Eval 81.4, BBH 54.3; 34B-Chat GSM8K 90.2. Competitive in 2024 mid-size class.", "confidence": "medium", "note": "Strong for a 2024 sub-35B model but well below 2026 frontier open models (DeepSeek-V4, Qwen 3.6); calibrates near Falcon 3 (C2) on the dated-generation ladder. Benchmarks from the 01-ai GitHub repo (not the HF card).", "sources": [ { "url": "https://arxiv.org/abs/2403.04652", "shows": "Yi foundation-models tech report with benchmark methodology", "accessed": "2026-06-04" }, { "url": "https://github.com/01-ai/Yi-1.5", "shows": "Yi-1.5 base benchmark results (MMLU 76.3, BBH 54.3, C-Eval 81.4)", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "Yi-1.5 base family (6B/9B/34B + 32K context variants), open sourced 2024-05-13 by 01.AI. Continuously pre-trained on Yi with +500B tokens (3.6T total). Base SKUs (Yi-1.5-9B, Yi-1.5-34B) scored here; Yi-1.5-*-Chat are the instruct SKUs (finetuned_chat). Verified live on HF June 2026; a dated 2024-era generation vs 2026 frontier." }, { "product": "InternLM 2.5", "org": "Shanghai AI Laboratory", "type": "model", "description": "Shanghai AI Lab's open-weights model family. The InternLM 2.5 generation (7B/20B, July 2024, 1M context via dynamic NTK interpolation) was superseded by InternLM3 in January 2025, an 8B model trained on only 4T tokens that reportedly matches GPT-4o-mini, with integrated 'deep thinking' chain-of-thought mode and ~4x the data efficiency (IQPT) of Llama 3.1. A leading Chinese academic-origin foundation model, heavily used in the Chinese research community.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(custom 'Free Commercial License', academic free, commercial use free but requires an application form);data:closed(pretraining corpus not released);code:open(Apache-2.0 training/inference code on GitHub);license:Apache-2.0(code, OSI) + custom weights license(non-OSI, application step)", "confidence": "medium", "note": "Code is OSI (Apache-2.0) but the weights license is a custom non-OSI 'Free Commercial License' requiring a commercial-license application form (HF card confirms form link, 2026-06-04), so open_weights not open_source; flagged for the non-OSI weights term.", "sources": [ { "url": "https://huggingface.co/internlm/internlm2_5-7b", "shows": "model card: code Apache-2.0; weights fully open for academic research and free commercial usage via application form (QQ survey link)", "accessed": "2026-06-04" }, { "url": "https://github.com/InternLM/InternLM/blob/main/model_cards/internlm2.5_7b.md", "shows": "InternLM2.5-7B base model card with benchmark results", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "HF monthly downloads modest in 2026 (3,521/mo for the 7B base, confirmed 2026-06-04) as a 2024-era model; distributed via HF, Ollama, ModelScope. Cumulative reach across the InternLM2.5 family + mirrors sits in the early-adopter band; specialist/research uptake, not production scale.", "sources": [ { "url": "https://huggingface.co/internlm/internlm2_5-7b", "shows": "3,521 downloads last month", "accessed": "2026-06-04" }, { "url": "https://ollama.com/internlm/internlm2.5", "shows": "InternLM2.5 distributed via Ollama", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU", "value": "InternLM2.5-7B base: MMLU 71.6 (5-shot, OpenCompass); matches Yi-1.5-9B, beats Llama-3-8B (66.4).", "confidence": "medium", "note": "Strong sub-8B model for 2024 with 1M-context, but below 2026 frontier; calibrates near Falcon 3 (C2) / Yi-1.5 (C2) on the dated small-model ladder. MMLU 71.6 confirmed on HF card 2026-06-04.", "sources": [ { "url": "https://huggingface.co/internlm/internlm2_5-7b", "shows": "InternLM2.5-7B base MMLU 71.6 (OpenCompass)", "accessed": "2026-06-04" }, { "url": "https://github.com/InternLM/InternLM/blob/main/model_cards/internlm2.5_7b.md", "shows": "InternLM2.5-7B base MMLU 71.6 (OpenCompass)", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "InternLM2.5-7B base (~8B params, 1M-token context), released 2024 (InternLM2 tech report arXiv:2403.17297). Base SKU scored here; InternLM2.5-7B-Chat is the instruct SKU (finetuned_chat). Verified live on HF June 2026; a 2024-era 7B model." }, { "product": "GPT-4o", "org": "OpenAI", "type": "model", "description": "OpenAI's legacy multimodal foundation model (May 2024) that natively handled text, image, and audio. Now two numerical generations behind: superseded by the GPT-5 line (GPT-5 in August 2025, then 5.2, 5.3, 5.4, and GPT-5.5 on April 23, 2026, the first fully retrained base since GPT-4.5). All access remains API-only via post-trained variants; GPT-4o is being sunset as the ChatGPT default in favor of GPT-5.5 Instant (May 5, 2026). Listed as the historical closed-source baseline that open-weights models were measured against through 2024-2025.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "No downloadable weights; API/ChatGPT-only, like the GPT-5 anchor (O1 closed). Confirmed API-only with 128K context and Oct 2023 cutoff on OpenAI docs 2026-06-04.", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4o", "shows": "GPT-4o API-only model, 128K context, text+image input, no open weights, Oct 2023 cutoff", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "GPT-4o was the default ChatGPT model for much of 2024-2025 before GPT-5; ChatGPT crossed ~900M WAU (Feb 2026) and ~1B (May 2026). Attributed honestly to the ChatGPT surface GPT-4o powered, not a standalone GPT-4o count; well above the >10M threshold. WAU figure corroborated via aggregator (demandsage), not a primary OpenAI disclosure, but the threshold is not close so no downgrade.", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4o", "shows": "GPT-4o served as a primary OpenAI API/ChatGPT model", "accessed": "2026-06-04" }, { "url": "https://www.demandsage.com/chatgpt-statistics/", "shows": "ChatGPT ~900M+ weekly active users (aggregator, corroborating surface scale)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU/MMMU", "value": "GPT-4o launch-era external figures: MMLU ~88.7, MMMU ~69.1 (NOT confirmed on current OpenAI docs, which carry no benchmark numbers); a 2024 frontier non-reasoning multimodal model.", "confidence": "medium", "note": "Frontier in 2024 but surpassed by 2026 reasoning frontier (GPT-5 C5, Gemini 3.5 C5); per the recipe, a mid-2025-and-earlier model that has been surpassed scores 3, not 5. Caveat: the cited OpenAI docs page (verified 2026-06-04) shows NO benchmark figures; it only calls GPT-4o 'our most capable model outside our o-series'; the MMLU/MMMU values are launch-era external numbers and need a primary benchmark source to back the headline.", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4o", "shows": "GPT-4o positioned as 'most capable model outside o-series', 128K context, text+image; no benchmark figures present", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "OpenAI GPT-4o, multimodal (text+image in, text out), 128K context; first snapshot gpt-4o-2024-05-13, knowledge cutoff Oct 2023. API-only, no released weights. Verified live in OpenAI's model docs June 2026 (developers.openai.com). A 2024-era flagship, since surpassed by the GPT-5 line but still served via API/ChatGPT." }, { "product": "GPT-4.1", "org": "OpenAI", "type": "model", "description": "Previous-generation general-purpose instruction-tuned model (April 2025). Strong on HumanEval and real-world coding with 1M token context, but two numerical generations behind by mid-2026; OpenAI's current production line is GPT-5 → 5.2 → 5.4 → 5.5 (April 23, 2026), with GPT-5.5 Pro and 5.5 Instant variants. GPT-4.1 remains available via API and is still used in legacy integrations.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "No downloadable weights; API-only, like the GPT-5 anchor (O1 closed). Confirmed API-only with 1M context (1,047,576) on OpenAI docs 2026-06-04.", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4.1", "shows": "GPT-4.1 API-only model, 1,047,576 context window, no open weights", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "Widely available developer API model (default API-tier non-reasoning model in the GPT-4.1 family, incl. mini/nano) but not the primary ChatGPT default surface (that's the GPT-5 line by 2026). No disclosed standalone user/usage count; placed at 4 on broad developer-API distribution rather than a verified figure. Directional.", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4.1", "shows": "GPT-4.1 served broadly via OpenAI API as the non-reasoning instruction/tool model", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:n/a", "value": null, "confidence": "low", "note": "OpenAI's docs market GPT-4.1 as a strong non-reasoning model (instruction-following, tool calling, 1M context) but the docs page surfaces no SWE-bench/MMLU figure (confirmed 2026-06-04: no quantitative benchmarks present), and it explicitly recommends GPT-5 for complex tasks. A non-reasoning 2025 model below the 2026 reasoning frontier; scored 3 on positioning, with no headline benchmark number asserted (value null).", "sources": [ { "url": "https://developers.openai.com/api/docs/models/gpt-4.1", "shows": "GPT-4.1 described as 'smartest non-reasoning model', recommends GPT-5 for complex tasks, no benchmark figures", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "OpenAI GPT-4.1, snapshot gpt-4.1-2025-04-14, ~1M-token context (1,047,576), knowledge cutoff Jun 2024; OpenAI's 'smartest non-reasoning' model, strong at instruction-following/tool calling. API-only, no released weights. Verified live in OpenAI's model docs June 2026; positioned below the GPT-5 reasoning line." }, { "product": "Amazon Nova", "org": "Amazon", "type": "model", "description": "Amazon's in-house foundation model family on AWS Bedrock. Originally launched December 2024 (Nova Micro/Lite/Pro), then extended in April 2025 with Amazon Nova Premier (1M context, designed as a 'teacher' for model distillation). The Nova 2 family was announced at AWS re:Invent 2025, Nova 2 Lite (cost-effective reasoning), Nova 2 Pro (most intelligent tier, in preview for Nova Forge customers), and Nova 2 Sonic (speech-to-speech). All Nova 2 models support up to 1M tokens of context with 65K-token responses. Amazon's play to reduce dependence on Anthropic and Meta within its own cloud stack; closed-source, Bedrock-only.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(Bedrock/SageMaker API-only)", "confidence": "high", "note": "Closed proprietary models served only via AWS Bedrock/SageMaker; no downloadable weights. Confirmed on live vendor page.", "sources": [ { "url": "https://aws.amazon.com/nova/models/", "shows": "Nova 2 Lite/Pro(Preview)/Sonic + Multimodal Embedding via Bedrock/SageMaker, proprietary", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Distributed to AWS Bedrock customer base; live vendor page (2026-06-04) names case studies Guardoc Health, LangChain, Cisco, Securonix (33% faster response), and AWS Transform (up to 60% tool-calling efficiency). RE-VERIFIED: Guardoc Health and Securonix are BOTH present on the live page, so the earlier 'source_attribution_corrected' edit (which removed Guardoc Health) was itself mistaken and is reverted. No disclosed user count; level 3 reflects platform distribution, not a measured figure.", "sources": [ { "url": "https://aws.amazon.com/nova/models/", "shows": "Bedrock/SageMaker distribution + named case studies (Guardoc Health, LangChain, Cisco, Securonix, AWS Transform); proprietary; no benchmark scores", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:n/a", "value": null, "confidence": "low", "note": "Live vendor page confirms NO standardized benchmark scores; positioning is price-performance + case studies. Mid-tier vs 2026 frontier; scored 3 directionally.", "sources": [ { "url": "https://aws.amazon.com/nova/models/", "shows": "price-performance positioning, no standardized benchmark scores", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Amazon Nova foundation models; current generation per live vendor page (June 2026): Nova 2 Lite, Nova 2 Pro (Preview), Nova 2 Sonic, plus Nova Multimodal Embedding (earlier Micro/Lite/Pro launched Dec 2024). Proprietary/API-only via Amazon Bedrock + SageMaker; no open weights. Listed as closed counterpart. Verified live June 2026." }, { "product": "Reka Core", "org": "Reka AI", "type": "model", "description": "Reka AI's multimodal foundation model that natively processes text, images, video, and audio (128K context). Founded by ex-DeepMind / Google / Meta researchers, Reka Core was originally pitched as a GPT-4V / Claude 3 Opus / Gemini Ultra rival. The company has since extended the stack with Reka Flash 3 (open-weights, 2025) and Reka Nexus (a multi-agent 'AI workforce' product). Reka Core remains API-only as the closed flagship; no verified 2026 successor to Core itself was found in this audit, though the surrounding product line has evolved.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed(mix of public+licensed proprietary);code:closed;license:Proprietary(API/on-prem, no weights)", "confidence": "high", "note": "Proprietary multimodal model; API/on-prem/on-device deployment but no downloadable weights. Confirmed on launch page.", "sources": [ { "url": "https://reka.ai/news/reka-core-our-frontier-class-multimodal-language-model", "shows": "Reka Core via API/on-prem/on-device, proprietary, no open weights", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "low", "note": "Distributed via Reka API + Snowflake Cortex + Oracle Cloud Marketplace; a 2024 frontier-challenger now well behind. No disclosed usage; level 2 directional. (Snowflake/Oracle partnerships are not enumerated on the cited launch page itself.)", "sources": [ { "url": "https://reka.ai/news/reka-core-our-frontier-class-multimodal-language-model", "shows": "availability via API/on-prem/on-device partnerships", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMMU", "value": "Comparable to GPT-4V on MMMU; outperforms Claude-3 Opus on multimodal human eval; surpasses Gemini Ultra on video (Apr 2024 claims)", "confidence": "low", "note": "Frontier-class for early 2024 (~GPT-4V tier) but ~2yr stale vs 2026 anchors. Headline claims CONFIRMED present on the primary launch page. Scored 2.", "sources": [ { "url": "https://reka.ai/news/reka-core-our-frontier-class-multimodal-language-model", "shows": "MMMU ~GPT-4V, beats Claude-3 Opus on multimodal human eval, surpasses Gemini Ultra on video", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "Reka Core, frontier-class multimodal LLM announced 15 Apr 2024; 128K context. Proprietary: API/on-prem/on-device; no open weights. Instruction/multimodal-chat model. Verified live: launch page confirms API/on-prem/on-device availability and the MMMU/Claude-3-Opus claims. NOTE: '~5T tokens / 32 languages / from scratch' details are NOT on the cited launch page (they originate in Reka's separate technical report); verify against that source if retained. >12mo old; scored best-effort and flagged stale." }, { "product": "Inflection 3.0", "org": "Inflection AI", "type": "model", "description": "Inflection AI's foundation model powering the Pi assistant and the company's enterprise 'Inflection for Enterprise' tier. After key staff (Mustafa Suleyman et al.) departed to Microsoft AI in March 2024, Inflection pivoted to enterprise. Inflection 3.0 is the current flagship, powered by Intel Gaudi 3 with on-prem or AI Cloud deployment options; no new generation publicly confirmed by May 2026. Notable as a case study in foundation-model market concentration dynamics: pace has slowed dramatically post-pivot.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API/enterprise-licensed, no weights)", "confidence": "high", "note": "Closed proprietary models via API + enterprise licensing; no downloadable weights; fine-tuned models are the customer's alone. Confirmed via primary Intel Newsroom press release (2026-06-04).", "sources": [ { "url": "https://www.maginative.com/article/inflection-ai-unveils-enterprise-offering-with-new-inflection-3-0-models-2/", "shows": "Inflection 3.0 (Pi + Productivity), enterprise/API, proprietary (aggregator source)", "accessed": "2026-06-04" }, { "url": "https://newsroom.intel.com/artificial-intelligence/inflection-ai-intel-launch-enterprise-ai-system", "shows": "PRIMARY: Inflection 3.0 (Pi 3.0 + Productivity 3.0) enterprise AI system, Oct 2024, proprietary; fine-tuned models customer-exclusive, never shared", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "reported_traction", "confidence": "low", "note": "Post-Microsoft acqui-hire pivot to B2B; Pi consumer surface wound down. Enterprise API exists but no disclosed usage; effectively stalled. Level 1 directional.", "sources": [ { "url": "https://en.wikipedia.org/wiki/Inflection_AI", "shows": "team absorbed by Microsoft, pivot to enterprise, Pi sunset", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:n/a", "value": null, "confidence": "low", "note": "2024 EQ/productivity-focused model, never a leaderboard frontier model; development effectively halted after Microsoft talent move. Low vs 2026 anchors. No fresh independent benchmark. Scored 2.", "sources": [ { "url": "https://www.maginative.com/article/inflection-ai-unveils-enterprise-offering-with-new-inflection-3-0-models-2/", "shows": "Pi (EQ) + Productivity positioning, no frontier-benchmark claims", "accessed": "2026-06-04" } ] }, "maturity": 1.7, "mature": false, "version_note": "Inflection 3.0 (Pi 3.0 + Productivity 3.0), released 7 Oct 2024 with 'Inflection for Enterprise'. Proprietary/API + enterprise-licensed; no open weights; fine-tuned models are customer-exclusive. Frontier research team absorbed by Microsoft (2024-25); pivot to B2B. EXISTENCE now confirmed via PRIMARY sources (Intel Newsroom press release + BusinessWire 'Introducing Inflection for Enterprise', 7 Oct 2024), not aggregator-only. >12mo old; scored best-effort, flagged stale." }, { "product": "GLM-4-9B", "org": "Zhipu AI", "type": "model", "description": "GLM-4-9B base (THUDM/glm-4-9b-hf), 9B dense, 8K context base version, released ~Jun 18 2024 (arXiv 2406.12793). VERIFIED on HF: genuine pretrained base (chat variant GLM-4-9B-Chat separate). Weights downloadable. >12mo old but a foundational open base; scored best-effort, flagged stale.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF, BF16 safetensors);data:closed;code:partial(inference/usage);license:glm-4(custom non-OSI: commercial-use registration, 'Built with glm-4' attribution, behavioral/national-security restrictions, Chinese jurisdiction)", "confidence": "high", "note": "Weights downloadable but glm-4 license is non-OSI with registration-gated commercial use + use restrictions, restricted, not open_weights. HF card confirms a custom glm-4 LICENSE file (full restriction text in the linked LICENSE).", "sources": [ { "url": "https://huggingface.co/THUDM/glm-4-9b-hf", "shows": "base GLM-4-9B, 8K context, glm-4 custom license, downloadable weights", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/THUDM/glm-4-9b-hf/blob/main/LICENSE", "shows": "registration-for-commercial, attribution + use restrictions, non-OSI", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "low", "note": "Base GLM-4-9B-hf ~1,497 downloads/month on HF (confirmed live; chat variants out-download the base); 2024-era 9B superseded by GLM-4.5/4.6/5.x. Level 2 on modest base-variant volume.", "sources": [ { "url": "https://huggingface.co/THUDM/glm-4-9b-hf", "shows": "~1,497 downloads last month (base variant)", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:n/a", "value": "Model card: MMLU 74.7, C-Eval 77.1, GSM8K 84.0, HumanEval 70.1; beats Llama-3-8B", "confidence": "low", "note": "Strong small base for 2024 (beats Llama-3-8B per card) but low vs 2026 frontier and vs the GLM-5.x family. Added the card's reported benchmark figures. Scored 2.", "sources": [ { "url": "https://huggingface.co/THUDM/glm-4-9b-hf", "shows": "model card: MMLU 74.7/C-Eval 77.1/GSM8K 84.0/HumanEval 70.1, outperforms Llama-3-8B, 9B base", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "GLM-4-9B base (THUDM/glm-4-9b-hf), 9B dense, 8K context base version, released ~Jun 18 2024 (arXiv 2406.12793). VERIFIED on HF: genuine pretrained base (chat variant GLM-4-9B-Chat separate). Weights downloadable. >12mo old but a foundational open base; scored best-effort, flagged stale." }, { "product": "Kimi K2.6", "org": "Moonshot AI", "type": "model", "description": "Moonshot AI's open-weight 1T-parameter MoE released 2026-04-20 under a Modified MIT license, with 256K context. Top-10 on SWE-bench Verified at 80.2% and ties GPT-5.4 on agentic/coding benchmarks. Notable for a 300-sub-agent / 4000-step swarm capability. Moonshot ships native multi-agent scaffolding with the weights. One of the four credible open-weight frontier contenders (alongside DeepSeek V4, Qwen3-Coder-Next, and GLM-4.6) for chat/coding workloads in 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference/deploy; no training data or full pipeline);license:Modified-MIT(standard MIT below 100M MAU / $20M monthly rev; above thresholds requires 'Kimi K2' UI attribution, attribution-only, not a use cap)", "confidence": "high", "note": "weights:open(downloadable on HF);data:closed;code:partial(inference/deploy; no training data or full pipeline);license:Modified-MIT(standard MIT below 100M MAU / $20M monthly rev; above thresholds requires 'Kimi K2' UI attribution, attribution-only, not a use cap)", "sources": [ { "url": "https://huggingface.co/moonshotai/Kimi-K2.6", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://miraflow.ai/blog/kimi-k2-6-explained-moonshot-ai-open-source-model-ties-gpt-5-5-coding", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.verdent.ai/guides/what-is-kimi-k2-6", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Free on kimi.com + Kimi app + API + Kimi Code; partners Vercel, Baseten, Ollama, Factory.ai validated K2.6 in production (Vercel reports >50% improvement on Next.js benchmark). OpenRouter shows sustained post-launch usage spikes indicating genuine production adoption. No exact download/user count published; placed at 4 (1-10M-equivalent) on production-traction signal.", "sources": [ { "url": "https://www.marktechpost.com/2026/04/20/moonshot-ai-releases-kimi-k2-6-with-long-horizon-coding-agent-swarm-scaling-to-300-sub-agents-and-4000-coordinated-steps/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://blog.kilo.ai/p/kimi-k26-has-arrived-an-open-weight", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench/Terminal-Bench", "value": null, "confidence": "high", "note": "SWE-bench Verified 80.2%, SWE-bench Pro 58.6% (up from 50.7% in K2.5), Terminal-Bench 2.0 66.7% (up from 50.8%). Reported tying GPT-5.5 on coding and beating some top US models; specialized for long-horizon agentic coding (300-agent swarms, 4,000 coordinated steps). Frontier-tier for agentic/coding work.", "sources": [ { "url": "https://benchlm.ai/models/kimi-2-6", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://officechai.com/ai/kimi-k2-6-benchmarks/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.kimi.com/blog/kimi-k2-6", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Kimi K2.6, 1T total / 32B active MoE (384 experts, MLA), 256K context, 400M-param MoonViT vision encoder. Released April 20 2026 (rolled to all subscribers Apr 13). Verified live on HF June 2026." }, { "product": "OLMo 2", "org": "Ai2", "type": "model", "description": "OLMo 2 family (1B/7B/13B/32B), base + Instruct. Latest 32B-Instruct (0325) and 1B (0425, updated May 2025). Verified existing June 2026; succeeded by OLMo 3 announced late 2025/2026.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(Dolma/Tulu, all stages incl pre/mid/post-training);code:open(training code+recipes+logs);checkpoints:open(thousands of intermediate);license:Apache-2.0(OSI)", "confidence": "high", "note": "weights:open(Apache-2.0);data:open(Dolma/Tulu, all stages incl pre/mid/post-training);code:open(training code+recipes+logs);checkpoints:open(thousands of intermediate);license:Apache-2.0(OSI)", "sources": [ { "url": "https://allenai.org/blog/olmo2", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/allenai/OLMo-2-1124-7B", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/pdf/2501.00656", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/collections/allenai/olmo-2", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "ATOM Report (Apr 2026) measures ~14.8M cumulative downloads for the OLMo family on Hugging Face; OLMo 2 1B alone ~323k downloads. Falls in 1-10M cumulative downloads band. Strong in research/interpretability circles but small vs Qwen/Llama scale.", "sources": [ { "url": "https://arxiv.org/html/2604.07190v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/allenai/OLMo-2-0425-1B", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU/GPQA", "value": null, "confidence": "high", "note": "OLMo 2 32B is the most capable; reported competitive with GPT-3.5-Turbo / GPT-4o-mini class and Qwen 2.5 on academic benchmarks but below current frontier open models (DeepSeek-V4, Qwen 3.6). Mid-tier capability; the value proposition is full openness, not raw SOTA.", "sources": [ { "url": "https://allenai.org/blog/olmo2", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/pdf/2501.00656", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "OLMo 2 family (1B/7B/13B/32B), base + Instruct. Latest 32B-Instruct (0325) and 1B (0425, updated May 2025). Verified existing June 2026; succeeded by OLMo 3 announced late 2025/2026." }, { "product": "Pythia", "org": "EleutherAI", "type": "model", "description": "EleutherAI's suite of 16 models ranging from 70M to 12B parameters, all trained on the same data (The Pile) in the same order, with intermediate checkpoints released throughout training. Designed as a scientific tool for studying how language models learn, not for production use. Fully open (model, data, training code, intermediate checkpoints). The gold standard for LLM interpretability and scaling research.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(the Pile, with dataloader reconstruction tools);code:open(training+analysis code, GitHub);checkpoints:open(154 per model);license:Apache-2.0(OSI)", "confidence": "high", "note": "weights:open(Apache-2.0);data:open(the Pile, with dataloader reconstruction tools);code:open(training+analysis code, GitHub);checkpoints:open(154 per model);license:Apache-2.0(OSI)", "sources": [ { "url": "https://arxiv.org/abs/2304.01373", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/EleutherAI/pythia", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/EleutherAI/pythia-12b/blob/main/README.md", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "Heavily used as a research/interpretability benchmark suite (de facto standard for training-dynamics studies); cited extensively. Hugging Face download volume across the 16 models is in the 100k-1M aggregate range, meaningful for research but not production deployment. Honest signal: research adoption, not end-user scale.", "sources": [ { "url": "https://huggingface.co/EleutherAI/pythia-6.9b", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.eleuther.ai/papers-blog/pythia-a-suite-for-analyzing-large-language-modelsacross-training-and-scaling", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 1, "basis": "benchmark:n/a", "value": null, "confidence": "high", "note": "Not built for capability. Pythia models (max 12B, 2023-era Pile training) are intentionally research instruments for studying scaling/learning dynamics, not competitive on modern capability benchmarks. Scored 1 on a capability axis where the comparison set is 2026 frontier models.", "sources": [ { "url": "https://arxiv.org/abs/2304.01373", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 1.6, "mature": false, "version_note": "Pythia suite: 16 models, 70M-12B params (8 sizes x Pile/deduped-Pile), 154 intermediate checkpoints each. Released 2023 (arXiv 2304.01373); remains the standard interpretability/training-dynamics suite as of June 2026." }, { "product": "Llama 4 Scout / Maverick", "org": "Meta", "type": "model", "description": "Llama 4 herd, released April 2025: Scout (17B active / 109B total MoE, 10M-token context) and Maverick (17B active / 400B total MoE, 1M-token context), natively multimodal. Behemoth still unreleased/preview as of June 2026. Both Scout/Maverick verified live on HF June 2026.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference/utils via llama-models, no training code);license:Llama-4-Community(non-OSI: 700M MAU commercial cap as of Apr 2025 + no-compete/no-train-competing-LLM clause)", "confidence": "high", "note": "weights:open(downloadable on HF);data:closed;code:partial(inference/utils via llama-models, no training code);license:Llama-4-Community(non-OSI: 700M MAU commercial cap as of Apr 2025 + no-compete/no-train-competing-LLM clause)", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/blog/llama4-release", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://royfactory.net/posts/ai/202512/meta-llama4-open-weights-scout-maverick-license/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "ATOM Report (Apr 2026): Llama family ~476.0M cumulative HF downloads (second to Qwen). Llama 4 specifically is a fraction of that and lost derivative-base share (Meta's new-fine-tune share fell to ~11% by Feb 2026 from a 44% peak). Family-level 1-10M+ easily; Llama 4 alone solidly in the 1-10M band.", "sources": [ { "url": "https://arxiv.org/html/2604.07190v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/meta-llama", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU-Pro/GPQA", "value": null, "confidence": "high", "note": "Maverick (instruct) reports MMLU-Pro 80.5, GPQA Diamond 69.8; positioned at/above GPT-4o and Gemini 2.0 Flash at launch (Apr 2025). By June 2026 this is mid-tier vs DeepSeek-V4 / Qwen 3.6 frontier open models. Scout's long-context retrieval reported weak (Fiction.LiveBench 15.6% at 128K).", "sources": [ { "url": "https://royfactory.net/posts/ai/202512/meta-llama4-open-weights-scout-maverick-license/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Llama 4 herd, released April 2025: Scout (17B active / 109B total MoE, 10M-token context) and Maverick (17B active / 400B total MoE, 1M-token context), natively multimodal. Behemoth still unreleased/preview as of June 2026. Both Scout/Maverick verified live on HF June 2026." }, { "product": "DeepSeek-V3.2", "org": "DeepSeek", "type": "model", "description": "Previous-generation general-purpose instruction-tuned MoE model (671B total / 37B active). Still widely used (4.2M+ monthly downloads) but superseded by DeepSeek-V4-Pro / V4-Flash (April 2026, 1M default context, 384K max output). The legacy `deepseek-chat` and `deepseek-reasoner` aliases route to V4-Flash from 2026-07-24.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF);data:closed(training-data synthesis described in report but corpus not released);code:partial(inference code in HF repo; no full training pipeline);license:MIT(OSI-style permissive, no use restrictions)", "confidence": "high", "note": "weights:open(MIT, on HF);data:closed(training-data synthesis described in report but corpus not released);code:partial(inference code in HF repo; no full training pipeline);license:MIT(OSI-style permissive, no use restrictions)", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/pdf/2512.02556", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://stable-learn.com/en/deepseek-v32-tech-report/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "ATOM Report (Apr 2026): China open models reached ~1.15B cumulative HF downloads with DeepSeek a primary driver; DeepSeek-V3.2 widely served (OpenRouter, multiple inference providers) with sustained post-launch production usage and frontier-at-10x-lower-cost positioning. Real usage well into the >10M-equivalent band across web app + API + derivatives.", "sources": [ { "url": "https://arxiv.org/html/2604.07190v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/deepseek/deepseek-v3.2", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://introl.com/blog/deepseek-v3-2-open-source-ai-cost-advantage", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:MMLU-Pro/GPQA/SWE-bench", "value": null, "confidence": "high", "note": "Tech report (arXiv 2512.02556): MMLU-Pro 85.0, GPQA Diamond 82.4, SWE-bench Verified 77.2; Speciale variant earns gold-medal IMO/IOI reasoning and AIME 2025 96.0. Near-frontier open model; sits just below DeepSeek-V4-Pro on the same family's curve.", "sources": [ { "url": "https://arxiv.org/html/2512.02556v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://stable-learn.com/en/deepseek-v32-tech-report/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "DeepSeek-V3.2, 671B total / 37B active MoE, introduces DeepSeek Sparse Attention (DSA). Open V3.2 + V3.2-Speciale (gold-medal IMO/IOI reasoning) released; Speciale dated Dec 1 2025; tech report arXiv 2512.02556. Verified live June 2026." }, { "product": "DeepSeek-V4-Pro", "org": "DeepSeek", "type": "model", "description": "DeepSeek-V4-Pro, 1.6T total / 49B active MoE, 1M-token context, hybrid CSA+HCA attention. Released April 24 2026 under MIT, alongside V4-Flash (284B). V4-Pro-Max = max-reasoning mode. Verified live June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF, both Pro and Flash);data:closed;code:partial(inference/serving; no training pipeline or data);license:MIT(permissive, no use restrictions)", "confidence": "high", "note": "weights:open(MIT, on HF, both Pro and Flash);data:closed;code:partial(inference/serving; no training pipeline or data);license:MIT(permissive, no use restrictions)", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://api-docs.deepseek.com/news/news260424", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://simonwillison.net/2026/apr/24/deepseek-v4/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Released Apr 24 2026; ~6 weeks of production at scoring time. Available open-weights + API (permanent pricing set May 22 2026) and served across providers (OpenRouter, DeepInfra). Strong early traction consistent with DeepSeek family scale, but too recent to confirm >10M-equivalent reach independently; placed at 4 pending sustained data.", "sources": [ { "url": "https://openrouter.ai/deepseek/deepseek-v4-pro", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://codersera.com/blog/deepseek-v4-pro-review-benchmarks-pricing-2026/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://deepinfra.com/blog/deepseek-v4-pro-model-overview", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench/MMLU-Pro/GPQA", "value": null, "confidence": "high", "note": "SWE-bench Verified 80.6% (trails Claude Opus 4.6 by ~0.2), MMLU-Pro 87.5, GPQA Diamond 90.1, LiveCodeBench Pass@1 93.5 (reported best of any model). V4-Pro-Max described as the strongest open source model available, at/near closed frontier on coding and reasoning.", "sources": [ { "url": "https://codersera.com/blog/deepseek-v4-pro-review-benchmarks-pricing-2026/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://benchlm.ai/models/deepseek-v4-pro-high", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.morphllm.com/deepseek-v4", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "DeepSeek-V4-Pro, 1.6T total / 49B active MoE, 1M-token context, hybrid CSA+HCA attention. Released April 24 2026 under MIT, alongside V4-Flash (284B). V4-Pro-Max = max-reasoning mode. Verified live June 2026." }, { "product": "Qwen 3.6", "org": "Alibaba", "type": "model", "description": "Qwen3.6 series, released early-mid 2026: open-weight variants incl Qwen3.6-35B-A3B (MoE, Apr 16 2026) and dense Qwen3.6-27B (Apr 22 2026), plus hosted flagship tiers Qwen3.6-Plus and Qwen3.6-Max (Preview) that are proprietary/API-only. GitHub QwenLM/Qwen3.6 verified live June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0 for open variants e.g. 27B/35B-A3B);data:closed;code:partial(inference/usage code on GitHub; no training pipeline/data);license:mixed(open variants Apache-2.0; Plus/Max flagship tiers proprietary, commercial-restricted/API-only)", "confidence": "high", "note": "weights:open(Apache-2.0 for open variants e.g. 27B/35B-A3B);data:closed;code:partial(inference/usage code on GitHub; no training pipeline/data);license:mixed(open variants Apache-2.0; Plus/Max flagship tiers proprietary, commercial-restricted/API-only)", "sources": [ { "url": "https://github.com/QwenLM/Qwen3.6", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.marktechpost.com/2026/04/22/alibaba-qwen-team-releases-qwen3-6-27b-a-dense-open-weight-model-outperforming-397b-moe-on-agentic-coding-benchmarks/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://llm-stats.com/models/qwen3.6-plus", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://aimlapi.com/blog/qwen-3-6-series-alibabas-open-source-llm-revolution-in-2026", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "ATOM Report (Apr 2026): Qwen family ~942.1M cumulative HF downloads (>50% of all global open source downloads), overtook Llama in Sept 2025; >113,000 derivative models (>200,000 incl tagged); 69% of new fine-tunes by Feb 2026 build on Qwen. Qwen 3.6 is the current flagship generation of that dominant family, unambiguously >10M reach.", "sources": [ { "url": "https://arxiv.org/html/2604.07190v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.scmp.com/tech/big-tech/article/3349552/alibabas-qwen-family-captures-over-50-global-open-source-downloads-report-finds", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.arturmarkus.com/alibabas-qwen-hits-700-million-downloads-on-hugging-face-overtakes-metas-llama-as-worlds-most-popular-open-source-ai-model/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench/Terminal-Bench", "value": null, "confidence": "high", "note": "Dense Qwen3.6-27B: SWE-bench Verified 77.2, Terminal-Bench 2.0 59.3 (reported matching Claude 4.5 Opus class), outperforming a 397B MoE on agentic coding. Flagship Qwen3.6-Plus reported matching Claude Opus 4.5 on SWE-bench/Terminal-Bench. Frontier-tier open family.", "sources": [ { "url": "https://www.marktechpost.com/2026/04/22/alibaba-qwen-team-releases-qwen3-6-27b-a-dense-open-weight-model-outperforming-397b-moe-on-agentic-coding-benchmarks/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.caixinglobal.com/2026-04-02/alibaba-releases-qwen-36-plus-ai-model-with-enhanced-coding-capabilities-102430395.html", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Qwen3.6 series, released early-mid 2026: open-weight variants incl Qwen3.6-35B-A3B (MoE, Apr 16 2026) and dense Qwen3.6-27B (Apr 22 2026), plus hosted flagship tiers Qwen3.6-Plus and Qwen3.6-Max (Preview) that are proprietary/API-only. GitHub QwenLM/Qwen3.6 verified live June 2026." }, { "product": "Mistral Large 3", "org": "Mistral AI", "type": "model", "description": "Mistral AI's first mixture-of-experts model since Mixtral and the current open-weights flagship, released December 2, 2025 under Apache 2.0. 41B active / 675B total parameters, 256K context, native multimodal support (text + images). Trained from scratch on 3000 H200 GPUs. Debuted at #2 on the LMArena OSS non-reasoning leaderboard. Combined with Ministral 3 (smaller dense variants) and Mistral Vibe CLI in the broader Mistral 3 stack. Runs at the compute cost of a ~41B dense model while accessing the capacity of a 675B one, Europe's flagship open-weights frontier model and the direct successor to the retired Mistral Large 2 (March 2025).", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0);data:closed;code:closed;license:Apache-2.0(OSI)", "confidence": "high", "note": "weights:open(Apache-2.0);data:closed;code:closed;license:Apache-2.0(OSI)", "sources": [ { "url": "https://mistral.ai/news/mistral-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Distributed via Hugging Face (NVFP4 checkpoint, vLLM-ready) and Microsoft Foundry as a production model. No published hard download/user counts found; level 3 reflects multi-platform enterprise distribution rather than a verified usage_volume number.", "sources": [ { "url": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://mistral.ai/news/mistral-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:LMArena", "value": null, "confidence": "high", "note": "Debuts #2 in LMArena OSS non-reasoning category (#6 among all OSS models). Per-benchmark MMLU-Pro/GPQA/SWE-bench not given in the launch post.", "sources": [ { "url": "https://mistral.ai/news/mistral-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Mistral Large 3 (mistral-large-3-25-12), announced 2 Dec 2025. Sparse MoE, 41B active / 675B total params. Base + instruct released; reasoning variant announced but not shipped as of mid-2026." }, { "product": "Gemma 3", "org": "Google DeepMind", "type": "model", "description": "Google's current open-weights model family, released March 12, 2025, and the first open-weights Gemma generation with native vision-language capability. Available in 1B/4B/12B/27B variants (the 1B is text-only; the 4B/12B/27B are vision-language models), with a 128K context window on the larger variants and native multilingual support for 140+ languages. The Gemma 3 27B fits at Q4 on a single 3090 yet competes with much larger 400B-600B models on standard evals. Released under the Gemma license on Hugging Face and Kaggle. Google's primary contribution to the open-weights ecosystem in 2025-2026.", "openness": { "score": 2, "class": "restricted", "components": "weights:open;data:closed;code:closed;license:Gemma-License(non-OSI,Prohibited-Use-Policy)", "confidence": "high", "note": "weights:open;data:closed;code:closed;license:Gemma-License(non-OSI,Prohibited-Use-Policy)", "sources": [ { "url": "https://ai.google.dev/gemma/docs/core/model_card_3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/blog/gemma3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.deeplearning.ai/the-batch/google-releases-gemma-3-vision-language-models-with-open-weights/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Gemma family is among the most-downloaded open model families on Hugging Face; Gemma 3 variants run on consumer hardware and are widely deployed via HF, Ollama, and Google AI tooling. Family-level downloads are in the tens-of-millions+ range.", "sources": [ { "url": "https://huggingface.co/blog/gemma3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://blog.google/technology/developers/gemma-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU-Pro", "value": null, "confidence": "high", "note": "Gemma 3 27B-IT: MMLU-Pro 67.5, GPQA Diamond 42.4 (0-shot), LMArena/Chatbot Arena Elo ~1338 (top-10, above much larger non-thinking OSS models). Strong for size but a non-reasoning, mid-2025 model.", "sources": [ { "url": "https://ai.google.dev/gemma/docs/core/model_card_3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/html/2503.19786v1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Gemma 3 family (1B, 4B, 12B, 27B), released 12 Mar 2025. 4B/12B/27B are vision-language; 1B is text-only. Both pretrained and instruction-tuned weights released. (Successor Gemma 4 has since appeared but Gemma 3 is the scored product.)" }, { "product": "GLM-5.1", "org": "Zhipu / Z.ai", "type": "model", "description": "GLM-5.1 released 7 Apr 2026 by Z.ai (formerly Zhipu AI). Post-training upgrade on GLM-5 base; 744B-param MoE. Weights on Hugging Face under MIT.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT);data:closed;code:closed;license:MIT(OSI)", "confidence": "high", "note": "weights:open(MIT);data:closed;code:closed;license:MIT(OSI)", "sources": [ { "url": "https://www.buildfastwithai.com/blogs/glm-5-1-open-source-review-2026", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://mlhive.com/2026/04/zhipu-ai-glm-5-1-744b-open-source-release", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://llm-stats.com/models/glm-5.1", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Widely covered as a top open coding model and available via Z.ai API, OpenRouter, and HF; strong reported uptake but only ~2 months old at scoring time, no verified hard user/download count. Level 3 reflects reported traction, not a measured usage_volume figure.", "sources": [ { "url": "https://www.buildfastwithai.com/blogs/glm-5-1-open-source-review-2026", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://wavespeed.ai/blog/posts/glm-5-1-vs-claude-gpt-gemini-deepseek-llm-comparison/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench", "value": null, "confidence": "high", "note": "SWE-Bench Pro 58.4 (reported #1 at release, edging GPT-5.4 57.7 and Claude Opus 4.6 57.3); Terminal-Bench 2.0 63.5. Frontier-tier agentic coding for an open-weights model.", "sources": [ { "url": "https://serenitiesai.com/articles/glm-5-1-zhipu-coding-benchmark-claude-opus-comparison-2026", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.modemguides.com/blogs/ai-news/glm-5-1-open-source-benchmarks-local-ai", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.vals.ai/models/zai_glm-5.1-thinking", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "GLM-5.1 released 7 Apr 2026 by Z.ai (formerly Zhipu AI). Post-training upgrade on GLM-5 base; 744B-param MoE. Weights on Hugging Face under MIT." }, { "product": "Phi-4", "org": "Microsoft", "type": "model", "description": "Microsoft's 14B-parameter small language model that punches well above its weight class on reasoning, math, and code, released December 2024 under MIT license. Trained on heavily curated and synthetic data, demonstrating the 'small models with high-quality data' thesis. As of May 2026 the Phi-4 line is still current: Microsoft has extended it with Phi-4-mini, Phi-4-multimodal, and Phi-4-reasoning / Phi-4-reasoning-plus (May 2025), plus Phi-4-reasoning-vision-15B (March 2026). No Phi-5 has been announced.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT);data:closed;code:closed;license:MIT(OSI)", "confidence": "high", "note": "weights:open(MIT);data:closed;code:closed;license:MIT(OSI)", "sources": [ { "url": "https://the-decoder.com/microsoft-releases-full-phi-4-model-with-weights-under-mit-license/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://venturebeat.com/ai/microsoft-makes-powerful-phi-4-model-fully-open-source-on-hugging-face", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Phi-4 is a widely downloaded small model on Hugging Face, distributed via Azure AI Foundry, Ollama, and many local-inference stacks; download volume for the Phi-4 line is in the millions range. Level 4 reflects broad small-model adoption.", "sources": [ { "url": "https://venturebeat.com/ai/microsoft-makes-powerful-phi-4-model-fully-open-source-on-hugging-face", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://artificialanalysis.ai/models/phi-4", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:GPQA", "value": null, "confidence": "high", "note": "14B model with outsized math/reasoning: ~56.1% on graduate-level (GPQA-style) science questions, 80.4% on competition math, strong AMC results; reportedly beats GPT-4 (its teacher) on some STEM. Strong-for-size but a small/older model vs 2026 frontier.", "sources": [ { "url": "https://the-decoder.com/microsoft-releases-full-phi-4-model-with-weights-under-mit-license/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://artificialanalysis.ai/models/phi-4", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Phi-4, 14B dense model, weights released on Hugging Face under MIT (full release Jan 2025 following Dec 2024 announcement). Reasoning variants (Phi-4-reasoning / -reasoning-plus) released later under open weights but are separate SKUs; this record scores base Phi-4 14B." }, { "product": "Falcon 3", "org": "Technology Innovation Institute", "type": "model", "description": "TII's open-weights model line from Abu Dhabi. The original Falcon 3 family (1B/3B/7B/10B Apache-2.0, December 2024) has been extended in 2025-2026 by Falcon-H1 (a hybrid Mamba-Transformer architecture spanning 0.5B-34B with native 18-language support) and Falcon-H1 Arabic (3B/7B/34B, January 2026), positioned as the world's leading Arabic open model. Falcon H1R 7B (January 2026) outperforms Qwen and Nemotron variants up to 7x its size on reasoning. Adoption remains modest vs Llama/Qwen at equivalent size (Falcon3-7B-Instruct ~23K monthly HF downloads).", "openness": { "score": 2, "class": "restricted", "components": "weights:open;data:closed;code:closed;license:TII-Falcon-License-2.0(Apache-based+AUP,non-OSI)", "confidence": "high", "note": "weights:open;data:closed;code:closed;license:TII-Falcon-License-2.0(Apache-based+AUP,non-OSI)", "sources": [ { "url": "https://huggingface.co/blog/falcon3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/tiiuae/Falcon3-7B-Base", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.tii.ae/news/falcon-3-uaes-technology-innovation-institute-launches-worlds-most-powerful-small-ai-models", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "high", "note": "Available on Hugging Face with moderate uptake at launch as a sub-13B SLM; no large verified usage figures and relevance has faded against newer SLMs. Level 2 reflects modest reported traction.", "sources": [ { "url": "https://venturebeat.com/ai/uaes-falcon-3-challenges-open-source-leaders-amid-surging-demand-for-small-ai-models", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/tiiuae/Falcon3-7B-Base", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU-Pro", "value": null, "confidence": "high", "note": "Falcon3-10B-Base: MMLU 73.1, MMLU-Pro 42.5, GSM8K 83.0; Falcon3-7B-Base: MMLU 67.4, MMLU-Pro 39.2. Competitive among sub-13B in 2024 but low vs 2026 frontier.", "sources": [ { "url": "https://huggingface.co/blog/falcon3", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "Falcon 3 family (1B, 3B, 7B, 10B), released 17 Dec 2024 by TII. Trained on 14T tokens. Sub-13B SLMs; now a dated generation relative to 2026 frontier." }, { "product": "MiniMax M3", "org": "MiniMax", "type": "model", "description": "MiniMax M3 launched 1 Jun 2026: API-first at launch with MiniMax Sparse Attention, 1M-token context, native multimodal. Open weights have since been released on Hugging Face (ungated, ~131k downloads/month) under a custom minimax-community license; training data and pipeline remain closed.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(on HF; ungated; safetensors/BF16; MoE);data:closed;code:partial(inference only; no training pipeline/data);license:minimax-community(custom, see HF LICENSE)", "confidence": "high", "note": "Weights released publicly on Hugging Face after the 2026-06-04 scoring (then announced-not-yet-released); ungated download under a custom minimax-community license. Data and training pipeline closed, so open_weights not open_source. Matches the MiniMax-M2.5 sibling precedent.", "sources": [ { "url": "https://huggingface.co/api/models/MiniMaxAI/MiniMax-M3", "shows": "Public ungated weights repo (safetensors/MoE), license=minimax-community, created 2026-06-02", "accessed": "2026-06-23" }, { "url": "https://venturebeat.com/technology/minimax-m3-debuts-eclipsing-gpt-5-5-and-gemini-3-1-pro-on-key-benchmark-performance-for-just-5-10-of-the-cost", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": null, "signal_type": "usage_volume", "confidence": "medium", "note": "About 131k downloads in the last month and 1,215 likes on the HF model card (100K-1M band), matching sibling MiniMax-M2.5 (level 3). Weights released post-scoring, so a download signal now exists.", "sources": [ { "url": "https://huggingface.co/api/models/MiniMaxAI/MiniMax-M3", "shows": "HF API reports downloads=131057 (30d) and likes=1215", "accessed": "2026-06-23" } ] }, "capability": { "score": null, "basis": "benchmark:SWE-bench", "value": null, "confidence": "low", "note": "Vendor reports SWE-Bench Pro scores edging out GPT-5.5 and Gemini 3.1 Pro, but benchmarks are vendor-run and not independently verified at scoring time; declining to assign a score to avoid relying on unverified claims.", "sources": [ { "url": "https://www.techtimes.com/articles/317532/20260601/minimax-m3-open-weight-coding-model-frontier-claims-unverified-benchmarks.htm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://venturebeat.com/technology/minimax-m3-debuts-eclipsing-gpt-5-5-and-gemini-3-1-pro-on-key-benchmark-performance-for-just-5-10-of-the-cost", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "MiniMax M3 launched 1 Jun 2026: API-first at launch with MiniMax Sparse Attention, 1M-token context, native multimodal. Open weights have since been released on Hugging Face (ungated, ~131k downloads/month) under a custom minimax-community license; training data and pipeline remain closed." }, { "product": "Nemotron 3", "org": "NVIDIA", "type": "model", "description": "NVIDIA's first-party open-weight instruction-tuned model family for agentic reasoning, post-trained with SFT + RL on top of Nemotron Base. Current SKUs are Nemotron 3 Nano (30B total / 3B active hybrid MoE, March–April 2026) and Nemotron 3 Super (120B / 12B active Mamba-Transformer MoE, March 11, 2026); Nemotron 3 Ultra (~500B / 50B active) is announced for H1 2026. Differentiates on (a) open training data and recipes published alongside the weights, (b) 1M-token context with strong RULER scores (Super 91.8 @ 1M), and (c) being NVIDIA's reference post-trained line for NeMo and NIM, optimized end-to-end against NVIDIA hardware. Adoption signal: 1.38M monthly Hugging Face downloads on Nano BF16 plus 758K on Super BF16, and free hosting on OpenRouter and build.nvidia.com.", "openness": { "score": 4, "class": "open_weights", "components": "weights:open;data:open(~10T-token pretraining + 40M post-training);code:open(recipes+21 RL env configs);license:NVIDIA-Open-Model-License(permissive,commercial,non-OSI)", "confidence": "high", "note": "weights:open;data:open(~10T-token pretraining + 40M post-training);code:open(recipes+21 RL env configs);license:NVIDIA-Open-Model-License(permissive,commercial,non-OSI)", "sources": [ { "url": "https://research.nvidia.com/labs/nemotron/Nemotron-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://llm-stats.com/blog/research/nemotron-3-super-launch", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Distributed free via OpenRouter (nemotron-3-super-120b-a12b:free), NVIDIA build/NIM, and Hugging Face; positioned for agentic/enterprise use. Strong distribution but no verified hard usage count; level 3 = reported traction.", "sources": [ { "url": "https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b:free", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://research.nvidia.com/labs/nemotron/Nemotron-3/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench", "value": null, "confidence": "high", "note": "Nemotron 3 Super: SWE-Bench Verified 60.47, AIME 2025 90.21, GPQA (no tools) 79.23, MMLU-Pro 83.73; ~2.2x inference throughput vs GPT-OSS-120B. Frontier-tier open model.", "sources": [ { "url": "https://llm-stats.com/blog/research/nemotron-3-super-launch", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "Scored on Nemotron 3 Super (120B hybrid MoE, 12B active, 1M context), released 11 Mar 2026 at GTC. Family also includes Nemotron 3 Nano (Dec 2025) and Nemotron 3 Nano Omni (Apr 2026, 30B multimodal). NVIDIA releases weights + pretraining data (10T tokens) + post-training samples (40M) + training recipes + RL env configs." }, { "product": "GPT-5", "org": "OpenAI", "type": "model", "description": "OpenAI's current closed foundation model line: GPT-5 (August 2025), then post-training iterations GPT-5.2, 5.3, 5.4, and finally GPT-5.5 (April 23, 2026), the first fully retrained base since GPT-4.5. GPT-5.5 and GPT-5.5 Pro shipped via API on April 24, 2026; GPT-5.5 Instant became the default ChatGPT model on May 5, 2026, replacing GPT-5.3 Instant. The o-series reasoning brand has been folded into the GPT-5 line. GPT-5.5 is positioned as 'smartest and most agentic' with major gains in agentic coding, computer use, and knowledge work. API-only, no weights distributed; the base pretrained model is never exposed directly. The benchmark closed-source frontier that open-weights models are measured against in 2026.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "sources": [ { "url": "https://openai.com/index/introducing-gpt-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://en.wikipedia.org/wiki/GPT-5", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "GPT-5 was rolled out as the default model for logged-in ChatGPT users (Aug 2025). ChatGPT reached ~900M weekly active users (Feb 2026) and ~1B by May 2026, far exceeding the >10M threshold. Adoption attributed to the surface GPT-5 powers, not a standalone GPT-5 download count.", "sources": [ { "url": "https://almcorp.com/blog/chatgpt-900-million-weekly-active-users/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.demandsage.com/chatgpt-statistics/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": null, "confidence": "high", "note": "OpenAI-reported at launch: SWE-bench Verified 74.9%, MMMU 84.2%, AIME 2025 94.6% (no tools), Aider Polyglot 88%. Frontier-tier coding/reasoning. Vendor-reported figures.", "sources": [ { "url": "https://openai.com/index/introducing-gpt-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "OpenAI GPT-5, released Aug 7, 2025; default ChatGPT model. Successor versions (5.2/5.4/5.5) shipped through April 2026, but the scored entity is GPT-5 itself, which remains in service via API/ChatGPT." }, { "product": "Claude Opus 4.7", "org": "Anthropic", "type": "model", "description": "Anthropic's frontier closed-weights base model (April 2026), the largest tier in the Claude 4.x line. The base is never exposed directly. All access is to the Constitutional-AI + RLHF post-trained variant (Claude Opus 4.7), which tops SWE-bench Verified at 87.6% and holds two of the top four LMArena Elo slots (1492-1502). A leaked 'Claude Mythos Preview' trained on the same family reportedly hits 93.9% SWE-bench Verified. Direct competitor to GPT-5 and Gemini 3.1 Pro on the agentic-coding frontier.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-7", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://aws.amazon.com/blogs/aws/introducing-anthropics-claude-opus-4-7-model-in-amazon-bedrock/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": null, "signal_type": "reported_traction", "confidence": "medium", "note": "Directional market-position estimate (not a measured usage figure). Opus is Anthropic's premium tier top-tier reach but lower volume than the Sonnet workhorse, so 4 rather than 5. Anthropic's launch post lists 28 enterprise testimonials (Stripe, Replit, Vercel, etc.); Opus flagships appear on public LLM marketplace leaderboards (e.g. OpenRouter). Overrides the prior frozen-anchor convention (held null), which systematically understated dominant closed incumbents and could mask the openness gap.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-7", "shows": "28 enterprise testimonials; no quantitative per-model usage figures", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/rankings", "shows": "public per-model token-share leaderboard where Claude flagships rank among the most-used models", "accessed": "2026-06-23" } ] }, "capability": { "score": 5, "basis": "benchmark:GDPval-AA", "value": null, "confidence": "high", "note": "Anthropic-reported: state-of-the-art on GDPval-AA and Finance Agent; CursorBench 70% (vs 58% for Opus 4.6); BigLaw Bench 90.9% at high effort; gains on SWE-bench Verified/Pro/Multilingual. Independent Gemini 3.5 Flash comparisons place Opus 4.7 in the frontier field. Vendor-reported.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-7", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Anthropic Claude Opus 4.7, released Apr 16, 2026 (model id claude-opus-4-7). Superseded by Opus 4.8 on May 28, 2026, but 4.7 remains available across API/Bedrock/Vertex/Foundry as of June 2026." }, { "product": "Claude Fable 5", "org": "Anthropic", "type": "model", "description": "Anthropic's first Mythos-class model for general availability (June 2026), sitting above Opus in capability. Same underlying weights as Claude Mythos 5 but wrapped with safety classifiers that route sensitive cybersecurity, biology/chemistry, and distillation queries to Claude Opus 4.8 (~5% of sessions). Built for long-horizon agentic coding, knowledge work, vision, and scientific research with a 1M-token context window and up to 128K output. Tops Cognition FrontierCode and Hebbia Finance among frontier models; state-of-the-art on CursorBench and FrontierBench per early partner testing. $10/$50 per Mtok via API, Bedrock, Vertex, and Foundry.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "sources": [ { "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "shows": "Mythos-class flagship, proprietary, claude-fable-5 via API and cloud marketplaces only", "accessed": "2026-06-11" }, { "url": "https://platform.claude.com/docs/en/about-claude/models/introducing-claude-fable-5-and-claude-mythos-5", "shows": "API model id claude-fable-5, no weights release, Covered Model retention policy", "accessed": "2026-06-11" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": null, "signal_type": "reported_traction", "confidence": "low", "note": "Directional market-position estimate (not a measured usage figure). Anthropic's most capable GA model, with day-one enterprise testimonials (Stripe, GitHub, Cursor, Cognition) and a staged subscription rollout. Capped at 4 rather than 5 because it launched Jun 9 2026 (~2 weeks before scoring), so sustained reach is unproven. Overrides the prior frozen-anchor convention (held null) for dominant closed incumbents.", "sources": [ { "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "shows": "launch testimonials and staged availability, no quantitative usage figures", "accessed": "2026-06-11" }, { "url": "https://openrouter.ai/rankings", "shows": "public per-model token-share leaderboard where Claude flagships rank among the most-used models", "accessed": "2026-06-23" } ] }, "capability": { "score": 5, "basis": "benchmark:FrontierCode", "value": null, "confidence": "high", "note": "Anthropic-reported: highest among frontier models on Cognition FrontierCode (even at medium effort); top on Hebbia Finance Benchmark; SOTA on CursorBench and FrontierBench per partner testing; leads Opus 4.8 on long-horizon coding, knowledge work, vision, and memory tasks. Vendor-reported; underlying Mythos weights with classifier fallback on ~5% of sessions.", "sources": [ { "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "shows": "FrontierCode highest among frontier models; SOTA across software engineering, knowledge work, vision", "accessed": "2026-06-11" }, { "url": "https://www.anthropic.com/claude/fable", "shows": "most capable generally available Claude model; long-horizon agentic flagship positioning", "accessed": "2026-06-11" } ] }, "maturity": 4.7, "mature": false, "version_note": "Anthropic Claude Fable 5 (model id claude-fable-5), released Jun 9 2026. Mythos-class tier above Opus; generally available counterpart to restricted Claude Mythos 5 (Project Glasswing). 30-day mandatory data retention (Covered Model). Verified via Anthropic Fable 5 launch post and API docs June 2026." }, { "product": "Gemini 3.5 Flash", "org": "Google", "type": "model", "description": "Released at Google I/O 2026 (May 19, 2026) as the new default in the Gemini app. Marketed as 'Pro-grade reasoning at Flash-level speed': #9 on the May 2026 LMArena text leaderboard (Elo 1480) and #2 on LiveCodeBench at 90.8%. Configurable thinking budgets let users trade reasoning depth for latency. Powers Antigravity / Gemini CLI and most Google-side coding-assistant deployments where cost matters.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://deepmind.google/models/gemini/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "Gemini 3.5 Flash distributed via the Gemini app and AI Mode in Google Search 'to billions globally.' The Gemini app surpassed 750M monthly active users (TechCrunch, Feb 2026, Q4 2025 figure). Well above >10M. Attributed to the Gemini app surface 3.5 Flash powers, not a standalone count.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:Terminal-Bench 2.1", "value": null, "confidence": "high", "note": "Google-reported for 3.5 Flash: Terminal-Bench 2.1 76.2%, MCP Atlas 83.6%, CharXiv Reasoning 84.2%, GDPval-AA 1656 Elo; reported to lead Opus 4.7 and GPT-5.5 on MCP Atlas, Toolathlon, Finance Agent v2, CharXiv, MMMU-Pro. Top-right of Artificial Analysis Intelligence Index. Vendor-reported.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Google Gemini 3.5 family; 3.5 Flash launched at I/O 2026 on May 19, 2026 as the first 3.5 model. Gemini 3.5 Pro announced as in internal use, expected ~June 2026 (not yet GA at scoring time). Scored facts are for the released 3.5 Flash; Pro availability unconfirmed." }, { "product": "Grok 4.20", "org": "xAI", "type": "model", "description": "xAI Grok 4.20; beta Feb 17, 2026, API GA Mar 10, 2026 (Grok 4.20 + Multi-agent). Revision '4.20 0309 v2' dated Apr 7, 2026. Multi-agent architecture (4 parallel agents); 2M-token context.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "sources": [ { "url": "https://artificialanalysis.ai/models/grok-4-20", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.x.ai/developers/release-notes", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "high", "note": "Grok 4.20 is selectable by free X users (with limits) and via SuperGrok (~$30/mo) / X Premium+. No clean standalone Grok-4.20 user/usage count is published; available on aggregators (OpenRouter, MindStudio). Conservative tier from API rate-limited distribution rather than a confirmed user figure; treat as directional.", "sources": [ { "url": "https://grokipedia.com/page/Grok_420", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/x-ai/grok-4.20", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:Artificial Analysis Intelligence Index", "value": null, "confidence": "high", "note": "Artificial Analysis Intelligence Index v4.0 score of 49, ranked #24 of 150 (median 36); index includes GPQA Diamond and SciCode (individual subscores not disclosed). Provisional LMSYS/LMArena Elo ~1505-1535 (vs 1483 for Grok 4.1). Strong but a notch below the GPT-5/Gemini 3.5/Opus 4.7 leaders on the AA index.", "sources": [ { "url": "https://artificialanalysis.ai/models/grok-4-20", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.nextbigfuture.com/2026/02/xai-launches-grok-4-20-and-it-has-4-ai-agents-collaborating.html", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "xAI Grok 4.20; beta Feb 17, 2026, API GA Mar 10, 2026 (Grok 4.20 + Multi-agent). Revision '4.20 0309 v2' dated Apr 7, 2026. Multi-agent architecture (4 parallel agents); 2M-token context." }, { "product": "Apertus", "org": "Swiss AI Initiative", "type": "model", "description": "Switzerland's national open LLM and one of the most transparent large models released to date, built by the Swiss AI Initiative (EPFL, ETH Zurich, and the CSCS supercomputing centre) and shipped 2 Sep 2025 in 8B and 70B variants under Apache 2.0. 'Apertus' is Latin for 'open': the release covers not just weights but the full training data (via reconstruction scripts), the Megatron-LM training recipe, intermediate checkpoints, a technical report, and EU AI Act / Swiss data-protection compliance documentation. Trained on 15T tokens spanning 1,811 languages (~40% non-English, including Swiss German and Romansh) on 4,096 GH200 GPUs. Capability is mid-tier, roughly Llama 3.1-70B class on pretraining benchmarks and exceptionally strong multilingually, though below the 2026 frontier on MMLU-style reasoning, but it sets the bar for fully reproducible, legally compliant open models.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0, BF16 safetensors);data:open(full pretraining-data reconstruction scripts, github.com/swiss-ai/pretrain-data);code:open(Megatron-LM training recipe + reconstruction scripts);checkpoints:open(intermediate checkpoints on HF branches);license:Apache-2.0(OSI)", "confidence": "high", "note": "Among the most transparent large-model releases to date: open weights, open data (reconstruction scripts rather than a raw dump, to respect opt-out/robots.txt and strip personal data), open training code/recipe, intermediate checkpoints, a technical report (arXiv 2509.14233), and EU AI Act + Swiss data-protection/copyright compliance docs. All Apache-2.0. Sits in the OLMo/Pythia full-openness tier.", "sources": [ { "url": "https://huggingface.co/swiss-ai/Apertus-70B-2509", "shows": "Apache-2.0; BF16 safetensors weights; training checkpoints on branches; data reconstruction scripts; EU AI Act docs; 15T tokens; 4,096 GH200 GPUs; Megatron-LM", "accessed": "2026-06-08" }, { "url": "https://ethz.ch/en/news-and-events/eth-news/news/2025/09/press-release-apertus-a-fully-open-transparent-multilingual-language-model.html", "shows": "ETH Zurich press release: fully open weights + data + training recipe; Swiss data-protection/copyright + EU AI Act compliance; released 2 Sep 2025 by EPFL/ETH/CSCS", "accessed": "2026-06-08" }, { "url": "https://github.com/swiss-ai/pretrain-data", "shows": "open pretraining-data reconstruction scripts", "accessed": "2026-06-08" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "Newer release (Sep 2025) with strong institutional/sovereign-AI interest but modest raw download volume; swiss-ai/Apertus-70B-2509 shows ~1.5K downloads/month and 149 likes on HF (June 2026); cumulative across the 8B/70B base+Instruct SKUs lands in the 10K-100K band. Distributed via Swisscom, Hugging Face, and the Public AI network. Comparable adoption tier to Falcon 3 / Yi-1.5.", "sources": [ { "url": "https://huggingface.co/swiss-ai/Apertus-70B-2509", "shows": "~1,462 downloads/month, 149 likes (June 2026)", "accessed": "2026-06-08" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU / pretraining suite", "value": "Apertus-70B pretraining-eval avg 67.5% (ARC/HellaSwag/WinoGrande/XNLI/XCOPA/PIQA), ~on par with Llama 3.1-70B (67.3%); Apertus-70B-Instruct MMLU 69.6%, trailing Llama 3.3-70B (87.5%) and Qwen2.5-72B (86.6%) by ~17-20 pts.", "confidence": "medium", "note": "Mid-tier capability: competitive with the prior open generation (Llama 3.1-70B class) on pretraining benchmarks and unusually strong multilingually, but below the 2026 frontier on MMLU-style reasoning/knowledge. The value proposition is full openness + multilingual breadth + legal compliance, not raw SOTA, same profile as OLMo 2.", "sources": [ { "url": "https://huggingface.co/swiss-ai/Apertus-70B-2509", "shows": "pretraining benchmark table: Apertus-70B avg 67.5% vs Llama 3.1-70B 67.3%", "accessed": "2026-06-08" }, { "url": "https://arxiv.org/pdf/2509.14233", "shows": "Apertus technical report, benchmark results and methodology", "accessed": "2026-06-08" } ] }, "maturity": 2.7, "mature": false, "version_note": "Apertus family (8B + 70B, base + Instruct), released 2 Sep 2025 by the Swiss AI Initiative (EPFL, ETH Zurich, CSCS). 15T training tokens across 1,811 natively supported languages (~40% non-English, incl. Swiss German and Romansh), 65,536-token context, trained on 4,096 GH200 GPUs (Alps/CSCS) with Megatron-LM. Verified live on Hugging Face June 2026 (swiss-ai/Apertus-70B-2509)." }, { "product": "SmolLM2", "org": "Hugging Face", "type": "model", "description": "Hugging Face's fully-open small-model family (135M, 360M, 1.7B; base + Instruct). The 1.7B was trained on 11T tokens (FineWeb-Edu, DCLM, The Stack, plus math/code), with the SmolLM-Corpus and training recipes published - a fully-open-pipeline model alongside OLMo 2 and Pythia, light enough to run on-device.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(SmolLM-Corpus,odc-by);code:open(github.com/huggingface/smollm);license:Apache-2.0(OSI)", "confidence": "high", "note": "Fully-open pipeline: open weights, open training corpus, and public training code under OSI licenses.", "sources": [ { "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B", "shows": "Apache-2.0, 11T-token training, sizes", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "usage_volume", "confidence": "high", "note": "~184k monthly downloads on SmolLM2-1.7B; widely used as an on-device/research small model.", "sources": [ { "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B", "shows": "~184,032 downloads last month", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "benchmark:HumanEval/MMLU/GSM8K", "value": "best-in-class for its parameter band (beats Qwen2.5-1.5B, Llama-3.2-1B on several evals)", "confidence": "medium", "note": "Strong for its size; small-model tier overall - the value is full openness + efficiency.", "sources": [ { "url": "https://github.com/huggingface/smollm", "shows": "SmolLM2 paper, benchmarks, recipes", "accessed": "2026-06-17" } ] }, "maturity": 3.0, "mature": false, "version_note": "Apache-2.0 weights; SmolLM-Corpus training data (odc-by) and training code public. ~184k monthly downloads on SmolLM2-1.7B." }, { "product": "Gemma 4", "org": "Google", "type": "model", "description": "Google's latest open-model family (launched March 2026, built from the same research as Gemini 3): E2B/E4B edge variants, a 26B-A4B MoE, and a 31B dense model, with context windows up to 256K, native vision + audio, and 140+ languages. Notably released under Apache 2.0 - a shift from earlier Gemma generations' custom license.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0);data:closed;code:closed;license:Apache-2.0(OSI,but no open data/code)", "confidence": "medium", "note": "OSI-licensed weights, but no open training data or code -> open-weights, not a fully-open pipeline. (License-weighted rubrics could argue toward 4.)", "sources": [ { "url": "https://ai.google.dev/gemma/docs/releases", "shows": "Gemma 4 sizes, release dates, Apache-2.0 license", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~8.39M monthly downloads on gemma-4-31b-it.", "sources": [ { "url": "https://huggingface.co/google/gemma-4-31b-it", "shows": "Apache-2.0, ~8.39M downloads last month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena", "value": "high intelligence-per-parameter; multimodal (vision+audio); Google claims top-tier Arena rankings vs much larger models", "confidence": "medium", "note": "Frontier-class for an open model; benchmark ranking is vendor-reported.", "sources": [ { "url": "https://cloud.google.com/blog/products/ai-machine-learning/gemma-4-available-on-google-cloud", "shows": "Gemma 4 capabilities and rankings", "accessed": "2026-06-17" } ] }, "maturity": 4.7, "mature": false, "version_note": "Apache-2.0 (OSI) - unlike CodeGemma/Gemma 3, which use the Gemma Terms of Use. Training data and recipe not released. ~8.4M monthly downloads on gemma-4-31b-it. Benchmark claims (LMArena) are vendor-reported." }, { "product": "RWKV", "org": "RWKV Project", "type": "model", "description": "Open-weights LLM family with a novel architecture that is 100% RNN yet trains in parallel like a GPT transformer - combining linear-time inference, constant memory (no KV-cache), and effectively unbounded context. The current generation, RWKV-7 'Goose', introduces 'Dynamic State Evolution' for expressivity beyond the limits of attention.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(3.1T-token multilingual corpus);code:open(github.com/BlinkDL/RWKV-LM);license:Apache-2.0(OSI)", "confidence": "high", "note": "Fully-open pipeline: open weights, open training data, and open training code under Apache-2.0.", "sources": [ { "url": "https://arxiv.org/abs/2503.14456", "shows": "RWKV-7 Goose: Apache-2.0 weights + data + code, 3.1T open corpus", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~14.6k stars on RWKV-LM; HF download volume low/noisy - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/BlinkDL/RWKV-LM", "shows": "~14.6k stars, RWKV-7 Goose", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "benchmark:RWKV-7 Goose (arXiv:2503.14456)", "value": "RWKV-7 'Goose' 2.9B: claimed 3B SoTA multilingual and matching 3B SoTA English on far fewer training tokens; MMLU rose to ~43% across generations", "confidence": "medium", "note": "Strong intelligence-per-token for its size via a linear-time RNN architecture; not at frontier absolute capability.", "sources": [ { "url": "https://arxiv.org/abs/2503.14456", "shows": "RWKV-7 3B SoTA-multilingual claims, Pareto vs transformers", "accessed": "2026-06-18" } ] }, "maturity": 3.7, "mature": false, "version_note": "Apache-2.0 with open weights, an open 3.1T-token multilingual corpus, and open training code - a fully-open pipeline (rare for a model). The RWKV Project is an LF AI & Data (Generative AI Commons) non-profit led by Bo Peng (BlinkDL). ~14.6k stars on RWKV-LM." }, { "product": "Qwen3.7-Max", "org": "Alibaba", "type": "model", "description": "Alibaba's flagship proprietary model in the Qwen 'Max' line (announced ~mid-2026), positioned as an agent foundation for coding, office-workflow automation, and autonomous multi-step execution. Offers OpenAI- and Anthropic-compatible APIs and integrates with Claude Code, OpenClaw, and Qwen Code. Sizes/architecture not disclosed; distinct from Qwen's open-weights line.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:proprietary(Alibaba Model Studio API);API-only(distinct from Qwen open-weights line)", "confidence": "high", "note": "Alibaba's own announcement calls it a proprietary model; no weights distributed.", "sources": [ { "url": "https://www.alibabacloud.com/blog/qwen3-7-the-agent-frontier_603154", "shows": "official: 'proprietary', Model Studio API-only", "accessed": "2026-06-18" } ], "bucket": "closed" }, "adoption": { "level": 4, "signal_type": "reported_traction", "confidence": "low", "note": "Commercial API live on Model Studio, cross-listed on OpenRouter/Together; no download numbers (not on HF).", "sources": [ { "url": "https://huggingface.co/Qwen", "shows": "official Qwen HF org confirms no Max open-weights model", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena/GPQA-Diamond/SWE-Pro/Terminal-Bench", "value": "GPQA Diamond 92.4, SWE-Pro 60.6, Terminal-Bench 69.7; agent-focused; LMArena frontier (rank mirror-reported)", "confidence": "medium", "note": "Frontier agent model; official benchmarks, Arena rank from third-party mirrors.", "sources": [ { "url": "https://www.alibabacloud.com/blog/qwen3-7-the-agent-frontier_603154", "shows": "official GPQA/SWE-Pro/Terminal-Bench benchmarks", "accessed": "2026-06-18" } ] }, "maturity": 4.7, "mature": false, "version_note": "Proprietary/API-only (Alibaba Cloud Model Studio / DashScope; cross-listed on OpenRouter/Together). The Max line is closed, unlike Qwen's open-weights releases. LMArena frontier (~#17 per mirrors; vendor cites ~1475 Elo Text)." }, { "product": "GLM-5.2", "org": "Zhipu / Z.ai", "type": "model", "description": "Z.ai's flagship open-weights frontier model (mid-June 2026, successor to GLM-5.1), built for long-horizon agentic and coding tasks. A sparse MoE (~753B total params) with a 1M-token context, introducing 'IndexShare' sparse attention (~2.9x FLOP cut at 1M context) and improved speculative decoding. Positioned as a leading open-weights coding model.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT);data:closed;code:open(inference);license:MIT(OSI)", "confidence": "high", "note": "Permissive MIT open weights + inference code, but no open training data -> open_weights.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-5.2/blob/main/LICENSE", "shows": "MIT, (c) Zhipu AI 2026", "accessed": "2026-06-18" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "signal_type": "usage_volume", "confidence": "low", "note": "~666 HF downloads last month (model was days old at audit; volatile/low).", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-5.2", "shows": "666 downloads last month, 753B MoE, 1M context", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena/FrontierSWE/Terminal-Bench", "value": "FrontierSWE 74.4, SWE-bench Pro 62.1, Terminal-Bench 2.1 ~82, HLE 40.5, AIME 2026 99.2; strongest-open-weights-coder positioning; LMArena frontier", "confidence": "medium", "note": "Frontier open-weights model; official benchmarks, Arena rank mirror-reported.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-5.2", "shows": "official FrontierSWE/Terminal-Bench/HLE benchmarks", "accessed": "2026-06-18" } ] }, "maturity": 4.1, "mature": false, "version_note": "MIT-licensed open weights (HF model card + LICENSE, (c) Zhipu AI 2026). ~753B-param MoE, 1M context. Model only days old at audit (~666 HF downloads/month, volatile)." }, { "product": "ERNIE 5.1", "org": "Baidu", "type": "model", "description": "Baidu's flagship fifth-generation LLM (released May 2026): a Mixture-of-Experts model with elastic depth/width/sparsity, focused on text generation, reasoning, agentic tool-use, and creative writing. Baidu emphasizes cost efficiency - total params ~1/3 and active params ~1/2 of ERNIE 5.0, trained at ~6% of comparable models' pre-training cost. 128K context.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:proprietary(Baidu Qianfan API);API-only(ERNIE 4.5 was open, 5.x flagship closed)", "confidence": "high", "note": "No weights distributed for the 5.x flagship; closed API model.", "sources": [ { "url": "https://huggingface.co/baidu", "shows": "Baidu HF org lists only ERNIE 4.5 + Image open, no 5.x weights", "accessed": "2026-06-18" } ], "bucket": "closed" }, "adoption": { "level": 4, "signal_type": "reported_traction", "confidence": "low", "note": "Baidu's ERNIE/Wenxiaoyan assistant app reported ~200M MAUs around the 5.0 launch (app-level, not 5.1-specific); not on HF.", "sources": [ { "url": "https://ernie.baidu.com/blog/posts/ernie-5.1-0508-release/", "shows": "ERNIE 5.1 release; API access", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena/AIME", "value": "AIME 2026 (with tools) 99.6, strong GPQA/MMLU-Pro; MoE, 128K context; LMArena frontier (Baidu claims #13 Text)", "confidence": "medium", "note": "Frontier proprietary model; Arena rank disputed between mirrors (#28) and Baidu's own claim (#13 Text).", "sources": [ { "url": "https://ernie.baidu.com/blog/posts/ernie-5.1-0508-release/", "shows": "official MoE design, benchmarks, training-cost efficiency", "accessed": "2026-06-18" } ] }, "maturity": 4.7, "mature": false, "version_note": "Proprietary/API-only (Baidu Qianfan API, ernie.baidu.com, AI Studio). Note ERNIE 4.5 was open sourced (Apache-2.0, June 2025) but the 5.x flagship line is closed. LMArena frontier (~#28 per mirrors; Baidu claims 5.1-Preview #13 Text / #1 Chinese)." }, { "product": "MiMo-V2.5-Pro", "org": "Xiaomi", "type": "model", "description": "Xiaomi's open-weights MoE model (released April 2026): ~1.02T total / 42B active params (70 layers, 8 experts/token), hybrid attention interleaving sliding-window and global attention (6:1), 3-layer Multi-Token Prediction, and a 1M-token context. Built for demanding agentic, complex software-engineering, and long-horizon tasks.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT per HF card);data:closed;code:open;license:MIT(HF card; older GitHub repo shows Apache-2.0)", "confidence": "medium", "note": "Open weights under a permissive license (MIT per the model card); no open training data -> open_weights.", "sources": [ { "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro", "shows": "MIT, 1.02T/42B MoE, 1M context, benchmarks", "accessed": "2026-06-18" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~66,027 HF downloads last month, 651 likes, plus third-party quantizations (unsloth GGUF, MLX, vLLM).", "sources": [ { "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro", "shows": "66,027 downloads last month, 651 likes", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "benchmark:LMArena/MMLU-Pro", "value": "MMLU 89.4, MMLU-Pro 68.5, GSM8K 99.6, HumanEval+ 75.6; efficiency-focused; lower end of the LMArena frontier band (~#29 per mirrors)", "confidence": "medium", "note": "Frontier open-weights model, efficiency-focused; official benchmarks, Arena rank mirror-reported.", "sources": [ { "url": "https://mimo.xiaomi.com/mimo-v2-5-pro", "shows": "official architecture + benchmarks, Apr 2026 release", "accessed": "2026-06-18" } ] }, "maturity": 4.0, "mature": false, "version_note": "Open weights, MIT per the HF model card (an older MiMo GitHub repo lists Apache-2.0; the model card is authoritative for this release). ~1.02T/42B MoE, 1M context. ~66k HF downloads/month plus third-party quantizations." }, { "product": "ALIA-40b", "org": "Barcelona Supercomputing Center", "type": "model", "description": "Spain's national open LLM: a 40.4B-parameter transformer decoder-only model pretrained from scratch by the Barcelona Supercomputing Center (BSC-LT) on the MareNostrum 5 supercomputer, as part of the ALIA project under Spain's National Language Technology Plan. Trained on 9.37 trillion tokens across 35 European languages plus 92 programming languages, with a focus on Spanish and the co-official languages (Catalan, Valencian, Basque, Galician). Released under Apache 2.0 with downloadable weights and fully published training code, though the pretraining corpus is documented rather than released. Distributed through the broader ALIA Kit (models, corpora, integration tools, docs). Capability is mid-tier and below the 2026 frontier; the value proposition is permissive licensing, multilingual breadth, and digital sovereignty for the ~600M Spanish speakers and Iberian-language communities.", "openness": { "score": 4, "class": "open_weights", "components": "weights:open(Apache-2.0, safetensors on HF BSC-LT/ALIA-40b);data:documented-not-released(60+ named sources described in the model card, but the pretraining corpus \"will not be released or distributed to third parties\");code:open(all training scripts + configuration files at github.com/langtech-bsc/alia);checkpoints:not-released;license:Apache-2.0(OSI)", "confidence": "high", "note": "Strong open-weights release that stops one notch short of full openness: permissive Apache-2.0 weights plus the complete training code/recipe published on GitHub, but the pretraining corpus is only documented (60+ named sources) rather than released or reconstructable. That missing open-data component is the bright line that keeps it at open_weights (4) rather than the OLMo/Pythia/Apertus open_source (5) tier, where Apertus reaches 5 by shipping data-reconstruction scripts. No intermediate checkpoints.", "sources": [ { "url": "https://huggingface.co/BSC-LT/ALIA-40b", "shows": "Apache-2.0 license; downloadable safetensors weights; 40.4B params; 9.37T training tokens; corpus documented across 60+ sources but \"will not be released or distributed\"; links training code at github.com/langtech-bsc/alia", "accessed": "2026-06-26" }, { "url": "https://github.com/langtech-bsc/alia", "shows": "open training scripts and configuration files for the ALIA models", "accessed": "2026-06-26" } ], "bucket": "open-ish" }, "adoption": { "level": 1, "reach": "1K-10K", "signal_type": "usage_volume", "confidence": "medium", "note": "Public-funded sovereign model (Spain's national LLM, BSC / MareNostrum 5) with strong institutional and policy interest but modest raw volume. HF BSC-LT/ALIA-40b shows ~59 downloads/month and 88 likes (June 2026); the niche multilingual focus (Spanish + co-official + 35 European languages) keeps adoption below Apertus's tier. Cumulative across the wider ALIA Kit (instruct, GGUF, Salamandra) would be higher, but the base model alone sits in the low band.", "sources": [ { "url": "https://huggingface.co/BSC-LT/ALIA-40b", "shows": "~59 downloads/month, 88 likes (June 2026)", "accessed": "2026-06-26" } ] }, "capability": { "score": 2, "basis": "scale + multilingual pretraining (full benchmarks pending technical report)", "value": "40.4B params, decoder-only, 9.37T pretraining tokens, 35 European languages + 92 programming languages; mid-tier and below the 2026 frontier. Detailed benchmarks are deferred to a forthcoming technical report; the model card cites the Salamandra technical report (arXiv:2502.08489) for the shared methodology.", "confidence": "low", "note": "A serious 40B multilingual base model, but smaller than Apertus-70B (capability 3) and below the 2026 frontier on reasoning/knowledge. Positioned for Iberian and European-language coverage and digital sovereignty rather than SOTA capability. Score kept conservative pending published benchmarks.", "sources": [ { "url": "https://huggingface.co/BSC-LT/ALIA-40b", "shows": "40.4B params, 48 layers, 9.37T tokens, 35 European languages + 92 programming languages", "accessed": "2026-06-26" }, { "url": "https://arxiv.org/abs/2502.08489", "shows": "Salamandra technical report (BSC-LT) describing the shared training methodology and benchmarks", "accessed": "2026-06-26" } ] }, "maturity": 1.7, "mature": false, "version_note": "ALIA-40b base model from BSC-LT (Barcelona Supercomputing Center, Language Technologies Lab), Apache 2.0. 40.4B params, 48 layers, 32,768 context, 256k vocab; 9.37T pretraining tokens (+6.3B for context extension). Trained on MareNostrum 5 under Spain's National Language Technology Plan with EU NextGenerationEU funds. Docs portal: langtech-bsc.gitbook.io/alia-kit. Instruct variant (ALIA-40b-instruct) and the earlier Salamandra family exist but are not separately scored. Verified live on Hugging Face, June 2026 (59 downloads/month, 88 likes)." } ] }, "finetuned_chat": { "label": "Fine-tuned / chat models", "arc": "Model components", "layer": "model_components", "stage": { "num": 2, "name": "Emerging Alternatives" }, "gaps": [ "maturity", "openness" ], "products": [ { "product": "Qwen3-Coder", "org": "Alibaba Cloud", "type": "model", "description": "Qwen3-Coder-480B-A35B-Instruct (MoE, 480B total / 35B active, 160 experts, 256K native context to 1M w/ Yarn), non-thinking instruct/agentic-coding SKU; arXiv 2505.09388. HF card lists release 2025-05-14. Verified live on HF June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed(no training corpus released);code:partial(inference/usage + chat template; no training pipeline or post-training/SFT data);license:Apache-2.0(OSI permissive, no use restrictions)", "confidence": "high", "note": "Open weights under genuine OSI Apache-2.0, but post-training data and training code are not released -> open_weights, not full open_source.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct", "shows": "Apache-2.0 license, 480B-A35B MoE instruct model card, downloadable BF16 weights", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "Flagship 480B coder card shows ~28.8K downloads last month on HF (heavy size limits direct pulls); it is the agentic-coding SKU of the dominant Qwen family (Qwen ~942M cumulative HF downloads per ATOM Apr 2026) and is widely served via Qwen Code / providers. Aggregate family reach is far higher, but the honest per-SKU download signal places this single 480B variant in the 100K-1M band. Smaller quant/derivative coder variants add to reach.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct", "shows": "~28.8K downloads last month for the 480B-A35B-Instruct SKU", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench/agentic-coding", "value": "Vendor positions agentic coding comparable to Claude Sonnet class; 480B-A35B MoE, repository-scale long context; non-thinking mode", "confidence": "medium", "note": "Strong open-weight coding model (Claude Sonnet-class agentic coding per vendor card), a notch below 2026 frontier coders (DeepSeek-V4-Pro/Kimi K2.6/GLM-5.1). HF card is vendor-reported; no independent SWE-bench number captured this run, so capped at 4 with medium confidence.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct", "shows": "vendor claim of Claude-Sonnet-class agentic coding; MoE architecture and long-context coding focus", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Qwen3-Coder-480B-A35B-Instruct (MoE, 480B total / 35B active, 160 experts, 256K native context to 1M w/ Yarn), non-thinking instruct/agentic-coding SKU; arXiv 2505.09388. HF card lists release 2025-05-14. Verified live on HF June 2026." }, { "product": "Qwen2.5-Coder-Instruct", "org": "Alibaba Cloud", "type": "model", "description": "Scored on Qwen2.5-Coder-32B-Instruct (32.5B dense, 128K context), the flagship instruct SKU of the Qwen2.5-Coder family; tech report arXiv 2409.12186 (Sep 2024). Instruction-tuned (pretrain + post-train on 5.5T tokens). Verified live on HF June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed(5.5T-token mix described, corpus not released);code:partial(inference/chat template; no training pipeline);license:Apache-2.0(OSI permissive)", "confidence": "high", "note": "Genuine OSI Apache-2.0 open weights; training data/code not released -> open_weights.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", "shows": "Apache-2.0 license, 32.5B instruct coder model card, downloadable BF16 weights", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.36M downloads last month for the 32B-Instruct SKU alone on HF; the broader Qwen2.5-Coder family (0.5B-32B base+instruct) multiplies this. Squarely in the 1-10M reach band on measured monthly download volume.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", "shows": "~1,360,411 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Aider/HumanEval", "value": "Aider code-repair 73.7 (comparable to GPT-4o); McEval 65.9, MdEval 75.2 (1st among open at 2024 release); SOTA open coder at launch", "confidence": "high", "note": "Best open source code LLM at its Sep 2024 release (GPT-4o-class on Aider), but a 2024 dense model now mid-tier vs 2026 frontier coders. Aider 73.7 from the primary Qwen blog.", "sources": [ { "url": "https://qwenlm.github.io/blog/qwen2.5-coder-family/", "shows": "Aider 73.7 (GPT-4o-comparable), McEval 65.9, MdEval 75.2; SOTA among open coders at release", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", "shows": "vendor claim of GPT-4o-matching coding for the 32B instruct", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Scored on Qwen2.5-Coder-32B-Instruct (32.5B dense, 128K context), the flagship instruct SKU of the Qwen2.5-Coder family; tech report arXiv 2409.12186 (Sep 2024). Instruction-tuned (pretrain + post-train on 5.5T tokens). Verified live on HF June 2026." }, { "product": "Qwen2.5-14B-Instruct", "org": "Alibaba Cloud", "type": "model", "description": "Qwen2.5-14B-Instruct (14.7B dense, 48 layers, GQA, 128K context, 29+ languages), instruction-tuned general chat SKU; Qwen2.5 release Sep 2024. Verified live on HF June 2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed;code:partial(inference/chat template; no training pipeline or SFT/RLHF data);license:Apache-2.0(OSI permissive)", "confidence": "high", "note": "OSI Apache-2.0 open weights; post-training data and training code closed -> open_weights.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct", "shows": "Apache-2.0 license, 14.7B instruct model card, downloadable BF16 weights", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~2.22M downloads last month for the 14B-Instruct SKU on HF; one of the most-pulled mid-size instruct models in the dominant Qwen2.5 family. Solidly in the 1-10M band on measured monthly downloads.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct", "shows": "~2,216,468 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU/instruction-following", "value": "General-purpose 14B instruct; vendor reports gains in coding, math, instruction-following and long-text vs predecessor; detailed scores in Qwen2.5 blog", "confidence": "medium", "note": "Capable mid-size (14B) 2024 general chat model; strong-for-size but well below 2026 frontier and below larger Qwen2.5/3 SKUs. HF card does not inline MMLU/MMLU-Pro numbers; scored 3 on size/class with medium confidence.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct", "shows": "14B instruct general-chat model card, vendor capability claims; defers benchmarks to Qwen2.5 blog", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Qwen2.5-14B-Instruct (14.7B dense, 48 layers, GQA, 128K context, 29+ languages), instruction-tuned general chat SKU; Qwen2.5 release Sep 2024. Verified live on HF June 2026." }, { "product": "DeepSeek-R1", "org": "DeepSeek", "type": "model", "description": "Previous-generation reasoning-first model (January 2025) using large-scale GRPO RL to develop chain-of-thought before answering. 671B MoE plus 1.5B–70B distilled variants. Superseded by DeepSeek-R2 (April 2026), which inverts the architecture thesis; R2 is a 32B dense MIT model that runs on a single 24GB consumer GPU and reaches 92.7% AIME 2025. R1 retains 4.3M+ monthly downloads and remains the most-used open reasoning model.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF; distillation explicitly permitted);data:closed(RL/SFT training data not released);code:partial(inference; no full RL training pipeline);license:MIT(OSI-style permissive, no use restrictions)", "confidence": "high", "note": "MIT-licensed open weights with distillation rights, but RL/SFT post-training data and full training code are not released -> open_weights.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", "shows": "MIT license (commercial use + distillation permitted), 671B-A37B MoE reasoning model card", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~5.58M downloads last month for the base R1 repo on HF, plus a very large family of R1 distills and the DeepSeek app/API surface. The R1 release was a landmark open-reasoning event; measured monthly downloads place it firmly in 1-10M, with broader app/API/derivative reach pushing toward the top of the band.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", "shows": "~5,577,155 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:AIME/MATH-500/GPQA/SWE-bench", "value": "AIME 2024 79.8, MATH-500 97.3, GPQA Diamond 71.5, MMLU 90.8, SWE-bench Verified 49.2, LiveCodeBench 65.9 (HF card, vs OpenAI o1)", "confidence": "high", "note": "Frontier open reasoning model at its Jan 2025 release (o1-class: matched/beat o1 on AIME, MATH-500, LiveCodeBench). As of June 2026 it is surpassed by DeepSeek-V3.2-Speciale/V4 and other 2026 frontier reasoners, so a 4 rather than a category-5 in mid-2026 terms. Figures are DeepSeek-reported on the HF card.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", "shows": "AIME 79.8 / MATH-500 97.3 / GPQA 71.5 / MMLU 90.8 / SWE-bench Verified 49.2 / LiveCodeBench 65.9", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "DeepSeek-R1 (671B total / 37B active MoE, 128K context), reasoning model RL+SFT post-trained on DeepSeek-V3-Base (cold-start data + RL + SFT pipeline; R1-Zero is pure-RL variant); arXiv 2501.12948, released 2025-01-22. Verified live on HF June 2026." }, { "product": "DeepSeek-Coder-V2-Instruct", "org": "DeepSeek", "type": "model", "description": "Dedicated coding MoE model (seed product) instruction-tuned for code generation, completion, and chat. Available in full (236B) and Lite (16B) variants. The Lite variant remains popular for local deployment with 1.1M+ downloads. Predecessor to the V3/R1 generation that proved open source could compete on coding benchmarks.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:open(MIT, repo code);model-license:DeepSeek-Model-License(non-OSI; Attachment A acceptable-use restrictions - military/illegal/discrimination/etc.; commercial allowed but use-based restrictions must propagate to derivatives)", "confidence": "high", "note": "Mixed tier: repo CODE is MIT, but the WEIGHTS are governed by the non-OSI DeepSeek Model License with use-based restrictions (Attachment A) -> restricted class for the model, not open_weights. Distinct from newer DeepSeek models (R1/V3.2/V4) which ship weights under plain MIT.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct", "shows": "model license = 'DeepSeek Model Agreement', code license = MIT, 236B-A21B MoE instruct coder", "accessed": "2026-06-04" }, { "url": "https://github.com/deepseek-ai/DeepSeek-Coder-V2/blob/main/LICENSE-MODEL", "shows": "DeepSeek Model License with Attachment A use-based restrictions; non-OSI custom license", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "~15.7K downloads last month on HF for this 236B SKU; superseded by DeepSeek-V2.5/V3/R1 coders and 2026 coders. Modest, declining usage -> low band (~10K-100K including quants/derivatives).", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct", "shows": "~15,679 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:code-tasks", "value": "Vendor: comparable to GPT-4 Turbo on code-specific tasks, beats Claude 3 Opus / Gemini 1.5 Pro on coding+math at 2024 release; 338-language support", "confidence": "medium", "note": "GPT-4-Turbo-class open coder at its 2024 release, but a 2024-generation model now mid-tier vs 2026 frontier coders (Qwen3-Coder, DeepSeek-V4, GLM-5.1, Kimi K2.6). HF card defers exact HumanEval/MBPP/SWE-bench numbers to the paper; scored 3 with medium confidence on vendor positioning.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct", "shows": "vendor claim of GPT-4-Turbo-comparable code performance, 338 languages", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "DeepSeek-Coder-V2-Instruct (236B total / 21B active DeepSeekMoE, 128K context), instruction-tuned coder built on a DeepSeek-V2 checkpoint + 6T additional code tokens, 338 languages; arXiv 2401.06066. Verified live on HF June 2026; a dated (2024-gen) coder." }, { "product": "DeepSeek-V4-Pro / V4-Flash", "org": "DeepSeek", "type": "model", "description": "DeepSeek's open-weight April 2026 flagship line, replacing V3.2. Released 2026-04-24 with 1M default context and 384K max output. V4-Pro-Max posts 80.6% on SWE-bench Verified (top-10) and a later snapshot reports 93.5% on LiveCodeBench. Available via the `deepseek-chat` (V4-Flash) and `deepseek-reasoner` aliases from 2026-07-24, with weights on Hugging Face under MIT. Demonstrates that open weights can match closed-API frontier coding performance at the same context window.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF);data:closed;code:partial(inference/serving; no training pipeline or data);license:MIT(OSI-style permissive, no use restrictions)", "confidence": "high", "note": "V4-Flash weights are MIT open (matching the V4-Pro anchor's open_weights class); training data/code closed -> open_weights. Consistent with the frozen V4-Pro anchor's openness ladder.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "shows": "MIT license, 284B-A13B MoE, 1M context, downloadable FP4/FP8 weights", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~3.50M downloads last month for V4-Flash on HF; the cheaper/faster sibling of V4-Pro is attractive for self-hosting, driving strong pull volume. Squarely 1-10M on measured monthly downloads (V4-Pro anchor itself sits at level 4).", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "shows": "~3,503,796 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:MMLU-Pro/LiveCodeBench", "value": "Instruct (Max mode): MMLU-Pro 86.2, LiveCodeBench 91.6, MRCR-1M 78.7; base MMLU 88.7, HumanEval 69.5, GSM8K 90.8 (HF card)", "confidence": "medium", "note": "Near-frontier capability (MMLU-Pro 86.2, LiveCodeBench 91.6) but deliberately the lighter sibling of the V4-Pro anchor (capability 5); placed at 4 to keep the family curve consistent - Flash trails Pro. Figures DeepSeek-reported on the HF card.", "sources": [ { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "shows": "MMLU-Pro 86.2, LiveCodeBench 91.6, MRCR-1M 78.7 (Max mode); base MMLU 88.7", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Compound SKU. DeepSeek-V4-Pro (1.6T total / 49B active MoE) is a FROZEN flagship anchor (base_pretrained shelf) - NOT rescored here; calibrate to it. This record scores the non-anchor sibling DeepSeek-V4-Flash (284B total / 13B active MoE, 1M context, FP4+FP8, hybrid CSA+HCA attention, mHC, Muon optimizer), the lighter instruct/thinking SKU of the V4 family. Verified live on HF June 2026." }, { "product": "CodeLlama Instruct", "org": "Meta", "type": "model", "description": "Code-specialized instruction-tuned Llama 2 family (7B to 70B, seed product) trained on 500B+ tokens of code and fine-tuned for coding chat. Pioneered the open source coding LLM wave in 2023 and catalyzed the entire open code model ecosystem. Still used as a baseline with ~104K combined monthly downloads across sizes.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:partial(inference utils, no training pipeline);license:Llama-2-Community(custom Meta license, non-OSI; AUP + commercial conditions)", "confidence": "high", "note": "Weights downloadable but under Meta's non-OSI custom/Llama-2 Community license; marketed as open -> open_washed.", "sources": [ { "url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", "shows": "live model card; license=Llama 2 / custom Meta commercial license; 13B instruct", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "13B-Instruct SKU shows ~3.3k downloads/last-month on HF (faded post-2024), but the Code Llama Instruct family was historically heavily adopted and seeded many derivative coders; placed at 3 on cumulative family usage rather than current monthly volume. Now declining vs modern coders.", "sources": [ { "url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", "shows": "~3,294 downloads last month for 13B-Instruct; large adapter/finetune ecosystem", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:HumanEval", "value": "Code Llama Instruct 13B ~HumanEval pass@1 ~42%; family 34B/70B-Instruct higher but all 2023-era", "confidence": "medium", "note": "Competitive coding model in 2023; well below 2026 frontier coders (Devstral, Qwen3.6, GLM-5.1 at 53-80%+ SWE-bench). Dated.", "sources": [ { "url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf", "shows": "Code Llama 13B-Instruct coding model card (arXiv:2308.12950)", "accessed": "2026-06-04" } ] }, "maturity": 2.3, "mature": false, "version_note": "Code Llama Instruct (instruct-tuned coding variant; 7B/13B/34B/70B; scored on the 13B-Instruct SKU as representative). Trained Jan-Jul 2023, paper arXiv:2308.12950 (Aug 2023). >12 months old and superseded by Llama 3.x/Codestral-class coders, but foundational open coding-instruct model; verified live on HF June 2026." }, { "product": "Llama-3.1-8B-Instruct", "org": "Meta", "type": "model", "description": "Instruction-tuned 8B Llama 3.1 model for chat and assistant workflows, post-trained with SFT and DPO on Meta's Llama 3.1 base. The 128K context window and Llama Community License made it the default open instruct checkpoint for production chat deployments. Currently the most-downloaded Llama variant on Hugging Face at ~10.8M downloads/month, surpassing every other open chat checkpoint by an order of magnitude.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable);data:closed;code:partial(no training pipeline);license:Llama-3.1-Community(non-OSI; 700M-MAU commercial cap + use policy)", "confidence": "high", "note": "Weights open but under Meta's non-OSI Llama 3.1 Community License (700M MAU cap); commonly marketed as open -> open_washed.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct", "shows": "live card; license=Llama 3.1 Community License w/ 700M MAU cap; 11,077,966 downloads last month", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "~11.1M downloads in the last month on HF alone (plus large finetune/quantization ecosystem) -> clearly >10M reach; a default small instruct workhorse.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct", "shows": "11,077,966 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU/IFEval", "value": "Llama 3.1 8B-Instruct ~MMLU ~69, solid 8B chat; no reasoning/agentic frontier scores", "confidence": "medium", "note": "Strong general-purpose small instruct model for 2024, but well below 2026 frontier chat/agentic models; capability is mid/low on a scale where 5 = frontier.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct", "shows": "8B instruct model card; 128k context; multilingual chat", "accessed": "2026-06-04" } ] }, "maturity": 2.9, "mature": false, "version_note": "Llama 3.1 8B Instruct, released 23 Jul 2024. 8B dense, 128k context, SFT+RLHF post-trained chat model. Verified live on HF June 2026; one of the most-downloaded instruct models on the platform." }, { "product": "Devstral", "org": "Mistral AI", "type": "model", "description": "Mistral's dedicated open-weight agentic coding model family, purpose-built for software engineering tasks including SWE-bench. The Small-2 (24B) instruct variant delivers competitive coding performance in a deployable size with 321K downloads. The larger Devstral-2-123B pushes frontier-class performance under a research license.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed;code:partial(inference/serving; no training or post-training data);license:Apache-2.0(OSI permissive, no use restrictions)", "confidence": "high", "note": "Apache-2.0 open weights, but training/post-training data and pipeline undisclosed -> open_weights, not open_source.", "sources": [ { "url": "https://huggingface.co/mistralai/Devstral-Small-2507", "shows": "live card; license=apache-2.0; 24B agentic coding model; SWE-bench Verified 53.6%", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~37k downloads last month for the Small-2507 SKU on HF; specialist coding-agent model with moderate uptake (OpenHands/coding-agent niche), not a mass-market chat model.", "sources": [ { "url": "https://huggingface.co/mistralai/Devstral-Small-2507", "shows": "~37,129 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:SWE-bench Verified", "value": "Devstral Small 1.1 SWE-bench Verified 53.6% (OpenHands scaffold); #1 open source at its release", "confidence": "high", "note": "Best open coding model in its release window at a small 24B size, but below 2026 frontier coders (Qwen3.6 27B 77.2, GLM-5.1, Kimi K2.6 80%+). Strong-for-size, mid-tier overall.", "sources": [ { "url": "https://huggingface.co/mistralai/Devstral-Small-2507", "shows": "SWE-bench Verified table: Devstral Small 1.1 = 53.6%, top open source", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "Devstral Small 1.1 (mistralai/Devstral-Small-2507), 24B dense agentic coding model, 128k context, built with All Hands AI; updated from Devstral Small 1.0 (2505). Agentic/coding post-trained SKU. Verified live on HF June 2026." }, { "product": "Mistral-7B-Instruct-v0.2", "org": "Mistral AI", "type": "model", "description": "Instruction-tuned 7B chat model from Mistral that follows user prompts in a dialogue format. It was the first widely adopted European open instruct release (Dec 2023) and remains the canonical 7B chat baseline in eval papers and fine-tuning tutorials. Still pulls ~3.2M Hugging Face downloads per month nearly 2.5 years after release.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0, on HF);data:closed;code:closed(no training/post-training pipeline);license:Apache-2.0(OSI permissive)", "confidence": "high", "note": "Apache-2.0 open weights; instruction-tuning data and recipe undisclosed -> open_weights.", "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", "shows": "live card; license=apache-2.0; 7B instruct; 32k context; 2,504,301 downloads last month", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~2.5M downloads last month on HF plus a very large derivative ecosystem (~1,089 finetunes, 1,256 adapters, 349 merges); still a heavily used permissive 7B instruct base years after release.", "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", "shows": "2,504,301 downloads last month; 1,089 finetunes / 1,256 adapters", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MT-Bench/MMLU", "value": "Mistral 7B Instruct v0.2 ~MT-Bench ~7.6 / MMLU ~60 class; 2023-era 7B chat quality", "confidence": "medium", "note": "Excellent for a 7B in 2023-24; clearly below 2026 frontier and below modern small instruct models (Phi-4-mini, Qwen 3.x small). Capability low on a frontier-anchored scale.", "sources": [ { "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", "shows": "7B instruct model card; arXiv:2310.06825", "accessed": "2026-06-04" } ] }, "maturity": 2.6, "mature": false, "version_note": "Mistral 7B Instruct v0.2, 7B dense, 32k context (up from 8k in v0.1); instruct fine-tune of Mistral-7B-v0.2 (base paper arXiv:2310.06825, Oct 2023). >12 months old and superseded by v0.3 and the Ministral/Mistral Small lines, but foundational permissive instruct model; verified live on HF June 2026." }, { "product": "StarCoder2", "org": "BigCode Project", "type": "model", "description": "Scored on StarCoder2-15B-Instruct-v0.1 (the instruct SKU; 16B, self-aligned, released 31 Oct 2024, arXiv:2410.24198). NOTE: base StarCoder2-15B (Feb 2024) is a pretrained model belonging in base_pretrained; only the instruct variant fits finetuned_chat. Both verified live on HF June 2026.", "openness": { "score": 4, "class": "open_source", "components": "weights:open;data:open(self-oss-instruct-sc2-exec-filter-50k + 7 documented intermediate datasets);code:open(starcoder2-self-align pipeline on GitHub);base-data:open(The Stack v2);license:BigCode-OpenRAIL-M(open but RAIL behavioral-use restrictions, not OSI)", "confidence": "high", "note": "Fully reproducible self-alignment pipeline (open weights + open instruct data + open code + open base corpus) -> open_source tier; held to 4 rather than 5 because OpenRAIL-M carries non-OSI behavioral-use restrictions.", "sources": [ { "url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", "shows": "self-aligned; open dataset + open self-align code + OpenRAIL-M license", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/bigcode/starcoder2-15b", "shows": "base StarCoder2 trained on open The Stack v2; OpenRAIL-M", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "medium", "note": "Instruct SKU ~1.1k downloads/last-month; base StarCoder2-15B ~9.4k/last-month. Research-significant (fully-open self-alignment recipe) but low production reach; placed at 1 on the instruct SKU's measured volume.", "sources": [ { "url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", "shows": "~1,120 downloads last month", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/bigcode/starcoder2-15b", "shows": "~9,449 downloads last month (base)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:HumanEval", "value": "StarCoder2-15B-Instruct-v0.1 HumanEval pass@1 72.6% (HumanEval+ 63.4%)", "confidence": "high", "note": "Strong HumanEval for a fully-open 16B in 2024 (best fully-self-aligned code model at release), but HumanEval is saturated and the model is below 2026 frontier coders on harder agentic benchmarks (SWE-bench). Mid-tier.", "sources": [ { "url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1", "shows": "HumanEval pass@1 72.6%, HumanEval+ 63.4%", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "Scored on StarCoder2-15B-Instruct-v0.1 (the instruct SKU; 16B, self-aligned, released 31 Oct 2024, arXiv:2410.24198). NOTE: base StarCoder2-15B (Feb 2024) is a pretrained model belonging in base_pretrained; only the instruct variant fits finetuned_chat. Both verified live on HF June 2026." }, { "product": "OLMo 2 Instruct", "org": "Ai2", "type": "model", "description": "Instruction-tuned chat SKUs of Ai2's OLMo 2 family (7B/13B/32B-Instruct), post-trained with the fully-open Tulu 3 pipeline (SFT -> DPO -> RLVR) on top of the open OLMo 2 base. Distinct from the base OLMo 2 family entry (base_pretrained): separate HF repos, separate post-training, and the chat-relevant capability axis (IFEval, AlpacaEval). Verified live on HF June 2026.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(Dolma/OLMo-Mix pretraining + Tulu 3 post-training mixtures);code:open(OLMo + open-instruct training code, recipes);checkpoints:open(intermediate);license:Apache-2.0(OSI)", "confidence": "high", "note": "Genuinely fully-open instruct line: Apache-2.0 weights + open pretraining and post-training data + open training code/recipe + intermediate checkpoints. Open_source tier 5. Post-trained with the open Tulu 3 SFT->DPO->RLVR pipeline. Cards note limited safety training.", "sources": [ { "url": "https://huggingface.co/allenai/OLMo-2-0325-32B-Instruct", "shows": "License Apache-2.0; Tulu 3 SFT+DPO+RLVR post-training; fully-open data/code/checkpoints", "accessed": "2026-06-25" }, { "url": "https://allenai.org/blog/olmo2", "shows": "\"best fully open language model to date\"; releases weights, data, code, recipes, intermediate checkpoints, and instruction-tuned models", "accessed": "2026-06-25" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "Combined ~58k HF downloads/month across the three named instruct SKUs (7B-Instruct ~45.2k, 13B-Instruct ~6.9k, 32B-Instruct ~5.9k). 32B-Instruct carries the highest likes (147). Solid research/practitioner reach; below the Qwen/Llama instruct workhorses.", "sources": [ { "url": "https://huggingface.co/allenai/OLMo-2-1124-7B-Instruct", "shows": "~45,191 downloads last month", "accessed": "2026-06-25" }, { "url": "https://huggingface.co/allenai/OLMo-2-0325-32B-Instruct", "shows": "~5,918 downloads last month; 147 likes", "accessed": "2026-06-25" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU/IFEval/AlpacaEval", "value": "OLMo-2-32B-Instruct MMLU 77.3, GSM8K 87.6, IFEval 85.6, AlpacaEval2 LC 42.8", "confidence": "high", "note": "32B-Instruct competitive with GPT-4o-mini / Qwen2.5-32B class and outperforms GPT-3.5 Turbo on most benchmarks; mid-tier on a scale where 5 = 2026 frontier chat. The value proposition is full openness, not raw SOTA.", "sources": [ { "url": "https://huggingface.co/allenai/OLMo-2-0325-32B-Instruct", "shows": "MMLU 77.3, GSM8K 87.6, IFEval 85.6; comparison vs GPT-4o-mini and Qwen2.5-32B", "accessed": "2026-06-25" } ] }, "maturity": 3.0, "mature": false, "version_note": "Scored on the OLMo 2 Instruct SKUs (0325 32B-Instruct is the most capable). Apache-2.0; fully-open weights + data + recipe + intermediate checkpoints. Post-training via the Tulu 3 recipe." }, { "product": "OLMoE Instruct", "org": "Ai2", "type": "model", "description": "Ai2's fully-open Mixture-of-Experts instruct model (OLMoE-1B-7B-0924-Instruct: 1.3B active / 6.9B total, SFT + DPO). Released with open pretraining data (OLMoE-mix-0924, built on Dolma + DataComp), open post-training data, open training code, and 244 intermediate checkpoints. Verified live on HF June 2026.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(OLMoE-mix-0924 pretraining + Tulu/UltraFeedback post-training);code:open(OLMoE training code, SFT/DPO/KTO recipes);checkpoints:open(244 intermediate);license:Apache-2.0(OSI)", "confidence": "high", "note": "Fully-open MoE: Apache-2.0 weights + open pretraining data (OLMoE-mix-0924, on Dolma/DataComp) + open post-training data + open code + 244 intermediate checkpoints. Open_source tier 5.", "sources": [ { "url": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct", "shows": "License Apache-2.0; SFT+DPO instruct; open data and code", "accessed": "2026-06-25" }, { "url": "https://allenai.org/blog/olmoe-an-open-small-and-state-of-the-art-mixture-of-experts-model-c258432d0514", "shows": "released with open data, code, evaluations, logs, and intermediate training checkpoints", "accessed": "2026-06-25" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "~37.9k HF downloads/month on the instruct SKU; 96 likes. Solid reach for a fully-open research MoE.", "sources": [ { "url": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct", "shows": "~37,880 downloads last month; 96 likes", "accessed": "2026-06-25" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU/AlpacaEval", "value": "OLMoE-1B-7B-Instruct MMLU 51.9, GSM8k 45.5, HumanEval 54.8, AlpacaEval1 84.0, IFEval 48.1", "confidence": "medium", "note": "State-of-the-art among ~1B-active-parameter models at release (beats Gemma2, Llama2-13B-Chat, OLMo-7B, DeepSeekMoE-16B per the blog), but absolute capability is low-tier vs full-size 2026 chat models given the 1.3B active budget. Scored 2 on absolute capability. Exact per-competitor numbers live in arXiv 2409.02060 (blog charts are images), so the comparative claims are directional -- hence medium confidence.", "sources": [ { "url": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct", "shows": "MMLU 51.9, GSM8k 45.5, HumanEval 54.8, AlpacaEval 84.0, IFEval 48.1", "accessed": "2026-06-25" }, { "url": "https://allenai.org/blog/olmoe-an-open-small-and-state-of-the-art-mixture-of-experts-model-c258432d0514", "shows": "positioned state-of-the-art at the ~1B-active cost class", "accessed": "2026-06-25" } ] }, "maturity": 2.3, "mature": false, "version_note": "Apache-2.0; fully-open MoE (weights + data + code + intermediate checkpoints). State-of-the-art among ~1B-active-parameter models at release." }, { "product": "Tulu 3", "org": "Ai2", "type": "model", "description": "Ai2's open post-training model family (Llama-3.1-Tulu-3 8B / 70B / 405B), the reference fully-open post-training recipe: open SFT + preference data, open training code (open-instruct), and the RLVR (Reinforcement Learning with Verifiable Rewards) method, applied on top of Meta's Llama 3.1 base. Note: the recipe/data/code are Apache-2.0 and fully reproducible, but the released weights inherit Meta's Llama 3.1 Community License (non-OSI), so the artifact is open-recipe over restricted weights. Verified live on HF June 2026.", "openness": { "score": 3, "class": "restricted", "components": "weights:open(downloadable);license:Llama-3.1-Community(non-OSI;700M-MAU cap + use policy);post-training-data:open(Apache-2.0/ODC-BY);code:open(open-instruct Apache-2.0);recipe:open(SFT->DPO->RLVR);evals:open(OLMES)", "confidence": "high", "note": "NOT fully-open: the released weights inherit Meta's Llama 3.1 Community License (non-OSI, 700M-MAU cap), so the class is restricted -- same calibration as other Llama-Community models in the map. Scored 3 rather than the bare-Llama-Instruct 2 because the entire post-training layer (data, code, RLVR recipe, eval framework) is genuinely open and reproducible (Apache-2.0). Data license is ODC-BY/Apache per Ai2 practice; the exact dataset-card license string is medium confidence, but the headline open-recipe-over-restricted-weights call is high confidence.", "sources": [ { "url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B", "shows": "License: Llama 3.1 Community License Agreement (non-OSI); base meta-llama/Llama-3.1-8B", "accessed": "2026-06-25" }, { "url": "https://allenai.org/blog/tulu-3", "shows": "fully-transparent post-training release -- open data + data mixes, training code/recipe, eval framework, and weights", "accessed": "2026-06-25" }, { "url": "https://github.com/allenai/open-instruct", "shows": "Apache-2.0; Tulu 3 SFT/DPO/RLVR post-training framework + decontamination tools", "accessed": "2026-06-25" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "Research/reference traction, not a production endpoint. HF downloads/month ~3.1k (8B), ~493 (70B), ~832 (405B); likes 179/61/112. Aggregate low-thousands monthly. Level 2.", "sources": [ { "url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B", "shows": "~3,109 downloads last month; 179 likes", "accessed": "2026-06-25" }, { "url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3-405B", "shows": "~832 downloads last month; 112 likes", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "benchmark:MMLU/GSM8K/IFEval", "value": "Tulu-3-405B MMLU 87.0, GSM8K 95.5, IFEval 86.0, HumanEval 95.9, MATH 67.3", "confidence": "high", "note": "The 405B SKU trades blows with GPT-4o (2024-11) on math/reasoning (GSM8K 95.5 vs 91.7) and trails slightly on chat (AlpacaEval) and raw MMLU vs Llama-3.1-405B-Instruct. Strong upper-mid capability; scored 4. The smaller 8B is competitive in its size class (MMLU 68.2, GSM8K 87.6, IFEval 82.4).", "sources": [ { "url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3-405B", "shows": "MMLU 87.0, GSM8K 95.5, IFEval 86.0, HumanEval 95.9, MATH 67.3; vs GPT-4o comparison", "accessed": "2026-06-25" }, { "url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B", "shows": "MMLU 68.2, GSM8K 87.6, IFEval 82.4", "accessed": "2026-06-25" } ] }, "maturity": 3.4, "mature": false, "version_note": "Weights under Llama 3.1 Community License (non-OSI); Ai2's post-training data, code, recipe, and eval framework (open-instruct, OLMES) are Apache-2.0. Classed as restricted on the weights license, consistent with how other Llama-Community models are scored." }, { "product": "Zephyr", "org": "Hugging Face", "type": "model", "description": "Hugging Face H4's fully-open chat model line. The flagship zephyr-7b-beta is a DPO fine-tune of Mistral-7B, trained with the open alignment-handbook recipe on the public UltraChat and UltraFeedback datasets and relicensed MIT. A late-2023 model that remains a widely-downloaded, fully-reproducible 7B open baseline. Also includes the zephyr-orpo-141b MoE variant (Apache-2.0). Verified live on HF June 2026.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(MIT;base Mistral-7B Apache-2.0);data:open(UltraChat_200k + UltraFeedback_binarized, both MIT, ungated);code:open(alignment-handbook DPO recipe, Apache-2.0);license:MIT(OSI)", "confidence": "high", "note": "Genuinely fully-open: MIT weights, the two DPO training datasets are public and MIT, and the end-to-end training code (alignment-handbook) is Apache-2.0 and replicates zephyr-7b-beta. Open_source tier 5. Caveat (does not change the class): upstream UltraChat/UltraFeedback were partly generated with OpenAI model outputs, but the released HF artifacts are open and MIT.", "sources": [ { "url": "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta", "shows": "License MIT; DPO fine-tune of Mistral-7B on UltraChat/UltraFeedback", "accessed": "2026-06-25" }, { "url": "https://github.com/huggingface/alignment-handbook", "shows": "Apache-2.0; contains all training code to replicate zephyr-7b-beta", "accessed": "2026-06-25" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~164.9k HF downloads/month on zephyr-7b-beta with 1.85k likes -- a late-2023 model still pulling meaningful volume as a lightweight fully-open 7B baseline. The underlying datasets remain active (ultrachat_200k ~57.7k/mo). The 141B ORPO variant is effectively dormant (~52/mo). Level 4 on the leading SKU's sustained monthly volume.", "sources": [ { "url": "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta", "shows": "~164,917 downloads last month; 1.85k likes", "accessed": "2026-06-25" } ] }, "capability": { "score": 2, "basis": "benchmark:MT-Bench/AlpacaEval", "value": "zephyr-7b-beta MT-Bench 7.34, AlpacaEval 90.6% win rate, Open LLM Leaderboard avg 52.15", "confidence": "high", "note": "Notable late-2023 capability (a 7B beating much larger chat models on MT-Bench at release), but weak by 2026 standards (GSM8K 12.74, DROP 9.66 on the leaderboard). Low-tier on a scale where 5 = current frontier chat. Scored 2.", "sources": [ { "url": "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta", "shows": "MT-Bench 7.34; AlpacaEval 90.60%; Open LLM Leaderboard avg 52.15", "accessed": "2026-06-25" } ] }, "maturity": 2.6, "mature": false, "version_note": "Scored on zephyr-7b-beta (MIT; base Mistral-7B Apache-2.0). Fully-open: weights + open DPO datasets (UltraChat/UltraFeedback, MIT) + open training code (alignment-handbook, Apache-2.0). Late-2023 capability, still ~165k HF downloads/month." }, { "product": "K2 Chat", "org": "LLM360", "type": "model", "description": "LLM360's fully-open 65B instruction-tuned chat model (K2-Chat). LLM360 is the reference \"fully reproducible\" LLM project: it releases data-prep code, training code, intermediate checkpoints, and named instruction-tuning data, all Apache-2.0. A clean fully-open exemplar; modest adoption and last updated Jan 2025. Verified live on HF June 2026.", "openness": { "score": 5, "class": "open_source", "components": "weights:open(Apache-2.0);data:open(named instruction sets + open data-prep code);code:open(k2-train training code);checkpoints:open;license:Apache-2.0(OSI)", "confidence": "high", "note": "LLM360 is the reference fully-reproducible LLM project: open weights + open data-prep code + open training code + open checkpoints, all Apache-2.0. A clean open_source exemplar (tier 5) for the instruct category.", "sources": [ { "url": "https://huggingface.co/LLM360/K2-Chat", "shows": "License Apache-2.0; 65B instruct; open data/code/checkpoints", "accessed": "2026-06-25" }, { "url": "https://github.com/LLM360", "shows": "k2-train (training code) and k2-data-prep (data preparation code), Apache-2.0", "accessed": "2026-06-25" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "LOW-CONFIDENCE PROMINENCE CALL (flagged for review): ~651 HF downloads/month, 21,298 all-time, 37 likes; last modified Jan 2025. Niche, not a mass-adoption model. Included as a clean fully-open exemplar rather than on popularity. Aggregate all-time volume puts it just into the 10K-100K band -> level 2; monthly volume alone would argue level 1.", "sources": [ { "url": "https://huggingface.co/LLM360/K2-Chat", "shows": "~651 downloads last month; 21,298 all-time; 37 likes; last modified 2025-01-12", "accessed": "2026-06-25" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU/GSM8K", "value": "K2-Chat MMLU 69.14, GSM8K 77.10, BBH 70.37, HumanEval 60.82, TruthfulQA 57.32", "confidence": "medium", "note": "Reported to outperform Llama 2 70B Chat using ~35% less compute. Solid mid-tier for a 2024 fully-open 65B; below 2026 frontier chat models. Numbers are card-reported (not independently re-run) -> medium confidence.", "sources": [ { "url": "https://huggingface.co/LLM360/K2-Chat", "shows": "MMLU 69.14, GSM8K 77.10, BBH 70.37, HumanEval 60.82; outperforms Llama 2 70B Chat with ~35% less compute", "accessed": "2026-06-25" } ] }, "maturity": 2.7, "mature": false, "version_note": "Apache-2.0; open weights + open data-prep + open training code + open checkpoints. 65B. Reported to outperform Llama 2 70B Chat using ~35% less compute. Niche adoption (~651 HF downloads/month)." }, { "product": "Phi-4-mini-instruct", "org": "Microsoft", "type": "model", "description": "Compact instruction-tuned small language model (3.8B) that punches well above its weight on coding benchmarks. Part of Microsoft's Phi-4 family, which expanded across 2025–early 2026 with Phi-4-reasoning, Phi-4-multimodal, and Phi-4-Reasoning-Vision-15B (March 2026); no Phi-5 has shipped as of May 2026. 1.6M monthly downloads make Phi-4-mini one of the most popular small models for local coding.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF);data:closed(synthetic-heavy training data undisclosed);code:closed(no training pipeline);license:MIT(OSI permissive)", "confidence": "high", "note": "MIT-licensed open weights; training/post-training data and code not released -> open_weights.", "sources": [ { "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "shows": "live card; license=MIT; 3.8B instruct; 1,537,561 downloads last month", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.54M downloads last month on HF; widely distributed via Azure AI Foundry / Ollama for on-device + local inference. Solidly in the 1-10M band.", "sources": [ { "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "shows": "1,537,561 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU-Pro/GPQA", "value": "MMLU 67.3, MMLU-Pro 52.8, GPQA 25.2, GSM8K 88.6, MATH 64.0 (overall 63.5)", "confidence": "high", "note": "Strong-for-size 3.8B with good math (GSM8K 88.6) but low GPQA (25.2) and overall well below 2026 frontier chat models; small-model tier on a frontier-anchored scale.", "sources": [ { "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "shows": "benchmark table: MMLU-Pro 52.8, GPQA 25.2, GSM8K 88.6, overall 63.5", "accessed": "2026-06-04" } ] }, "maturity": 2.6, "mature": false, "version_note": "Phi-4-mini-instruct, 3.8B dense, 128k context, 200k vocab, GQA; released Feb 2025 under MIT. SFT/DPO-aligned small instruct model. Verified live on HF June 2026." }, { "product": "Granite Code Instruct", "org": "IBM", "type": "model", "description": "IBM's enterprise-focused open source code model line, now folded into the unified Granite 4.1 family (3B, 8B, 30B dense, base + instruct, Apache-2.0) released April 29, 2026. Granite 4.1 delivers significant improvements over Granite 4.0 in tool calling, instruction following, coding, and math, while preserving IBM's IP-clean training-data provenance, still the go-to choice for enterprises needing legal clarity on training data. Available on watsonx and Hugging Face.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Apache-2.0,on HF);code:partial(inference + fine-tune sample scripts via Dolomite Engine; no full training pipeline in repo);data:described_not_released(CommitPackFT/MathInstruct/Glaive/HelpSteer/Open-Platypus named in card but processed corpus not redistributable);post-training-data:described(SFT mixture named, not released);paper:open(arXiv 2405.04324);model_card:open;license:Apache-2.0(OSI)", "confidence": "high", "note": "OSI Apache-2.0 + downloadable weights + paper + model card, but training/post-training data only described (not redistributable) and no full training pipeline => open_weights, not open_source. Strong within the open_weights class (genuinely permissive, no use caps).", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-3b-code-instruct-2k", "shows": "Apache-2.0 license, instruction-tuned model card, named training-data sources, HumanEvalSynthesis scores", "accessed": "2026-06-04" }, { "url": "https://github.com/ibm-granite/granite-code-models", "shows": "repo with Apache-2.0, fine-tune sample scripts (Dolomite Engine), but no training data download and no full training pipeline", "accessed": "2026-06-04" }, { "url": "https://research.ibm.com/blog/granite-code-models-open-source", "shows": "IBM describes data sources (GitHub Code Clean, StarCoderData, CodeNet) but does not release the processed datasets", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "medium", "note": "HF shows ~1,925 downloads/month on the granite-3b-code-instruct-2k variant; model is officially deprecated and superseded by Granite 4.x. Even summed across the four Instruct sizes the active monthly pull volume is small. Hobbyist/legacy-research tier.", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-3b-code-instruct-2k", "shows": "~1,925 downloads last month; deprecation notice", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:HumanEval", "value": "granite-3b-code-instruct-2k HumanEvalSynthesis pass@1: Python 51.2, JS 43.9, Java 41.5, C++ 40.2, Go 31.7, Rust 29.3; 8B variant ~SOTA-among-open-code-LLMs at 2024 release on HumanEvalPack", "confidence": "high", "note": "Strong-for-size 2024 code SLMs (matched open code LLMs of their era), but sub-13B and ~51% HumanEval Python is far below 2026 frontier chat/agentic coders (Opus 4.8 88.6 SWE-bench). Low-mid on a frontier-relative scale.", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-3b-code-instruct-2k", "shows": "HumanEvalSynthesis pass@1 per-language scores", "accessed": "2026-06-04" }, { "url": "https://research.ibm.com/blog/granite-code-models-open-source", "shows": "Granite-8B-Code matches SOTA among open source code LLMs on HumanEvalPack at 2024 release", "accessed": "2026-06-04" } ] }, "maturity": 1.7, "mature": false, "version_note": "IBM Granite Code Instruct family (3B/8B/20B/34B, 2K & 128K context), fine-tuned from Granite-Code-Base for code instruction-following; released May 6 2024 (arXiv 2405.04324). Instruct = post-trained (SFT on permissively-licensed code/math/NL instruction data). Verified live on HF June 2026 but marked DEPRECATED by IBM (superseded by Granite 4.x mainline)." }, { "product": "o3 / o4-mini", "org": "OpenAI", "type": "model", "description": "Previous reasoning tier (2025): full-scale o3 plus cost-efficient o4-mini, both using extended chain-of-thought. Powered Codex agent and ChatGPT's coding mode through 2025. Folded into the GPT-5 line in late 2025; OpenAI consolidated the o-series and GPT branches and is no longer maintaining o-series as a separately branded reasoning family. Current reasoning capability lives in GPT-5.x with thinking enabled and in GPT-5.3-Codex.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API+ChatGPT only)", "confidence": "high", "note": "Proprietary OpenAI reasoning models; no weights, data, or code released.", "sources": [ { "url": "https://en.wikipedia.org/wiki/OpenAI_o3", "shows": "o3/o4-mini are proprietary reasoning models served via ChatGPT/API, released Apr 16 2025", "accessed": "2026-06-04" }, { "url": "https://cdn.openai.com/pdf/2221c875-02dc-4789-800b-e7758f3722c1/o3-and-o4-mini-system-card.pdf", "shows": "OpenAI o3 and o4-mini System Card, Apr 16 2025 (primary)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "o3/o4-mini were rolled out as selectable models inside ChatGPT (Plus/Pro and some free) plus the API; ChatGPT reached ~900M WAU (Feb 2026) rising toward ~1B (May 2026). Reach attributed to the ChatGPT surface these models power, not a standalone o3 count. Now superseded as default by GPT-5.x but still used at scale.", "sources": [ { "url": "https://en.wikipedia.org/wiki/OpenAI_o3", "shows": "o3/o4-mini available to ChatGPT users and API", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench/GPQA/Codeforces/AIME", "value": "o3: SWE-bench Verified 71.7%, GPQA Diamond 87.7%, Codeforces Elo 2727; o4-mini: AIME 2025 99.5% pass@1 with Python tool, GPQA ~81%", "confidence": "high", "note": "Frontier-class reasoning at Apr 2025 launch (SOTA on Codeforces/SWE-bench/MMMU at the time). By June 2026 surpassed by GPT-5.x / Opus 4.8 (88.6 SWE-bench) / Gemini 3.5, so frontier-relative score is 4, not 5.", "sources": [ { "url": "https://en.wikipedia.org/wiki/OpenAI_o3", "shows": "o3 SWE-bench Verified 71.7, GPQA 87.7, Codeforces 2727; o4-mini AIME/GPQA figures", "accessed": "2026-06-04" }, { "url": "https://cdn.openai.com/pdf/2221c875-02dc-4789-800b-e7758f3722c1/o3-and-o4-mini-system-card.pdf", "shows": "primary system-card benchmark evaluations", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "OpenAI o3 and o4-mini reasoning models, both released Apr 16 2025 (o3-mini preceded Jan 31 2025). Post-trained reasoning/instruct models, API-only + ChatGPT, no downloadable weights. Verified via OpenAI system card + Wikipedia June 2026; succeeded by GPT-5.x reasoning line but still selectable/available." }, { "product": "GPT-4.1", "org": "OpenAI", "type": "model", "description": "Previous-generation general-purpose instruction-tuned model (April 2025). Strong on HumanEval and real-world coding with 1M token context, but two numerical generations behind by mid-2026; OpenAI's current production line is GPT-5 → 5.2 → 5.4 → 5.5 (April 23, 2026), with GPT-5.5 Pro and 5.5 Instant variants. GPT-4.1 remains available via API and is still used in legacy integrations.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only)", "confidence": "high", "note": "Proprietary OpenAI API model; no weights/data/code released.", "sources": [ { "url": "https://techcrunch.com/2025/04/14/openais-new-gpt-4-1-models-focus-on-coding/", "shows": "GPT-4.1 family is API-only, coding-focused, released Apr 14 2025", "accessed": "2026-06-04" }, { "url": "https://www.infoq.com/news/2025/05/openai-gpt-4-1/", "shows": "GPT-4.1 family with 1M-token context, API release", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "GPT-4.1 was an API-first developer/coding model (initially not in ChatGPT default), widely adopted as a cheaper GPT-4o replacement and integrated across tooling. No primary per-model usage count disclosed; level 4 reflects broad developer-platform distribution as a major OpenAI API model rather than a verified figure. Treat as directional.", "sources": [ { "url": "https://techcrunch.com/2025/04/14/openais-new-gpt-4-1-models-focus-on-coding/", "shows": "GPT-4.1 positioned as primary API coding model replacing GPT-4o", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 54.6% (vs GPT-4o 33.2%); 1M-token context", "confidence": "high", "note": "Solid non-reasoning coding model at Apr 2025 (54.6 SWE-bench, +21pts over GPT-4o), but well below 2026 reasoning frontier (o3 71.7, GPT-5 74.9, Opus 4.8 88.6). Mid-tier, frontier-relative. Headline 54.6% re-sourced to OpenAI primary announcement (was aggregator-only).", "sources": [ { "url": "https://openai.com/index/gpt-4-1/", "shows": "OpenAI primary announcement: GPT-4.1 SWE-bench Verified 54.6% vs GPT-4o 33.2%, 1M-token context (corroborated by InfoQ/TechCrunch)", "accessed": "2026-06-04" }, { "url": "https://www.helicone.ai/blog/gpt-4.1-full-developer-guide", "shows": "GPT-4.1 SWE-bench Verified 54.6% vs GPT-4o 33.2%", "accessed": "2026-06-04" }, { "url": "https://www.infoq.com/news/2025/05/openai-gpt-4-1/", "shows": "GPT-4.1 benchmark improvements over GPT-4o/4.5 and 1M context", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "OpenAI GPT-4.1 (+ mini, nano), released Apr 14 2025 as API-first coding/instruct models with 1M-token context. Post-trained instruct (non-reasoning) model, closed/API-only. Verified via OpenAI announcement coverage + InfoQ/TechCrunch June 2026; GPT-4.5 Preview deprecated July 2025 in its favor; now itself behind the GPT-5 line." }, { "product": "GPT-5 line", "org": "OpenAI", "type": "model", "description": "OpenAI's current general-purpose flagship line spanning GPT-5 (August 2025), GPT-5.2, GPT-5.4, and GPT-5.5 (April 23, 2026), plus the GPT-5.5 Pro (API-only flagship), GPT-5.5 Instant (low-latency consumer tier), and GPT-5.3-Codex (the unified Codex+reasoning coder variant that succeeded o3/o4-mini). GPT-5.5 is marketed as OpenAI's 'smartest, most agentic' model; GPT-5.2 hits 80.0% on SWE-bench Verified and GPT-5.5 reportedly reaches 88.7% on a competing benchmark snapshot. GPT-5.5-high sits at #8 on the May 2026 LMArena text leaderboard (Elo 1481). Note: OpenAI has reportedly stopped reporting SWE-bench Verified scores, preferring SWE-bench Pro, citing test-case quality issues with Verified.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API + ChatGPT only)", "confidence": "high", "note": "Proprietary OpenAI flagship line; no downloadable weights.", "sources": [ { "url": "https://en.wikipedia.org/wiki/GPT-5", "shows": "GPT-5 (Aug 7 2025) is OpenAI's proprietary unified model, free-tier + paid via ChatGPT, API access", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/05/05/openai-releases-gpt-5-5-instant-a-new-default-model-for-chatgpt/", "shows": "GPT-5.5 Instant is the new default ChatGPT model (proprietary)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "The GPT-5 line is the default model powering ChatGPT for all free and paid users since Aug 2025 (GPT-5.5 Instant default as of May 2026). ChatGPT ~900M WAU (Feb 2026) toward ~1B (2026), far above >10M. Attributed to the ChatGPT surface the line powers, not a standalone SKU count.", "sources": [ { "url": "https://en.wikipedia.org/wiki/GPT-5", "shows": "GPT-5 became default ChatGPT model for all free and paid users", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/05/05/openai-releases-gpt-5-5-instant-a-new-default-model-for-chatgpt/", "shows": "GPT-5.5 Instant is the new default model for ChatGPT", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": "GPT-5 SWE-bench Verified 74.9% (SOTA at launch); GPT-5.5 successor line (current default) reported still higher", "confidence": "medium", "note": "GPT-5 set SOTA on SWE-bench Verified (74.9%) at Aug 2025 launch; the line has iterated through GPT-5.5 and remains a current-frontier chat/agentic family alongside Opus 4.8 / Gemini 3.5. Frontier-definer within finetuned_chat => 5. Per-SKU 5.5 numbers are vendor-side; headline 74.9 is the GPT-5 launch figure.", "sources": [ { "url": "https://en.wikipedia.org/wiki/GPT-5", "shows": "GPT-5 SWE-bench Verified 74.9% SOTA at launch", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "OpenAI GPT-5 line: GPT-5 (Aug 7 2025, became ChatGPT default replacing GPT-4o) through GPT-5.1/5.2/5.4 and GPT-5.5 / GPT-5.5 Instant (the current ChatGPT default as of May 2026). Unified reasoning+chat post-trained model family, closed/API + ChatGPT only. Verified via OpenAI pages + Wikipedia + TechCrunch June 2026." }, { "product": "Claude Sonnet 4", "org": "Anthropic", "type": "model", "description": "Previous-generation Sonnet tier (May 2025). Strong on SWE-bench Verified and agentic tool use with extended thinking mode and 200K context, but superseded by Sonnet 4.5 (2025-09-29) and Sonnet 4.6 (2026-02-17), which is the current production default coding tier. Sonnet 4.7 was skipped per leaked Claude Code source; Sonnet 4.8 is the next planned tag.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API/Bedrock/Vertex/claude.ai)", "confidence": "high", "note": "Proprietary Anthropic model; no weights/data/code released.", "sources": [ { "url": "https://www.anthropic.com/news/claude-4", "shows": "Claude Sonnet 4 released May 22 2025, proprietary, via API/Bedrock/Vertex/claude.ai", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": null, "signal_type": "reported_traction", "confidence": "low", "note": "Directional market-position estimate (not a measured usage figure). Sonnet 4 was a high-volume workhorse on the mass-market claude.ai surface (third-party estimates ~30M+ consumer MAU, aggregator-only), but is now superseded by 4.5/4.6, so 4 rather than 5. Overrides the prior frozen-anchor convention (held null) for dominant closed incumbents.", "sources": [ { "url": "https://www.anthropic.com/news/claude-4", "shows": "launch post gives customer testimonials but no quantitative per-model usage figures", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/rankings", "shows": "public per-model token-share leaderboard where Claude flagships rank among the most-used models", "accessed": "2026-06-23" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 72.7%", "confidence": "high", "note": "Strong agentic/coding model at May 2025 (72.7 SWE-bench, near Opus 4's 72.5). By June 2026 surpassed within its own family (Sonnet 4.6 79.0, Opus 4.8 88.6) but still a capable mid-high chat/coding model; 4 frontier-relative.", "sources": [ { "url": "https://www.anthropic.com/news/claude-4", "shows": "Claude Sonnet 4 SWE-bench Verified 72.7%", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Anthropic Claude Sonnet 4, released May 22 2025 alongside Opus 4 (Claude 4 generation). Post-trained chat/agentic model, closed/API + Bedrock + Vertex + claude.ai (free tier). Superseded by Sonnet 4.5/4.6 but the May-2025 SKU is the scored entity. Verified via Anthropic Claude 4 announcement June 2026." }, { "product": "Claude Opus 4.x", "org": "Anthropic", "type": "model", "description": "Anthropic's heavy-reasoning Opus tier, tuned for sustained agentic coding and long-horizon planning across large codebases. Opus 4 scored 72.5% on SWE-bench Verified at release (May 2025 era); the line has iterated through Opus 4.5 (80.9%), Opus 4.6 (80.8%), and Opus 4.7 (April 2026), which hits 87.6% SWE-bench Verified and 98.5% computer-use visual acuity. Opus 4.6 and 4.7 are tied at the top of LMArena (Elo ~1500). A leaked 'Claude Mythos Preview' tops SWE-bench at 93.9% and is presumed to be the next-generation Opus / Claude 5 candidate.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API/Bedrock/Vertex/claude.ai)", "confidence": "high", "note": "Proprietary Anthropic flagship line; no downloadable weights.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-8", "shows": "Opus 4.8 (May 28 2026) proprietary, claude-opus-4-8 via API at $5/$25 per Mtok", "accessed": "2026-06-04" }, { "url": "https://www.anthropic.com/news/claude-4", "shows": "Opus 4 line origin (May 22 2025), proprietary, API/Bedrock/Vertex", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": null, "signal_type": "reported_traction", "confidence": "low", "note": "Directional market-position estimate (not a measured usage figure). The Opus line is Anthropic's premium frontier tier, top-tier but lower volume than the Sonnet workhorse, so 4. claude.ai is a mass-market surface and Opus flagships appear on public LLM marketplace leaderboards (e.g. OpenRouter); no primary per-SKU figure exists. Overrides the prior frozen-anchor convention (held null), which understated dominant closed incumbents and could mask the openness gap.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-8", "shows": "Opus 4.8 launch post provides no quantitative user/usage figures", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/rankings", "shows": "public per-model token-share leaderboard where Claude flagships rank among the most-used models", "accessed": "2026-06-23" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": "Opus 4.8 SWE-bench Verified 88.6% (Opus 4.7 87.6%); ~4x less likely than predecessor to let code flaws pass; line-best across coding/agentic/reasoning", "confidence": "high", "note": "Opus 4.x is the current frontier-definer for chat/agentic coding (4.8 leads SWE-bench Verified at 88.6%). Unambiguous 5 within finetuned_chat. Headline SWE-bench figure is aggregator-reported for the latest SKUs; Anthropic primary page confirms line-best benchmarks + the 4x honesty improvement.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-8", "shows": "Opus 4.8 improvements across coding/agentic/reasoning benchmarks vs 4.7; honesty 4x gain", "accessed": "2026-06-04" }, { "url": "https://www.vellum.ai/blog/claude-opus-4-8-benchmarks-explained", "shows": "Opus 4.8 SWE-bench Verified 88.6%, Opus 4.7 87.6% (corroborating)", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Anthropic Claude Opus 4.x flagship line: Opus 4 (May 22 2025) -> 4.5 (Nov 2025) -> 4.6 (Feb 2026) -> 4.7 (Apr 16 2026) -> 4.8 (May 28 2026, current frontier, model id claude-opus-4-8). Post-trained chat/agentic flagship, closed/API + Bedrock + Vertex + claude.ai. Note: Opus 4.7 is a frozen base_pretrained anchor; this record scores the Opus 4.x line in finetuned_chat. Verified via Anthropic Opus 4.8 + Claude 4 pages June 2026." }, { "product": "Claude Sonnet 4.6", "org": "Anthropic", "type": "model", "description": "Anthropic's current production default coding tier (released 2026-02-17), following Sonnet 4 (May 2025) and Sonnet 4.5 (Sept 2025). Posts 79.6% on SWE-bench Verified (top-10) with extended thinking mode and 200K context (1M in beta). Powers Claude Code CLI and most enterprise coding deployments where Opus 4.7 is reserved for the hardest agentic tasks. Sonnet 4.7 was skipped; Sonnet 4.8 is the next planned tag per leaked source.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API/claude.ai/Claude Code/clouds)", "confidence": "high", "note": "Proprietary Anthropic model; no weights/data/code released.", "sources": [ { "url": "https://www.anthropic.com/news/claude-sonnet-4-6", "shows": "Sonnet 4.6 released Feb 17 2026, proprietary, claude-sonnet-4-6 via API/claude.ai/Claude Code", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": null, "signal_type": "reported_traction", "confidence": "medium", "note": "Directional market-position estimate (not a measured usage figure). The Sonnet line is Anthropic's highest-volume model and the default workhorse for coding/agent use; Sonnet flagships sit at or near the top of public LLM marketplace token-share leaderboards (e.g. OpenRouter). Corroborating qualitative signal: users preferred 4.6 ~70% over predecessor, 59% over Opus 4.5; GitHub/Databricks/Replit testimonials; claude.ai mass-market surface. Overrides the prior frozen-anchor convention (held null), which systematically understated the dominant closed incumbents and could mask the openness gap.", "sources": [ { "url": "https://www.anthropic.com/news/claude-sonnet-4-6", "shows": "qualitative preference data + testimonials (no quantitative per-model usage figures)", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/rankings", "shows": "public per-model token-share leaderboard where Claude Sonnet flagships rank among the most-used models", "accessed": "2026-06-23" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 79.0% (80.2% with prompt modification); 72.5% OSWorld (computer use); 1.2pts behind Opus 4.6", "confidence": "high", "note": "Near-flagship: 79.0-80.2 SWE-bench at one-fifth Opus price, only ~1.2pts behind Opus 4.6, strong computer-use. Just below the Opus-4.8/GPT-5.5/Gemini-3.5 frontier => 4, not 5.", "sources": [ { "url": "https://www.anthropic.com/news/claude-sonnet-4-6", "shows": "Sonnet 4.6 SWE-bench Verified 79.0% (80.2% with prompt modification)", "accessed": "2026-06-04" }, { "url": "https://www.nxcode.io/resources/news/claude-sonnet-4-6-complete-guide-benchmarks-pricing-2026", "shows": "79.6% SWE-bench + 72.5% OSWorld (corroborating)", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "Anthropic Claude Sonnet 4.6 (model id claude-sonnet-4-6), released Feb 17 2026. Post-trained mid-tier chat/agentic model at $3/$15 per Mtok; first Sonnet-class with 1M-token full-codebase analysis. Closed/API + claude.ai + Claude Code + clouds. Verified via Anthropic Sonnet 4.6 page June 2026." }, { "product": "Claude Haiku 4.5", "org": "Anthropic", "type": "model", "description": "Anthropic's small/fast tier in the Claude 4 generation, designed for high-volume low-latency coding and chat workloads. Sits beneath Sonnet 4.6 and Opus 4.7 in capability but at substantially lower cost, the default Anthropic option for embedded agents, autocompletion, and high-QPS production routes. Released alongside the Sonnet 4.5/4.6 cadence as Anthropic's three-tier lineup (Haiku/Sonnet/Opus) stabilized at the 4.x generation.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API + Claude.ai/Bedrock/Vertex, no downloadable weights)", "confidence": "high", "note": "Closed proprietary model; no weights, data, or training code released.", "sources": [ { "url": "https://www.anthropic.com/news/claude-haiku-4-5", "shows": "proprietary release, available via API/Claude.ai/Bedrock/Vertex, no open weights", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Distributed across Claude.ai, Claude Code, Claude API, Amazon Bedrock and Vertex AI as Anthropic's default fast/cheap tier; positioned to power high-volume agentic and sub-agent workloads. Anthropic discloses no per-model user/usage count, so reported_traction rather than a measured figure; placed at 4 on breadth of production distribution as the cost-efficient workhorse SKU.", "sources": [ { "url": "https://www.anthropic.com/news/claude-haiku-4-5", "shows": "availability across Claude.ai, Claude Code, API, Bedrock, Vertex AI", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 73.3% (avg of 50 trials, 128K thinking budget); ~90% of Sonnet 4.5 agentic-coding performance; matches/exceeds Sonnet 4 on computer use", "confidence": "high", "note": "Very strong for a small/fast model, but a cost-tier SKU below the Opus/Gemini 3.x/GPT-5 frontier-definers in this category; scored 4, not 5.", "sources": [ { "url": "https://www.anthropic.com/news/claude-haiku-4-5", "shows": "SWE-bench Verified 73.3%, ~90% of Sonnet 4.5 agentic coding", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Anthropic Claude Haiku 4.5 (claude-haiku-4-5), released Oct 15, 2025. Post-trained instruct/agentic chat model; small/fast frontier tier (~90% of Sonnet 4.5 in agentic coding). Verified live June 2026 via anthropic.com; proprietary API-only (also Bedrock/Vertex). Successor Opus/Sonnet 4.7-4.8 exist but Haiku 4.5 remains the current Haiku SKU." }, { "product": "Gemini 2.5 Pro", "org": "Google", "type": "model", "description": "Previous-generation Gemini Pro tier (2025) with native multimodal understanding, 1M+ token context, and hybrid thinking mode. Superseded by Gemini 3 Pro and Gemini 3.1 Pro Preview, which leads LiveCodeBench Pro at 2887 Elo and posts 80.6% on SWE-bench Verified. Gemini 3.5 Pro is expected June 2026. 2.5 Pro remains available via Google AI Studio and Vertex AI for cost-conscious workloads.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(Gemini API/AI Studio/Vertex/Gemini app, no downloadable weights)", "confidence": "high", "note": "Closed proprietary model; Google released no weights/data/training code.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/", "shows": "proprietary launch via AI Studio/Gemini app/Vertex, no open weights", "accessed": "2026-06-04" }, { "url": "https://ai.google.dev/gemini-api/docs/models", "shows": "Gemini 2.5 Pro still listed as available API model June 2026", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Gemini 2.5 Pro was the Gemini app flagship for much of 2025 and remains a production API/Vertex model; the Gemini app surpassed 750M monthly active users (TechCrunch, Feb 2026). Reach attributed to the Gemini app + AI Studio/Vertex surface it powers, not a standalone count. Well above the >10M threshold.", "sources": [ { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "Gemini app >750M MAU (Q4 2025 figure)", "accessed": "2026-06-04" }, { "url": "https://ai.google.dev/gemini-api/docs/models", "shows": "Gemini 2.5 Pro production-available across API/Vertex", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:GPQA/SWE-bench/LMArena", "value": "GPQA Diamond 84.0%, SWE-bench Verified 63.8% (custom agent), Humanity's Last Exam 18.8% (no tools), LMArena #1 at launch (~1370 Elo, Mar 2025)", "confidence": "high", "note": "Frontier when launched (Mar 2025) but a 2025-generation model; now mid-tier vs Gemini 3.1/3.5, GPT-5.x and DeepSeek/Qwen frontier. Scored 3 against the 2026 frontier comparison set.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/", "shows": "GPQA Diamond SOTA, SWE-bench Verified 63.8%, HLE 18.8%, LMArena #1", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Google Gemini 2.5 Pro, first released as experimental preview Mar 25, 2025; thinking/reasoning instruct model, 1M-token context. Still actively listed on the Gemini API as 'most advanced for complex tasks' (not in the deprecated 'Previous models' list) as of June 2026, though superseded as flagship by Gemini 3/3.1/3.5. Proprietary API-only." }, { "product": "Gemini 2.5 Flash", "org": "Google", "type": "model", "description": "Previous-generation speed-optimized model (2025) with configurable thinking budgets. Superseded by Gemini 3 Flash and Gemini 3.5 Flash (May 19, 2026), the new default in the Gemini app: 'Pro-grade reasoning at Flash-level speed,' #9 on the May 2026 LMArena text leaderboard (Elo 1480) and #2 on LiveCodeBench at 90.8%. 2.5 Flash remains a popular cost-tier for high-volume deployments.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(Gemini API/AI Studio/Vertex/Gemini app, no downloadable weights)", "confidence": "high", "note": "Closed proprietary model; no open components.", "sources": [ { "url": "https://ai.google.dev/gemini-api/docs/models", "shows": "Gemini 2.5 Flash listed as proprietary API model, no weights, available June 2026", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "High-volume cheap tier powering the Gemini app and AI Mode in Search plus heavy API/Vertex traffic; Gemini app >750M MAU (TechCrunch, Feb 2026). Flash is the default high-volume serving model so its real usage is firmly >10M-equivalent. Attributed to the app/Search surface, not a standalone download count.", "sources": [ { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "Gemini app >750M MAU", "accessed": "2026-06-04" }, { "url": "https://ai.google.dev/gemini-api/docs/models", "shows": "Gemini 2.5 Flash positioned as high-volume low-latency production model", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:GPQA", "value": "GPQA Diamond ~82.8% (pass@1, gemini-2.5-flash-preview); thinking-enabled mid-tier reasoning", "confidence": "medium", "note": "Strong for a cheap/fast 2025 model and close to 2.5 Pro on GPQA, but a price-performance tier and a 2025 generation; mid-tier vs 2026 frontier. Scored 3. GPQA figure corroborated via Artificial Analysis/AI Studio model-id reporting; primary Google model card is a PDF not fetched this run, so confidence medium.", "sources": [ { "url": "https://ai.google.dev/gemini-api/docs/models", "shows": "Gemini 2.5 Flash as price-performance reasoning model", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Google Gemini 2.5 Flash, released June 17, 2025; price-performance thinking instruct model. Still listed on the Gemini API as 'best price-performance for low-latency high-volume' (not deprecated) as of June 2026. Proprietary API-only." }, { "product": "Gemini 3.1 Pro Preview", "org": "Google", "type": "model", "description": "Google's current top-coding-tier closed model. Leads LiveCodeBench Pro at 2887 Elo, posts 80.6% on SWE-bench Verified (#5 on the May 2026 leaderboard) and 88.5% on LiveCodeBench, with native multimodal understanding and a 1M+ token context window. Sits at #6 on the May 2026 LMArena text leaderboard (Elo 1488), effectively tied with Claude Opus 4.6/4.7 within the 95% CI. Gemini 3.5 Pro is expected June 2026.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API/Vertex/Gemini app, no downloadable weights)", "confidence": "high", "note": "Closed proprietary model; no weights/data/code released.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "shows": "proprietary preview launch across Gemini app/API/Vertex, no open weights", "accessed": "2026-06-04" }, { "url": "https://deepmind.google/models/model-cards/gemini-3-1-pro/", "shows": "official model card, proprietary", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Rolling out across the Gemini app, NotebookLM, AI Studio/CLI/Antigravity and Vertex/Gemini Enterprise; Gemini app >750M MAU (TechCrunch, Feb 2026). As the current Pro flagship it powers the high-traffic Gemini surfaces, putting real reach >10M. Attributed to the app/API surface, not a standalone count.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "shows": "rollout across Gemini app, NotebookLM, API, Vertex/Enterprise", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "Gemini app >750M MAU", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:GPQA/ARC-AGI-2/SWE-bench", "value": "GPQA Diamond 94.3% (Thinking High), ARC-AGI-2 77.1%, SWE-bench Verified 80.6%, Humanity's Last Exam 44.4% (no tools)/51.4% (with tools), MMMU-Pro 80.5%", "confidence": "high", "note": "Frontier-definer for chat/reasoning at scoring time, leads or near-leads GPQA, ARC-AGI-2 and SWE-bench. Scored 5; calibrates with the Gemini 3.5 Flash anchor (cap 5). Google-reported figures from the model card.", "sources": [ { "url": "https://deepmind.google/models/model-cards/gemini-3-1-pro/", "shows": "GPQA 94.3, ARC-AGI-2 77.1, SWE-bench Verified 80.6, HLE 44.4/51.4, MMMU-Pro 80.5", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Google Gemini 3.1 Pro, released in preview Feb 19, 2026; Google's most advanced reasoning instruct model, 1M-token context. Available via Gemini app/NotebookLM (Pro/Ultra), Gemini API (AI Studio, Antigravity, Gemini CLI, Android Studio) and Vertex AI/Gemini Enterprise. Proprietary; 'Preview' label is the current GA-track SKU. Verified live June 2026." }, { "product": "Gemini 3.5 Flash", "org": "Google", "type": "model", "description": "Released at Google I/O 2026 (May 19, 2026) as the new default in the Gemini app. Marketed as 'Pro-grade reasoning at Flash-level speed'; #9 on the May 2026 LMArena text leaderboard (Elo 1480) and #2 on LiveCodeBench at 90.8%. Configurable thinking budgets let users trade reasoning depth for latency. Powers Antigravity / Gemini CLI and most Google-side coding-assistant deployments where cost matters.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(Gemini app/Search/API/enterprise, no downloadable weights)", "confidence": "high", "note": "Closed proprietary model; no open components. Consistent with the flagship anchor.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "proprietary launch across Gemini app/Search/API/enterprise, no open weights", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Distributed via the Gemini app and AI Mode in Google Search 'to billions globally'; Gemini app >750M MAU (TechCrunch, Feb 2026). Well above >10M; attributed to the app/Search surface it powers. Matches flagship anchor (level 5).", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "distributed via Gemini app and AI Mode in Search to billions globally", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "Gemini app >750M MAU", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:Terminal-Bench 2.1", "value": "Terminal-Bench 2.1 76.2%, MCP Atlas 83.6%, CharXiv Reasoning 84.2%, GDPval-AA 1656 Elo; reported to lead Opus 4.7 and GPT-5.5 on several agentic/multimodal suites", "confidence": "high", "note": "Frontier-tier; top-right of Artificial Analysis Intelligence Index. Scored 5, consistent with the flagship anchor (cap 5). Google-reported figures.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "Terminal-Bench 2.1 76.2, MCP Atlas 83.6, CharXiv 84.2, GDPval-AA 1656 Elo", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Google Gemini 3.5 Flash, launched at I/O 2026 on May 19, 2026 as the first Gemini 3.5 model; price-performance thinking/agentic instruct model. Available via Gemini app, AI Mode in Search, Google Antigravity, Gemini API and enterprise. Proprietary API-only. (Matches the flagship anchor 'Gemini 3.5 Flash'; scored here for the finetuned_chat category, calibrated to that anchor.) Verified live June 2026." }, { "product": "Grok 3", "org": "xAI", "type": "model", "description": "Previous-generation xAI flagship (February 2025), trained on the 100K-H100 Colossus cluster. Superseded by Grok 4, Grok 4.1, and Grok 4.20 Heavy (Feb 2026); the latter uses a 16-agent specialized system. Grok 5 is still in training (claimed 10T params; ~33% chance of release by 2026-06-30 per prediction markets). Grok 3 remains available via X platform integration.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(grok.com / X / xAI API, no downloadable weights for Grok 3)", "confidence": "high", "note": "Closed proprietary model. (xAI has open-weighted older flagships e.g. Grok-1/2 after a lag, but Grok 3 weights are not released.)", "sources": [ { "url": "https://x.ai/news/grok-3", "shows": "Grok 3 Beta launch (proprietary), 'Age of Reasoning Agents'; confirmed via search; no open weights", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Available to X users and via grok.com/xAI API in 2025 as the flagship Grok generation; now a superseded model behind Grok 4.x default. No clean standalone Grok-3 user/usage count published. Placed at 3 on broad X-surface availability as a former flagship, downgraded from app-surface 5 because xAI discloses no honest per-model figure and it is no longer the default. Directional.", "sources": [ { "url": "https://x.ai/news/grok-3", "shows": "Grok 3 launched as xAI flagship reasoning model on X/grok.com", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:AIME/GPQA/LMArena", "value": "LMArena/Chatbot Arena Elo ~1402 (Mar 2025, #1 at launch); reported SOTA among non-reasoning models on AIME, GPQA, MMLU-Pro, LiveCodeBench at launch vs Gemini 2 Pro / GPT-4o / Claude 3.5 Sonnet", "confidence": "medium", "note": "Frontier at Feb-Mar 2025 launch but a 2025-generation model superseded by Grok 4/4.20/4.3; mid-tier vs 2026 frontier. Scored 3. Headline Elo from Chatbot Arena via xAI's primary blog (direct fetch 403; figure corroborated through search of the primary source).", "sources": [ { "url": "https://x.ai/news/grok-3", "shows": "Chatbot Arena Elo 1402, SOTA non-reasoning at launch on AIME/GPQA/LiveCodeBench", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "xAI Grok 3, released Feb 17, 2025; trained on the Colossus supercluster. Post-trained chat/assistant model (Grok 3 + Grok 3 Reasoning/mini variants). Superseded by Grok 4 / 4.1 / 4.20 / 4.3 but remains a foundational xAI assistant generation. Proprietary API-only / X-integrated. x.ai/news/grok-3 returned 403 on direct fetch; existence and figures confirmed via search of the primary blog and Chatbot Arena." }, { "product": "Grok 4.20 Heavy", "org": "xAI", "type": "model", "description": "xAI's current flagship instruction-tuned product (Feb 2026), built as a 16-agent specialized system, multiple Grok-4 instances coordinated around specialized roles rather than a single monolithic call. Marketed as xAI's top reasoning + agentic tier; ships alongside Grok 4 and Grok 4.1 as the standard API tiers. Grok 5 is still in training (claimed 10T params; ~33% chance of release by 2026-06-30 per prediction markets).", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(SuperGrok Heavy / xAI API, no downloadable weights; xAI open sources only its system prompts)", "confidence": "high", "note": "Closed proprietary model; only the Grok system prompts are open sourced, not weights/data/training code. Consistent with the Grok 4.20 anchor (closed).", "sources": [ { "url": "https://artificialanalysis.ai/models/grok-4-20", "shows": "Grok 4.20 proprietary API model, no open weights", "accessed": "2026-06-04" }, { "url": "https://natural20.com/coverage/grok-420-xai-four-agents-system-benchmarks-jailbreak", "shows": "Heavy = 16-agent high-effort tier; xAI open sources prompts only", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "low", "note": "Heavy is gated behind the $300/mo SuperGrok Heavy tier (highest, niche) plus high-effort API; far narrower than the base Grok 4.20 surface. The Grok 4.20 flagship anchor sits at level 2; Heavy is a more expensive niche variant so capped at 2. No published Heavy-specific user count. Directional.", "sources": [ { "url": "https://grok.com/plans", "shows": "SuperGrok Heavy $300/mo tier providing Heavy multi-agent access", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:Artificial Analysis Intelligence Index / AIME", "value": "AIME 93.3% (Grok 4.20); AA Intelligence Index 49 (Grok 4.20 0309 v2, #24/151, median 36); record 78% non-hallucination on AA Omniscience; IFBench 83%; provisional LMArena Elo ~1505-1535. Heavy = 16-agent high-effort mode (low/med uses 4 agents, high/xhigh uses 16).", "confidence": "medium", "note": "Strong but a notch below the GPT-5/Gemini 3.5/Opus 4.7 frontier leaders on the AA Intelligence Index. Heavy is the max-effort mode of the same model; scored 4 to match the Grok 4.20 anchor (capability 4); high-effort agent scaling does not move it into the 5 frontier-definer band. AA Index/AIME are independent; LMArena Elo provisional.", "sources": [ { "url": "https://artificialanalysis.ai/models/grok-4-20", "shows": "AA Intelligence Index 49, #24/151; reasoning variant, 2M context", "accessed": "2026-06-04" }, { "url": "https://natural20.com/coverage/grok-420-xai-four-agents-system-benchmarks-jailbreak", "shows": "AIME 93.3, LMArena Elo ~1505-1535, Heavy = 16 agents", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "xAI Grok 4.20 Heavy, the high-effort multi-agent tier of Grok 4.20 (beta Feb 17, 2026; multi-agent API GA Mar 9, 2026). 'Heavy' scales the default 4-agent debate (Grok/Harper/Benjamin/Lucas) to 16 specialized agents; 2M-token context. Accessed via SuperGrok Heavy ($300/mo). Distinct SKU from the 'Grok 4.20' flagship anchor (default-effort). Superseded by Grok 4.3 but still offered. Verified live June 2026." }, { "product": "Codestral", "org": "Mistral AI (API)", "type": "model", "description": "Mistral's proprietary code model, API-only via la Plateforme, Vertex AI, Azure AI Foundry, and Bedrock. Current SKU is Codestral 25.08 (August 2025), 30% more accepted completions and 50% fewer runaway generations versus 25.01. Part of Mistral's broader 2026 coding stack alongside Codestral Embed (retrieval-tuned embeddings), Devstral 2 (open-weight agentic coding counterpart), and Mistral Vibe CLI (Mistral's answer to Claude Code / Gemini CLI). Distributed to thousands of developers through Continue.dev, JetBrains, and VS Code integrations; reported usage trails Anthropic and OpenAI coding models.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF);data:closed;code:closed(no training pipeline);license:Mistral-AI-Non-Production-License(MNPL, non-OSI, research/testing only; commercial license on demand)", "confidence": "high", "note": "Weights downloadable but under MNPL (non-OSI, non-production); open-weights gated by a use-restricted license, so restricted not open_weights.", "sources": [ { "url": "https://mistral.ai/news/codestral/", "shows": "Codestral open-weight on HF under Mistral AI Non-Production License (research/testing; commercial on demand)", "accessed": "2026-06-04" }, { "url": "https://mistral.ai/news/codestral-25-08/", "shows": "Codestral 25.08 released 30 Jul 2025; API + VPC + on-prem deployment", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Embedded as the default FIM/code-completion backend in IDE assistants (Continue, Tabnine, and Mistral's own coding stack) and on multiple API gateways; Mistral reports per-version completion-acceptance gains but discloses no hard download/user count. Level 3 reflects multi-platform IDE distribution, not a verified usage_volume figure.", "sources": [ { "url": "https://mistral.ai/news/codestral-25-08/", "shows": "Codestral coding stack distributed via API/VPC/on-prem; +30% accepted completions vs prior version", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:HumanEval/MBPP", "value": "HumanEval pass@1 ~86.6%, MBPP ~91.2%; strong on completion/FIM and RepoBench long-context, but ~22B specialist, weak on agentic SWE-bench (single-run resolves only a small fraction)", "confidence": "medium", "note": "Strong code-completion specialist for its size, but a narrow FIM model, mid-tier on a chat/agentic capability axis where the comparison set is 2026 frontier agentic coders.", "sources": [ { "url": "https://mistral.ai/news/codestral/", "shows": "Mistral-reported HumanEval/MBPP/RepoBench/Spider code benchmarks for Codestral", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Codestral 25.08 (released 30 Jul 2025), Mistral's code-completion/FIM specialist; instruction-tuned with a chat mode (+5% instruction following per launch). Distributed via API (console.mistral.ai), VPC, and on-prem; open weights on HF under the Mistral AI Non-Production License (MNPL). NOTE: arguably a coding-specialist model rather than a general chat model; scored best-effort in finetuned_chat per assignment with a category caveat. (Separate 'Codestral 2', Apr 2026, was relicensed Apache-2.0, not the SKU scored here.) Verified live June 2026." }, { "product": "Amazon Nova Pro", "org": "Amazon", "type": "model", "description": "Amazon's proprietary multimodal model line in AWS Bedrock. Current flagship is Amazon Nova 2 Pro (preview), launched at re:Invent 2025 (December 2, 2025) alongside Nova 2 Lite (speed/cost tier), Nova 2 Omni (unified text/image/video/audio in + text/image out), and Nova 2 Sonic (speech-to-speech). Nova 2 Pro emphasizes agentic capability gains over the original Nova line. No 'Nova 2 Premier' exists as of May 2026; Nova 2 Pro is the top-tier model. Competitive pricing for high-volume AWS-native enterprise deployments with SLA guarantees.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(API-only via Amazon Bedrock)", "confidence": "high", "note": "No downloadable weights; accessible only through Bedrock API. Fully closed.", "sources": [ { "url": "https://docs.aws.amazon.com/nova/latest/userguide/what-is-nova.html", "shows": "Nova Pro accessed only through API via Amazon Bedrock; multimodal understanding model", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "Distributed broadly through Amazon Bedrock across many AWS regions (with fine-tuning, provisioned throughput, Bedrock Agents/KB support) and positioned as the price-performance default of the Nova family. No standalone per-model user/usage count published; level 4 reflects Bedrock's enterprise distribution surface rather than a verified figure.", "sources": [ { "url": "https://docs.aws.amazon.com/nova/latest/userguide/what-is-nova.html", "shows": "Nova Pro available across many AWS regions via Bedrock with full enterprise feature support", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:MMLU", "value": "MMLU 85.9% (vs 87.4% for Nova Premier); strong multimodal/agentic positioning but a late-2024 model", "confidence": "medium", "note": "Solid mid-tier multimodal instruct model at launch (Dec 2024); below 2026 frontier (GPT-5.x/Gemini 3.5/Opus 4.x). MMLU is corroborated via comparison pages, not the vendor leaderboard.", "sources": [ { "url": "https://assets.amazon.science/96/7d/0d3e59514abf8fdcfafcdc574300/nova-tech-report-20250317-0810.pdf", "shows": "Amazon Nova family technical report with Nova Pro benchmark results", "accessed": "2026-06-04" }, { "url": "https://docs.aws.amazon.com/nova/latest/userguide/what-is-nova.html", "shows": "Nova Pro is the price-performance multimodal model of the family", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Amazon Nova Pro (amazon.nova-pro-v1:0), released 3 Dec 2024; multimodal understanding/instruct model (text+image+video in, text out), 300K context. Proprietary, API-only via Amazon Bedrock. Succeeded by Amazon Nova 2 family (2026) but Nova Pro v1 remains in service. Verified live in AWS Bedrock docs June 2026." }, { "product": "GLM-4", "org": "Zhipu AI", "type": "model", "description": "Zhipu AI's flagship instruction-tuned model line, now at GLM-4.6 (late 2025) following GLM-4.5 (July 2025). Strong multilingual coding (especially Chinese-English) with 128K+ context and native tool use. Notable structural shift: GLM-4.5 and 4.6 ship open-weights variants (a reversal of the GLM-4 API-only stance); Zhipu now competes directly in the open-weight frontier alongside DeepSeek, Qwen, and Kimi. Widely deployed in Chinese enterprise environments via API and self-hosted weights.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(on HF);data:closed;code:partial(inference/demo);license:GLM-4-License(non-OSI: free commercial use requires registration + 'Built with glm-4' attribution + glm-4 name prefix on derivatives; field-of-use prohibitions)", "confidence": "high", "note": "Downloadable weights but governed by a non-OSI custom license with registration and attribution obligations, restricted, not open_weights. No MAU cap.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-4-9B-Chat/blob/main/LICENSE", "shows": "GLM-4 custom non-OSI license: commercial use needs registration, 'Built with glm-4' attribution, name-prefix requirement", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/zai-org/GLM-4-9B-Chat", "shows": "GLM-4-9B-Chat open-weight instruct model, 9B, 128K context, June 2024", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "HF model card shows 121,169 downloads in the last month (100K-1M band), confirming the level-3 placement on a primary usage figure. Adoption has shifted to the newer GLM-4.5/5/5.1 generations, but GLM-4-9B-Chat remains a meaningfully-pulled small Chinese open model. Headline figure is the HF monthly download count.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-4-9B-Chat", "shows": "GLM-4-9B-Chat model card: 121,169 downloads last month (primary usage figure)", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU", "value": "GLM-4-9B-Chat competitive with Llama-3-8B-Instruct class in 2024 (function calling, web browsing, code execution, 26 languages); a 9B mid-2024 model", "confidence": "medium", "note": "Strong for a 9B model at 2024 release, but well below 2026 frontier on a capability axis dominated by frontier agentic/reasoning models. The GLM line's frontier capability lives in GLM-5.1, not GLM-4-9B.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-4-9B-Chat", "shows": "GLM-4-9B-Chat capabilities (function calling, code execution, long context) and benchmark positioning", "accessed": "2026-06-04" } ] }, "maturity": 2.3, "mature": false, "version_note": "GLM-4-9B-Chat (Zhipu AI / Z.ai, org zai-org), instruction-tuned chat model; 9B params, 128K context (1M variant available), released ~June 2024 (arXiv 2406.12793). Open weights under the custom GLM-4 license. Dated relative to 2026 frontier (GLM-5/GLM-5.1 are the current generation). Verified live on HF June 2026." }, { "product": "Muse Spark", "org": "Meta Superintelligence Labs", "type": "model", "description": "Meta Superintelligence Labs' flagship closed-weight API model, launched April 2026. Closed top-5 on the May 2026 LMArena text leaderboard (Elo 1489), the first non-Anthropic/Google/OpenAI model to break the top 5 in 2026. Represents Meta's parallel closed-weights track alongside the open-weights Llama line (Llama 5, April 2026), effectively splitting Meta into two flagship brands. Chat/instruction-tuned tier sits in Cat 3 alongside other API-only frontier models.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(meta.ai/Meta AI app + private API preview; no downloadable weights)", "confidence": "high", "note": "No weights released, a closed assistant-surface model; only a private API preview. Fully closed.", "sources": [ { "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/", "shows": "Muse Spark available at meta.ai and Meta AI app + private API preview; no open weights", "accessed": "2026-06-04" }, { "url": "https://about.fb.com/news/2026/04/introducing-muse-spark-meta-superintelligence-labs/", "shows": "Meta primary newsroom post: Muse Spark proprietary (hope to open source future versions); powers Meta AI app/meta.ai + private API preview to select partners", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "active_users", "confidence": "low", "note": "Powers the Meta AI assistant on meta.ai and the Meta AI app (rolling out to WhatsApp/Instagram/Facebook/Messenger/AI glasses), an enormous consumer surface, but Meta discloses no Muse Spark-specific user count, and at scoring time it is not yet the model behind the full family of surfaces. Placed at 4 on the consumer-app surface it powers; not 5 because no standalone figure and rollout incomplete. Attributed honestly to the surface, not the model.", "sources": [ { "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/", "shows": "Muse Spark powers Meta AI assistant in Meta AI app / meta.ai, rolling out across Meta surfaces", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena/HLE", "value": "LMArena #6 overall (~1487 Elo, mid-2026); Humanity's Last Exam 58%, FrontierScience Research 38% (Contemplating mode); positioned vs Gemini Deep Think / GPT Pro extreme-reasoning modes", "confidence": "medium", "note": "Bumped 4 -> 5: the independent confirmation the prior note awaited now exists - LMArena ranks Muse Spark #6 overall (top of the non-Anthropic/Google frontier), corroborating Meta's vendor-reported reasoning scores.", "sources": [ { "url": "https://ai.meta.com/blog/introducing-muse-spark-msl/", "shows": "Meta-reported HLE 58% and FrontierScience 38% for Muse Spark Contemplating mode", "accessed": "2026-06-04" }, { "url": "https://lmarena.ai/leaderboard", "shows": "Muse Spark ranked", "accessed": "2026-06-18" } ] }, "maturity": 4.7, "mature": false, "version_note": "Muse Spark, first model of Meta Superintelligence Labs' new 'Muse' series; released 8 Apr 2026. Natively multimodal reasoning model with tool-use, visual chain-of-thought, and multi-agent 'Contemplating' mode. Proprietary/API-only: powers the Meta AI assistant (meta.ai + Meta AI app), private API preview to select users; no open weights. Verified via Meta's own blog June 2026." }, { "product": "MiniMax M2.5", "org": "MiniMax", "type": "model", "description": "MiniMax's flagship instruction-tuned coding model, top-10 on SWE-bench Verified at 80.2% as of May 2026. Part of the wave of Chinese frontier-tier coding models (alongside DeepSeek V4, Kimi K2.6, Qwen3-Coder-Next, and GLM-4.6) closing the gap with Anthropic/Google/OpenAI on standard coding evals. Available via MiniMax's hosted API with multi-modal extensions.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(on HF + ModelScope; F32/BF16/F8_E4M3);data:closed;code:partial(inference via SGLang/vLLM/Transformers; no training pipeline/data);license:Modified-MIT(permissive)", "confidence": "high", "note": "Downloadable weights under a permissive Modified-MIT; data and training pipeline closed, so open_weights not open_source. The newer MiniMax-M3 has since also released open weights (under a more restrictive community license).", "sources": [ { "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5", "shows": "MiniMax-M2.5 open weights on HF, Modified-MIT, 229B MoE, ~693K downloads/month", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "HF model card reports ~693,331 downloads in the last month (100K-1M band); additionally served via OpenRouter and free in coding tools (Kilo Code). Headline figure is the HF monthly download count.", "sources": [ { "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5", "shows": "~693,331 downloads last month on the official HF model page", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 80.2%, SWE-bench Pro 55.4%, GPQA-D 85.2%, AIME25 86.3%, BrowseComp 76.3%; VIBE-Pro on par with Opus 4.5", "confidence": "medium", "note": "Frontier-tier among open-weights agentic chat models (SWE-bench Verified 80.2% with strong reasoning/browse scores), at very low serving cost. Figures are MiniMax-reported on the HF card; not independently audited but consistent with peer open models (Kimi K2.6, DeepSeek-V4).", "sources": [ { "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5", "shows": "MiniMax-reported SWE-bench Verified 80.2%, GPQA-D 85.2%, AIME25 86.3% on the official HF card", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "MiniMax-M2.5 (MiniMaxAI), 229B total / 10B active MoE, agentic chat/instruct model RL-trained across 200K+ environments; released 12 Feb 2026. Open weights on HF (also ModelScope) under Modified-MIT. ~100 tok/s native; ~$0.30-$1.00/hour serving. Verified live on HF June 2026 (~693K downloads in the last month)." }, { "product": "Doubao Seed 2.0", "org": "ByteDance Seed / Volcano Engine", "type": "model", "description": "ByteDance's flagship closed-weight instruction-tuned model line, released February 14, 2026 in four variants (Pro, Lite, Mini, Code) via the Volcano Engine API. The Pro tier ($0.47 input / $2.37 output per million tokens) posts frontier-tier benchmarks (98.3 AIME 2025, 88.9 GPQA Diamond, 76.5 SWE-Bench Verified, 3020 Codeforces) at 3.7×–10× lower cost than GPT-5.2 / Claude Opus 4.5. Differentiates on (a) Chinese-language strength and Volcano Engine's mainland-China-native infrastructure, (b) aggressive price-to-performance positioning, and (c) deep distribution via the Doubao consumer chat app, which crossed 100M DAU in early 2026 and reached ~345M MAU by March 2026, the largest consumer AI chat product in China. Doubao Seed 1.6 (October 2025, 256K context) remains in the lineup as the budget tier; ByteDance ships 19 active Doubao API SKUs in 2026.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:Proprietary(Doubao app + Volcano Engine API only; no downloadable weights)", "confidence": "high", "note": "No weights released, served only through the Doubao app and Volcano Engine API. Fully closed.", "sources": [ { "url": "https://technode.com/2026/02/14/bytedance-releases-doubao-seed-2-0-positions-pro-model-against-gpt-5-2-and-gemini-3-pro/", "shows": "Doubao Seed 2.0 available on Doubao app and Volcano Engine API (no open weights)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Powers the Doubao app, China's #1 AI chatbot with ~155M weekly active users; Seed 2.0 is the model behind that surface plus Volcano Engine enterprise API. Far above >10M. Attributed to the Doubao app surface the model powers, not a standalone Seed 2.0 figure. WAU sourced from launch coverage; vendor primary page (seed.bytedance.com) returned no figure this run.", "sources": [ { "url": "https://technode.com/2026/02/14/bytedance-releases-doubao-seed-2-0-positions-pro-model-against-gpt-5-2-and-gemini-3-pro/", "shows": "Doubao Seed 2.0 launch; powers Doubao app (China's #1 AI chatbot) on Volcano Engine", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:AIME/Codeforces", "value": "Seed 2.0 Pro: AIME 2025 98.3, Codeforces rating 3020, VideoMME 89.5; positioned vs GPT-5.2 / Claude Opus 4.5 / Gemini 3 Pro", "confidence": "low", "note": "Vendor-reported (ByteDance) frontier-competitive math/coding/multimodal scores at ~10x lower token cost; figures are not independently audited. Held with low confidence as ByteDance-claimed; corroborated only via launch coverage, not a vendor benchmark page or independent leaderboard fetched this run.", "sources": [ { "url": "https://technode.com/2026/02/14/bytedance-releases-doubao-seed-2-0-positions-pro-model-against-gpt-5-2-and-gemini-3-pro/", "shows": "ByteDance-claimed leading multimodal/math/coding benchmark results for Seed 2.0 Pro vs GPT-5.2/Gemini 3 Pro", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Doubao Seed 2.0 (ByteDance), foundation model family released 14 Feb 2026; variants Pro / Lite / Mini / Code. Multimodal reasoning/agentic instruct model; proprietary/API-only via Volcano Engine (Volcengine) and the consumer Doubao app. Scored facts are for Seed 2.0 Pro (the flagship). Verified via TechNode coverage of ByteDance's launch June 2026." }, { "product": "Nemotron 3", "org": "NVIDIA", "type": "model", "description": "NVIDIA's first-party open-weight instruction-tuned model family for agentic reasoning, post-trained with SFT + RL on top of Nemotron Base. Current SKUs are Nemotron 3 Nano (30B total / 3B active hybrid MoE, March–April 2026) and Nemotron 3 Super (120B / 12B active Mamba-Transformer MoE, March 11, 2026); Nemotron 3 Ultra (~500B / 50B active) is announced for H1 2026. Differentiates on (a) open training data and recipes published alongside the weights, (b) 1M-token context with strong RULER scores (Super 91.8 @ 1M), and (c) being NVIDIA's reference post-trained line for NeMo and NIM, optimized end-to-end against NVIDIA hardware. Adoption signal: 1.38M monthly Hugging Face downloads on Nano BF16 plus 758K on Super BF16, and free hosting on OpenRouter and build.nvidia.com.", "openness": { "score": 4, "class": "open_weights", "confidence": "high", "components": "weights:open;data:open(~10T-token pretraining + ~40M post-training/SFT+RL samples released as Nemotron-SFT-Data/Nemotron-RL-Data);code:open(training recipes + RL env configs);license:NVIDIA-Nemotron-Open-Model-License(permissive,commercial,non-OSI)", "note": "Unusually open for post-trained models: NVIDIA releases weights + pretraining and post-training data + recipes, but under the non-OSI NVIDIA Open Model License, so open_weights not open_source. Calibrated to the Nemotron 3 base anchor (O4 open_weights).", "sources": [ { "url": "https://research.nvidia.com/labs/nemotron/Nemotron-3/", "shows": "Nemotron 3 Nano/Super/Ultra family; post-trained models; Nemotron-SFT-Data + Nemotron-RL-Data + training recipes released", "accessed": "2026-06-04" }, { "url": "https://developer.nvidia.com/blog/introducing-nemotron-3-super-an-open-hybrid-mamba-transformer-moe-for-agentic-reasoning/", "shows": "NVIDIA primary blog: Nemotron 3 Super 'fully open with open weights, datasets, and recipes'; license = NVIDIA Nemotron Open Model License (non-OSI)", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Distributed free via OpenRouter (nemotron-3-super-120b-a12b:free), NVIDIA build/NIM, and Hugging Face; positioned for agentic/enterprise use. No verified hard download/user count for the instruct SKUs at scoring (HF repo gated). Level 3 on reported multi-platform distribution, consistent with the base anchor (A3).", "sources": [ { "url": "https://research.nvidia.com/labs/nemotron/Nemotron-3/", "shows": "official distribution via NVIDIA build/NIM + HF for agentic use", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench/AIME/GPQA/MMLU-Pro", "value": "Nemotron 3 Super (per NVIDIA tech report, base anchor): SWE-Bench Verified 60.47, AIME 2025 90.21, GPQA (no tools) 79.23, MMLU-Pro 83.73; ~2.2x inference throughput vs GPT-OSS-120B", "confidence": "high", "note": "Frontier-tier among open instruct models on the Super SKU; calibrated to the base_pretrained Nemotron 3 anchor (C5); SWE-bench Verified 60.47 confirmed as highest open-weight score via NVIDIA technical report + developer blog. Nano is much smaller and would score lower, but the family's headline SKU defines the score.", "sources": [ { "url": "https://research.nvidia.com/labs/nemotron/Nemotron-3/", "shows": "Nemotron 3 reasoning/agentic benchmark positioning vs GPT-OSS-20B/Qwen3", "accessed": "2026-06-04" }, { "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "shows": "NVIDIA Nemotron 3 Super Technical Report (Apr 2026) with SWE-bench Verified 60.47 and full benchmark suite", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "Post-trained/instruct (SFT+RL) variants of the Nemotron 3 family: Nano (~3.2B active hybrid MoE, released Dec 2025) and Super (120B hybrid Mamba-Transformer MoE / 12B active, 1M context, released 11 Mar 2026 at GTC). Designed for agentic/conversational use. Verified live via NVIDIA Nemotron research page June 2026 (instruct HF repo gated/401 at scoring). Base of this family is the base_pretrained anchor." }, { "product": "Hermes-4.3-36B", "org": "Nous Research", "type": "model", "description": "Hybrid-mode reasoning model built on ByteDance Seed-OSS-36B-Base. Nous frames it as roughly matching Hermes-4-70B performance at half the size; the public Hugging Face checkpoint has 6.4K cumulative downloads and 209 likes, making it the current flagship 36B release.", "openness": { "score": 3, "class": "open_weights", "confidence": "high", "components": "weights:open(Apache-2.0);base:Seed-OSS-36B-Base;post-training-data:closed(proprietary ~5M-sample corpus);code:partial(inference only);license:Apache-2.0(OSI)", "note": "Permissive Apache-2.0 weights but undisclosed alignment/post-training data => open_weights, not open_source (no reproducible pipeline).", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4.3-36B", "shows": "Apache-2.0 license, Seed-OSS-36B base, proprietary post-training corpus not released", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "~26.2k HF downloads last month (the most recent flagship Hermes SKU); plus GGUF variant ~5k/mo. Early-adopter scale, well below mass-market open families.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4.3-36B", "shows": "~26,190 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:AIME/MMLU-Pro/MATH", "value": "AIME'24 71.9, MATH-500 93.8, MMLU 87.7, MMLU-Pro 80.7, BBH 86.4 (model card)", "confidence": "medium", "note": "Strong mid-tier instruct/reasoning model; competitive on knowledge/math for its size but below 2026 frontier open instruct models (DeepSeek-V4 / Qwen 3.6 / Kimi K2.6) on agentic coding. Vendor-reported card figures.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4.3-36B", "shows": "AIME24 71.9, MMLU-Pro 80.7, MATH-500 93.8 benchmark table", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "Hybrid-mode reasoning instruct model by Nous Research; post-trained on ByteDance Seed-OSS-36B-Base. ~5M post-training samples / ~60B tokens (SFT + reasoning-trace synthesis). Updated Dec 6 2025; verified live on HF June 2026. License Apache-2.0; post-training data NOT released." }, { "product": "Hermes-4-Llama-3.1-405B", "org": "Nous Research", "type": "model", "description": "Frontier hybrid-mode reasoning model built on Meta-Llama-3.1-405B. It is the largest Hermes 4 variant in the public release set, with 1.1K cumulative downloads and 269 likes on Hugging Face.", "openness": { "score": 2, "class": "restricted", "confidence": "high", "components": "weights:open(downloadable);base:Llama-3.1-405B;post-training-data:closed;code:partial(inference);license:Llama-3.1-Community(non-OSI: 700M MAU commercial cap + acceptable-use + naming requirements inherited from base)", "note": "Inherits Meta's non-OSI Llama 3.1 Community License (MAU cap + field-of-use restrictions) => restricted, not open. Marketed as 'open' on the model card despite the restricted license.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-405B-FP8", "shows": "License: Llama3, base Meta-Llama/Llama-3.1-405B", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "high", "note": "FP8 variant ~381 HF downloads/mo; BF16 405B comparably low. 405B size limits self-hosting; some managed serving (OpenRouter, Nous Portal, Chutes, Nebius) but no large verified usage figure. Hobbyist/specialist scale on download signal.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-405B-FP8", "shows": "381 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:AIME/MMLU-Pro/LiveCodeBench", "value": "MATH-500 96.2, AIME'24 81.9, AIME'25 78.1, GPQA Diamond 70.6, MMLU-Pro 80.6, LiveCodeBench 61.3, BBH 86.3 (tech report / OpenRouter)", "confidence": "medium", "note": "Strongest Hermes 4 SKU; near-frontier on math/knowledge for an open instruct model, though coding (LiveCodeBench 61.3) trails the 2026 frontier coders. Not a category-defining 5 (those are DeepSeek-V4/Qwen3.6/Kimi-class). Vendor/tech-report figures, corroborated by OpenRouter.", "sources": [ { "url": "https://arxiv.org/pdf/2508.18255", "shows": "Hermes 4 technical report with comprehensive benchmark evaluation across math/code/knowledge", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/nousresearch/hermes-4-405b", "shows": "Hermes 4 405B benchmark figures (AIME, MMLU-Pro, LiveCodeBench)", "accessed": "2026-06-04" } ] }, "maturity": 3.1, "mature": false, "version_note": "Hermes 4, hybrid-mode reasoning instruct model post-trained on Meta-Llama-3.1-405B (406B params). Released Sep 2 2025; single-checkpoint reason/non-reason toggle. Tech report arXiv:2508.18255. Verified live on HF June 2026. License = Llama3 (Llama 3.1 Community License, non-OSI, use-restricted)." }, { "product": "Hermes-4-Llama-3.1-70B", "org": "Nous Research", "type": "model", "description": "Smaller Hermes 4 hybrid-mode reasoning model built on Meta-Llama-3.1-70B. The public checkpoint has 1.9K cumulative downloads and 126 likes, positioning it as the mid-size option in the Hermes 4 family.", "openness": { "score": 2, "class": "restricted", "confidence": "high", "components": "weights:open(downloadable);base:Llama-3.1-70B;post-training-data:closed;code:partial(inference);license:Llama-3.1-Community(non-OSI: 700M MAU cap + acceptable-use)", "note": "Same non-OSI Llama 3.1 Community License as the 405B => restricted. Marketed as 'open' despite the restricted license.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-70B", "shows": "License: Llama3, base meta-llama/Llama-3.1-70B", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "high", "note": "~1,055 HF downloads/mo for the BF16 70B; modest even with FP8/GGUF variants. Hobbyist/specialist scale on download signal.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-70B", "shows": "1,055 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:AIME/MMLU-Pro", "value": "Hermes 4 family (tech report arXiv:2508.18255); 70B reasoning model on Llama-3.1-70B base, below the 405B SKU on math/knowledge/coding", "confidence": "low", "note": "Per-SKU 70B benchmark numbers not isolated in fetched sources; scored a 3 by interpolation from the 405B (C4) down a tier given the smaller Llama-3.1-70B base. Mid-tier open instruct, below 2026 frontier. Low confidence on the exact figure.", "sources": [ { "url": "https://arxiv.org/pdf/2508.18255", "shows": "Hermes 4 technical report covering 70B/405B/14B variants", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "Hermes 4 hybrid-mode reasoning instruct model post-trained on Meta-Llama-3.1-70B (71B params). Released Sep 2 2025; tech report arXiv:2508.18255. Verified live on HF June 2026. License = Llama3 (Llama 3.1 Community License, non-OSI, restricted)." }, { "product": "Hermes-4-14B", "org": "Nous Research", "type": "model", "description": "Small dense Hermes variant for local inference and low-cost deployment. The model card shows 2.8K cumulative downloads and 146 likes, making it the most lightweight Hermes 4 checkpoint in the public lineup.", "openness": { "score": 3, "class": "open_weights", "confidence": "high", "components": "weights:open(Apache-2.0);base:Qwen3-14B;post-training-data:closed(proprietary ~5M-sample corpus);code:partial(inference);license:Apache-2.0(OSI)", "note": "Apache-2.0 weights (Qwen3-14B base is also Apache-2.0) but undisclosed post-training data => open_weights, not open_source.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-14B", "shows": "License Apache-2.0, base Qwen/Qwen3-14B", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "~35,066 HF downloads/mo, the most-downloaded Hermes 4 SKU (small size + permissive license drive uptake). Early-adopter scale.", "sources": [ { "url": "https://huggingface.co/NousResearch/Hermes-4-14B", "shows": "35,066 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:AIME/MMLU-Pro", "value": "Hermes 4 family (tech report arXiv:2508.18255); 14B reasoning model on Qwen3-14B base", "confidence": "low", "note": "Per-SKU 14B numbers not isolated in fetched sources; strong-for-size reasoning instruct on a capable Qwen3-14B base, but a 14B in 2026 is mid-tier vs frontier open instruct models. Scored 3 by family interpolation; low confidence on exact figure.", "sources": [ { "url": "https://arxiv.org/pdf/2508.18255", "shows": "Hermes 4 technical report covering the 14B variant", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "Hermes 4 hybrid-mode reasoning instruct model post-trained on Qwen3-14B (14B params). Updated Jan 9 (2026); tech report arXiv:2508.18255. Verified live on HF June 2026. License Apache-2.0; post-training data NOT released." }, { "product": "TinyLlama-1.1B-Chat-v1.0", "org": "TinyLlama", "type": "model", "description": "A 1.1B instruction-tuned chat model from the TinyLlama project, built on the Llama architecture and trained on 3T tokens for ~90 days on 16 A100-40G GPUs. It became the canonical sub-2B open chat baseline for edge and local-inference research and still pulls ~2.5M Hugging Face downloads per month two years after release.", "openness": { "score": 4, "class": "open_weights", "confidence": "high", "components": "weights:open(Apache-2.0);base:TinyLlama-1.1B(open, 3T-token pretrain documented);post-training:SFT on UltraChat + DPO on UltraFeedback(both public datasets);code:open(pretraining project public);license:Apache-2.0(OSI)", "note": "Unusually transparent for a chat model: open base, and named public alignment datasets (UltraChat SFT + UltraFeedback DPO). Scored 4 open_weights, just short of full open_source since the base pretraining data is documented but the end-to-end reproducible pipeline is not packaged as one release. Strongest openness in this batch alongside Nemotron.", "sources": [ { "url": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0", "shows": "Apache-2.0; base TinyLlama-1.1B; SFT on UltraChat + DPO on UltraFeedback", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~2.31M HF downloads last month, a workhorse tiny-chat model for edge/testing/CI and educational use. Solidly in the 1-10M monthly-download band.", "sources": [ { "url": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0", "shows": "2,314,508 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 1, "basis": "benchmark:MMLU", "value": "1.1B Llama-2-era model; intended for lightweight/edge use, not competitive on 2026 capability benchmarks", "confidence": "high", "note": "Intentionally tiny (1.1B, 2023 Llama-2 architecture). Its value is footprint and openness, not capability; scored 1 against a 2026 frontier comparison set. Not built to compete on reasoning/coding suites.", "sources": [ { "url": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0", "shows": "1.1B params, Llama 2 architecture, chat-optimized lightweight model", "accessed": "2026-06-04" } ] }, "maturity": 1.9, "mature": false, "version_note": "1.1B chat model (Llama 2 architecture), DPO-aligned finetune of TinyLlama-1.1B (pretrained on 3T tokens). Released 2023/early-2024; v1.0. License Apache-2.0. Verified live on HF June 2026; foundational small-chat model still heavily pulled." }, { "product": "Command R+", "org": "Cohere", "type": "model", "description": "Cohere's enterprise foundation model line, currently anchored by Command A+ (released May 20, 2026), a 218B-parameter sparse MoE (25B active) with native citation generation, W4A4 lossless quantization, and the company's first fully Apache 2.0 licensed frontier model. Runs on a single B200 or two H100s. Positioned for sovereign AI and enterprise RAG with native support for 48 languages and explicit grounding spans linking every factual claim to source documents. Earlier Command R+ (104B, 2024) is the predecessor; with Command A+ the line has flipped from API-only to open-weights.", "openness": { "score": 2, "class": "restricted", "confidence": "high", "components": "weights:open(gated, contact-info required);data:closed;code:closed;license:CC-BY-NC(non-commercial, non-OSI)+Cohere-Acceptable-Use-Policy", "note": "Weights downloadable but under CC-BY-NC, a non-commercial, non-OSI license with field-of-use restrictions => restricted. Marketed as 'open weights' despite the non-commercial cap.", "sources": [ { "url": "https://huggingface.co/CohereLabs/c4ai-command-r-plus", "shows": "License CC-BY-NC + Acceptable Use Policy, gated access, 104B instruct model", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "high", "note": "~3,258 HF downloads/mo on the original checkpoint (gated + superseded by 08-2024 revision + newer Command models). Non-commercial license and 104B size cap self-host adoption; some legacy use via Cohere API. Hobbyist/legacy scale on the download signal.", "sources": [ { "url": "https://huggingface.co/CohereLabs/c4ai-command-r-plus", "shows": "3,258 downloads/month, gated, newer revision available", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "benchmark:MMLU", "value": "Open LLM Leaderboard avg 74.6 (MMLU 75.7, ARC-C 70.99, HellaSwag 88.6, GSM8K 70.7, TruthfulQA 56.3), strong vs 2024 peers (DBRX, Mixtral 8x7B)", "confidence": "medium", "note": "A 2024-era frontier-ish chat/RAG model, now well below the 2026 open instruct frontier (no modern reasoning/agentic coding scores; legacy MMLU-class benchmarks only). Scored 2 against a 2026 comparison set despite being capable for its time.", "sources": [ { "url": "https://huggingface.co/CohereLabs/c4ai-command-r-plus", "shows": "Open LLM Leaderboard avg 74.6 with per-benchmark breakdown", "accessed": "2026-06-04" } ] }, "maturity": 1.7, "mature": false, "version_note": "Cohere c4ai-command-r-plus: 104B instruct/chat model, 128K context, RAG + multi-step tool use, 10 languages. Original release Apr 2024 (newer 08-2024 revision exists). Verified live on HF June 2026 (gated, contact-info required). License CC-BY-NC (non-commercial, non-OSI). Correctly an instruct model, belongs in finetuned_chat (was miscategorized as base elsewhere)." }, { "product": "Jamba 1.5 Large", "org": "AI21 Labs", "type": "model", "description": "AI21-Jamba-1.5-Large (ai21labs/AI21-Jamba-1.5-Large on HF), hybrid SSM-Transformer (Mamba+Attention) MoE, 398B total / 94B active, 256K context, instruction-tuned chat model with tool use / grounded-RAG / JSON mode. Announced 22 Aug 2024 under the Jamba Open Model License. Verified live on HF June 2026, but stale: superseded by Jamba 1.6 (Mar 2025) and the Jamba2/Jamba Reasoning 3B generation now headlined on ai21.com/jamba. Recategorized from base_pretrained -> finetuned_chat: this is the instruct/chat SKU (post-trained for downstream chat/agentic use), not a raw base model.", "openness": { "score": 2, "class": "restricted", "components": "weights:open(downloadable on HF, contact-info gate);data:closed;code:partial(inference/usage only, no training pipeline);license:Jamba-Open-Model-License(non-OSI: $50M annual-revenue commercial cap + AUP + naming/attribution requirements)", "confidence": "high", "note": "Weights downloadable but the 'Jamba Open Model License' is non-OSI with a $50M revenue cap that bars large-entity commercial use -> restricted, not open_source, despite the 'Open Model' branding.", "sources": [ { "url": "https://huggingface.co/ai21labs/AI21-Jamba-1.5-Large", "shows": "Jamba Open Model License + downloadable Safetensors weights, instruction-following model card", "accessed": "2026-06-04" }, { "url": "https://www.ai21.com/licenses/jamba-open-model-license", "shows": "non-OSI license text: $50M annual-revenue commercial cap, AUP, naming/attribution clauses", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "usage_volume", "confidence": "high", "note": "HF model card reports ~1,072 downloads in the last month at scoring time -- a thin usage signal consistent with a dated (Aug 2024) model superseded by Jamba 1.6/Jamba2. No large-scale provider/app-surface adoption found.", "sources": [ { "url": "https://huggingface.co/ai21labs/AI21-Jamba-1.5-Large", "shows": "~1,072 downloads last month on the official model card", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:Arena-Hard/MMLU", "value": "Arena Hard 65.4 (AI21-reported, outpacing Llama 3.1 70B and 405B at launch); MMLU (CoT) 81.2; GSM-8K 87; RULER 256K effective context (93.9% at full length)", "confidence": "medium", "note": "Strong long-context instruct model for its Aug-2024 vintage, but mid-tier on a 2026 capability axis where the comparison set is current frontier open models (DeepSeek-V4, Qwen 3.6, GLM-5.1). Vendor-reported figures.", "sources": [ { "url": "https://www.ai21.com/blog/announcing-jamba-model-family/", "shows": "Arena Hard 65.4 for Jamba 1.5 Large, beats Llama 3.1 70B/405B; Jamba Open Model License", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/ai21labs/AI21-Jamba-1.5-Large", "shows": "MMLU 81.2, GSM-8K 87, 256K RULER context on model card", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "AI21-Jamba-1.5-Large (ai21labs/AI21-Jamba-1.5-Large on HF), hybrid SSM-Transformer (Mamba+Attention) MoE, 398B total / 94B active, 256K context, instruction-tuned chat model with tool use / grounded-RAG / JSON mode. Announced 22 Aug 2024 under the Jamba Open Model License. Verified live on HF June 2026, but stale: superseded by Jamba 1.6 (Mar 2025) and the Jamba2/Jamba Reasoning 3B generation now headlined on ai21.com/jamba. Recategorized from base_pretrained -> finetuned_chat: this is the instruct/chat SKU (post-trained for downstream chat/agentic use), not a raw base model." }, { "product": "GLM-4.6", "org": "Zhipu AI", "type": "model", "description": "Zhipu AI's current frontier model, released September 30, 2025, a 355B-parameter MoE with MIT-licensed open weights, a 200K context window, and published CC-Bench evaluation trajectories on Hugging Face for independent verification. Builds on GLM-4.5 (July 2025) and represents Zhipu's pivot to a fully open-weights frontier strategy. Z.ai (Zhipu's international brand) has also debuted GLM-4.6V, a native tool-calling vision variant. Deployable locally via vLLM or SGLang. Marks a sharp contrast with the prior closed-source GLM-4 positioning and one of the cleanest signals that Chinese frontier models have moved decisively open in 2025-2026.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(MIT, on HF, Safetensors);data:closed;code:partial(inference/usage + published eval questions & agent trajectories, no training pipeline or post-training data);license:MIT(OSI permissive, no use restrictions)", "confidence": "high", "note": "MIT-licensed downloadable weights with no use restrictions, but training data and post-training/alignment recipes are closed -> open_weights, not full open_source.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-4.6", "shows": "MIT license, downloadable MoE weights, 200K context, chat-template/instruct model card", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "HF model card reports ~17,472 downloads/month plus 103 Spaces and 40+ quantized derivatives; widely served beyond HF (Z.ai API, OpenRouter, vLLM) as a top open coding/agentic model. Direct HF download count alone is modest; level 3 reflects multi-provider production distribution rather than a single >1M usage figure.", "sources": [ { "url": "https://huggingface.co/zai-org/GLM-4.6", "shows": "~17,472 downloads/month, 103 Spaces, 40+ quantized versions, 1.23k likes", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:Artificial-Analysis-Intelligence-Index", "value": "Artificial Analysis Intelligence Index ranking #15/43 (non-reasoning); evaluated on GPQA Diamond, SWE-bench Verified, Terminal-Bench Hard, AIME 25, LiveCodeBench v6, HLE; Z.ai docs report parity with Claude Sonnet 4 on several leaderboards and surpassing Sonnet 4 on 74 real-world Claude Code coding tasks", "confidence": "medium", "note": "Near-frontier open coding/agentic model -- strong but a notch below the top open-weights leaders (GLM-5.1, DeepSeek-V4, Qwen 3.6) and the closed frontier. Capability 4, not 5: it is the prior-generation Z.ai coder superseded by GLM-5/5.1. AA index is independent; the Claude-Sonnet-4 parity claim is vendor-reported.", "sources": [ { "url": "https://artificialanalysis.ai/models/glm-4-6", "shows": "GLM-4.6 Intelligence Index rank #15/43, evaluated across GPQA/SWE-bench Verified/Terminal-Bench/AIME/LCB", "accessed": "2026-06-04" }, { "url": "https://docs.z.ai/guides/llm/glm-4.6", "shows": "8-benchmark eval (AIME 25, GPQA, LCB v6, HLE, SWE-Bench Verified); parity with Claude Sonnet 4/4.6 on several leaderboards", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "GLM-4.6 (zai-org/GLM-4.6 on HF) by Z.ai (Zhipu AI), GLM4_MoE ~357B-param MoE, 200K context (up from 128K in GLM-4.5), instruction/chat + reasoning + agentic tool-use tuned. Released late 2025 (post-GLM-4.5 which has arXiv 2508.06471). Verified live on HF June 2026 (succeeded by GLM-5/GLM-5.1, the latter a flagship anchor). Recategorized from base_pretrained -> finetuned_chat per pilot litmus: GLM-4.6 is an instruct/agentic chat SKU, not a raw base model." }, { "product": "CodeGemma", "org": "Google", "type": "model", "description": "Google's code-specialized Gemma family: a 2B pretrained model for fast code completion (fill-in-the-middle), a 7B pretrained model for infilling + natural language, and a 7B instruction-tuned variant for code chat. Further-trained on 500B-1T tokens of public code, math, and synthetic data.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open;data:closed;code:closed;license:Gemma-Terms-of-Use(custom,use-restricted,non-OSI)", "confidence": "high", "note": "Open weights under Google's custom, use-restricted Gemma license; no open training data or code.", "sources": [ { "url": "https://huggingface.co/google/codegemma-7b-it", "shows": "model card: Gemma license, sizes, intended use", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "signal_type": "usage_volume", "confidence": "medium", "note": "Modest current usage (~2k monthly downloads on codegemma-7b-it); largely superseded by newer coding models.", "sources": [ { "url": "https://huggingface.co/google/codegemma-7b-it", "shows": "~1,929 downloads last month", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "benchmark:HumanEval/MBPP", "value": "competent FIM/code completion at 2B/7B circa 2024", "confidence": "medium", "note": "Below current coding peers (Qwen2.5-Coder, StarCoder2, Granite Code) on benchmarks.", "sources": [ { "url": "https://github.com/huggingface/blog/blob/main/codegemma.md", "shows": "2B/7B variants, FIM training, benchmarks", "accessed": "2026-06-17" } ] }, "maturity": 2.7, "mature": false, "version_note": "Gemma Terms of Use (custom, use-restricted, NOT OSI). Training data and recipe not released. Largely superseded by Qwen2.5-Coder / Gemma 3 for coding." }, { "product": "Gemini 3 Pro", "org": "Google", "type": "model", "description": "Google DeepMind's flagship reasoning model in the Gemini 3 family (successor to Gemini 2.5), launched in preview Nov 2025. Natively multimodal (text, image, audio, video, PDFs, code) with a 1M-token context and thinking_level reasoning control; aimed at agentic building, coding, and multi-step planning. Proprietary, API-only; Google discloses no parameter count.", "openness": { "score": 1, "class": "closed", "components": "weights:closed;data:closed;code:closed;license:proprietary(Google Gemini API/Vertex terms);API-only", "confidence": "high", "note": "No public weights; closed API-only model.", "sources": [ { "url": "https://blog.google/products/gemini/gemini-3/", "shows": "official launch: 1M context, benchmarks, API-only", "accessed": "2026-06-18" } ], "bucket": "closed" }, "adoption": { "level": 5, "signal_type": "reported_traction", "confidence": "low", "note": "Frontier proprietary model; Google cites 650M+ Gemini app MAUs (generational, not model-specific); not on HF, no download numbers.", "sources": [ { "url": "https://blog.google/products/gemini/gemini-3/", "shows": "Gemini 3 Pro availability across Gemini app / API / Vertex", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "benchmark:LMArena/GPQA-Diamond/HLE", "value": "GPQA Diamond 91.9%, Humanity's Last Exam 37.5%, MMMU-Pro 81%; natively multimodal, 1M context; LMArena frontier (rank vendor/mirror-reported)", "confidence": "medium", "note": "Frontier proprietary model; benchmark figures vendor-published, Arena rank from third-party mirrors.", "sources": [ { "url": "https://blog.google/products/gemini/gemini-3/", "shows": "published GPQA/HLE/MMMU-Pro benchmarks", "accessed": "2026-06-18" } ] }, "maturity": 5.0, "mature": false, "version_note": "Proprietary/API-only (Gemini API + Vertex AI). On LMArena's frontier (Google claimed #1 / ~1501 Elo at launch; ~#8 mid-2026 per third-party mirrors). Note the original gemini-3-pro-preview endpoint was reportedly superseded by gemini-3.1-pro (~Mar 2026)." } ] }, "inference_code": { "label": "Inference code", "arc": "Model components", "layer": "model_components", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "llama.cpp", "org": "ggml-org (Georgi Gerganov)", "type": "software", "description": "CPU and GPU inference engine for LLMs written in portable C/C++. Pioneered the GGUF quantization format (2-8 bit) that became the ecosystem standard for local model distribution. Runs on consumer hardware (laptops, phones, Raspberry Pi) without requiring CUDA or Python. 112K+ GitHub stars make it the most-starred inference project; foundational infrastructure that powers Ollama, LM Studio, GPT4All, and dozens of other tools.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;governance:ggml-org/Hugging Face(community);no feature-gated core", "confidence": "high", "note": "Fully MIT-licensed C/C++ inference engine; entire source public, no proprietary tier.", "sources": [ { "url": "https://github.com/ggml-org/llama.cpp/blob/master/LICENSE", "shows": "MIT License text, (c) 2023-2026 The ggml authors", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "De facto standard local/edge inference engine: GGUF (its native format) is >60% of all quantized models on HF, and llama.cpp powers Ollama, LM Studio, GPT4All, Jan.ai, and HF Inference Endpoints, which collectively serve millions. 115k GitHub stars (fastest OSS-AI project to 100k, Mar 2026) corroborates but is not the basis. No clean PyPI/user count for the C++ engine itself, so usage inferred from ecosystem dominance; placed at 4 (1-10M-equivalent).", "sources": [ { "url": "https://github.com/ggml-org/llama.cpp", "shows": "115k stars; README lists Ollama/LM Studio/LocalAI/HF Inference Endpoints as downstream integrations", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/docs/hub/gguf-llamacpp", "shows": "GGUF/llama.cpp as the standard local-inference path on the HF Hub", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Backends: Metal/CUDA/HIP/Vulkan/SYCL/OpenVINO/MUSA/WebGPU; quantization 1.5/2/3/4/5/6/8-bit integer; 100+ model architectures incl. multimodal (LLaVA); continuous batching + server. No standardized MLPerf Inference submission for the engine itself.", "confidence": "high", "note": "Frontier for portable/CPU/consumer-hardware inference and quantization breadth, but not the datacenter-GPU throughput frontier held by vLLM/SGLang/TensorRT-LLM; scored 4 on a category where C5 is reserved for the throughput-defining server engines.", "sources": [ { "url": "https://github.com/ggml-org/llama.cpp", "shows": "README documents backend list, 1.5-8bit quantization, 100+ architectures, multimodal support", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Active 2026; latest release tag b9518 (June 4, 2026, GitHub). ggml.ai team joined Hugging Face full-time Feb 20, 2026; project stays open. Confirmed live June 2026." }, { "product": "Text Generation Inference", "org": "Hugging Face", "type": "software", "description": "Hugging Face's production LLM serving solution, purpose-built for the HF model ecosystem. Supports continuous batching, tensor parallelism, Flash Attention, and quantization (GPTQ, AWQ, EETQ). Powers Hugging Face Inference Endpoints and is the default backend when deploying models from the Hub. Tight integration with the transformers library makes it the path of least resistance for HF users going to production.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;note:was briefly under restrictive HFOIL license in 2023, reverted to Apache-2.0; current LICENSE confirmed Apache-2.0", "confidence": "high", "note": "Current LICENSE is standard Apache-2.0 (not HFOIL); fully open source despite being archived.", "sources": [ { "url": "https://github.com/huggingface/text-generation-inference/blob/main/LICENSE", "shows": "Apache License 2.0 text, Copyright 2022 Hugging Face", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Ran in production at HF (Hugging Chat, Inference API, Inference Endpoints) and was widely deployed 2023-2025, but the repo is now archived/maintenance-mode with HF steering users to vLLM/SGLang, so adoption is declining. 10.9k stars (secondary). Level 3 reflects a real but shrinking, sunsetting footprint.", "sources": [ { "url": "https://github.com/huggingface/text-generation-inference", "shows": "README: 'Used in production at Hugging Face to power Hugging Chat, the Inference API and Inference Endpoints'; archived Mar 21 2026, maintenance mode, recommends vLLM/SGLang", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Continuous batching, tensor parallelism, quantization (bitsandbytes/GPTQ/AWQ), distributed tracing (OpenTelemetry) + Prometheus metrics, OpenAI-compatible API. No active MLPerf submission; feature set frozen at v3.3.7.", "confidence": "medium", "note": "Was a capable production server, but development has stopped (archived); its feature set is frozen while vLLM/SGLang continue advancing, so it now sits mid-pack rather than at the C5 frontier.", "sources": [ { "url": "https://github.com/huggingface/text-generation-inference", "shows": "README documents continuous batching, tensor parallelism, OpenTelemetry tracing, Prometheus metrics, production-ready feature set", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Latest release v3.3.7 (Dec 19, 2025). Repo ARCHIVED Mar 21, 2026 and placed in maintenance mode; HF now recommends vLLM/SGLang/llama.cpp/MLX going forward. Verified live (archived) June 2026." }, { "product": "ONNX Runtime", "org": "Microsoft", "type": "software", "description": "Cross-platform inference accelerator supporting models from PyTorch, TensorFlow, and other frameworks via the ONNX interchange format. Runs on CPUs, GPUs, NPUs, and custom accelerators with hardware-specific execution providers. Used in production across Microsoft products (Windows, Office, Azure). Unique differentiator: hardware-agnostic: one runtime targets NVIDIA, AMD, Intel, Qualcomm, and Apple silicon. 20K+ stars, steady development over 7+ years.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(Microsoft);no proprietary core; plugin execution providers (CUDA/WebGPU/etc.) in-repo", "confidence": "high", "note": "Fully MIT-licensed open source from Microsoft; no feature-gated tier.", "sources": [ { "url": "https://github.com/microsoft/onnxruntime/blob/main/LICENSE", "shows": "MIT License text, Microsoft Corporation", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "~74.7M PyPI downloads in the last month for the onnxruntime package; GitHub reports 'used by' ~89.9k repositories. Embedded across Microsoft products and the broader ML deployment stack. Well into the >10M band on download volume. (Note: onnxruntime spans general ML inference, not only LLM serving; see capability/recategorize note.)", "sources": [ { "url": "https://pypistats.org/packages/onnxruntime", "shows": "~74.7M downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/microsoft/onnxruntime", "shows": "20.7k stars; 'used by 89.9k'; cross-platform inference accelerator description", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:MLPerf", "value": "Appears in MLPerf Inference v5.0 submissions (ResNet50, RetinaNet, offline; multiple ORT versions v1.20.1-v1.23.2). Execution providers across CUDA/WebGPU/CPU/TensorRT/OpenVINO/CoreML; cross-framework (PyTorch/TF/scikit-learn/XGBoost). Quantization + graph optimization.", "confidence": "medium", "note": "Strong, MLPerf-present general-purpose inference accelerator, but its MLPerf coverage is vision/classical models, not LLM-serving throughput; for transformer/LLM serving it trails the purpose-built vLLM/SGLang/TensorRT-LLM (C5) frontier. Scored 4 within this category. Arguably broader than inference_code (a general ML runtime), but it does load weights and serve predictions, so it fits the litmus test; flagging for reviewer consideration without forcing a move.", "sources": [ { "url": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0", "shows": "ONNX Runtime (v1.20.1-v1.23.2) submissions for ResNet50/RetinaNet offline", "accessed": "2026-06-04" }, { "url": "https://github.com/microsoft/onnxruntime", "shows": "cross-platform inference accelerator, plugin execution providers (CUDA/WebGPU)", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "Latest release v1.26.0 (May 8, 2026, GitHub). Cross-platform inference + training accelerator for ONNX models (PyTorch/TF export + classical ML). Confirmed live June 2026." }, { "product": "LMDeploy", "org": "Shanghai AI Laboratory", "type": "software", "description": "LMDeploy v0.13.0 (released May 12, 2026, GitHub); Apache-2.0 OSS inference/serving toolkit (TurboMind + PyTorch engines). Confirmed live on GitHub + PyPI June 2026.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(InternLM/lmdeploy);no-feature-gated-core;quantization:weight-only+kv(AWQ/W4A16)", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), source public, no proprietary core; broad model coverage (Llama/InternLM/Qwen/DeepSeek/Mixtral + VLMs).", "sources": [ { "url": "https://github.com/InternLM/lmdeploy", "shows": "Apache-2.0 license, v0.13.0 (May 12 2026), weight-only+kv quantization, Llama/InternLM/Qwen/DeepSeek/Mixtral + VLM support", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "~165K PyPI downloads last month (pypistats); 352 GitHub dependents. Real usage volume in the 100K-1M band. Smaller footprint than vLLM/SGLang anchors (millions/mo) but well beyond hobbyist; especially used in the InternLM/Chinese-model ecosystem. 7.9k stars corroborate only.", "sources": [ { "url": "https://pypistats.org/packages/lmdeploy", "shows": "~165,001 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/InternLM/lmdeploy", "shows": "352 dependent projects, 7.9k stars", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "TurboMind + PyTorch engines; persistent/continuous batching, blocked KV cache, weight-only + kv (AWQ/W4A16) quantization, tensor parallelism, broad LLM + VLM architecture coverage; vendor-reported 4-bit inference ~2.4x FP16", "confidence": "medium", "note": "Strong, full-featured GPU inference engine (continuous batching, quantization, TP, VLM support). No standardized MLPerf submission confirmed, so feature_matrix not benchmark; a notch below the vLLM/SGLang/TensorRT-LLM C5 anchors on throughput-leadership/ecosystem breadth.", "sources": [ { "url": "https://github.com/InternLM/lmdeploy", "shows": "TurboMind/PyTorch engines, continuous batching, AWQ/kv quantization, TP, VLM support, 2.4x 4-bit claim", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "LMDeploy v0.13.0 (released May 12, 2026, GitHub); Apache-2.0 OSS inference/serving toolkit (TurboMind + PyTorch engines). Confirmed live on GitHub + PyPI June 2026." }, { "product": "llamafile", "org": "Mozilla AI", "type": "software", "description": "llamafile v0.10.3 (released June 2, 2026, GitHub); Apache-2.0 (MIT for llama.cpp/whisper.cpp changes). Single-file cross-platform LLM executable (llama.cpp + Cosmopolitan Libc). Mozilla Builders initiative, actively maintained June 2026.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI)+MIT(OSI, for llama.cpp/whisper.cpp deltas);source:public(Mozilla-Ocho/llamafile);no-feature-gated-core", "confidence": "high", "note": "Fully OSI-licensed, source public, single-file llama.cpp+Cosmopolitan distribution; no proprietary tier.", "sources": [ { "url": "https://github.com/Mozilla-Ocho/llamafile", "shows": "Apache-2.0 (+ MIT deltas) license, v0.10.3 (June 2 2026), single-file executable, 24.7k stars, actively maintained", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No PyPI/download or pull-count signal available (distributed as single-file binaries, not a package); 24.7k GitHub stars + Mozilla backing is the only honest signal. Capped at 3 per stars_fallback rule. Popular local-inference convenience tool but no verified usage_volume figure.", "sources": [ { "url": "https://github.com/Mozilla-Ocho/llamafile", "shows": "24.7k stars, 40 releases, Mozilla Builders initiative", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Single-file cross-platform (Cosmopolitan Libc) llama.cpp-based local inference; CPU+GPU, GGUF models, OpenAI-compatible server, whisperfile STT; portability is the differentiator, not peak throughput", "confidence": "medium", "note": "Optimizes for portability/zero-install local inference rather than datacenter throughput; built on llama.cpp so no continuous-batching/TP server scaling like vLLM. Mid-tier on a capability axis whose frontier (C5) is the high-throughput serving engines.", "sources": [ { "url": "https://github.com/Mozilla-Ocho/llamafile", "shows": "single-file llama.cpp + Cosmopolitan Libc, cross-OS, whisperfile STT", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "llamafile v0.10.3 (released June 2, 2026, GitHub); Apache-2.0 (MIT for llama.cpp/whisper.cpp changes). Single-file cross-platform LLM executable (llama.cpp + Cosmopolitan Libc). Mozilla Builders initiative, actively maintained June 2026." }, { "product": "MAX", "org": "Modular", "type": "software", "description": "Unified inference engine built on the Mojo programming language that targets multiple hardware backends (NVIDIA, AMD, Intel, Apple) from a single codebase. Includes a graph compiler that automatically optimizes model execution for the target hardware. Supports PyTorch and Hugging Face models with claimed performance matching or exceeding TensorRT-LLM on NVIDIA GPUs while also running on non-NVIDIA hardware. Founded by former LLVM/Swift creator Chris Lattner. 26K+ stars on the combined Modular repo.", "openness": { "score": 3, "class": "source_available", "components": "license:Modular-Community-License(non-OSI; Apache-2.0-w/-LLVM-exceptions only for some components);source:public(~450k LOC incl MAX kernels + inference server + Mojo stdlib);commercial-tier:Modular enterprise/managed deployment", "confidence": "high", "note": "Large public codebase but MAX/Mojo usage and distribution are governed by the non-OSI Modular Community License (not OSI-approved), plus a commercial tier -> source_available, not open_source. Marketed as 'open'; flag open_washed (non-OSI license presented as open).", "sources": [ { "url": "https://github.com/modular/modular", "shows": "Modular Community License (non-OSI) governs MAX/Mojo usage+distribution; ~450k LOC public; MAX 26.3 / Mojo 1.0.0b1 (May 7 2026)", "accessed": "2026-06-04" }, { "url": "https://docs.modular.com/max/", "shows": "MAX serves 500+ open models with OpenAI-compatible endpoints across NVIDIA/AMD/Apple GPUs", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "No published download/user/tokens figure for MAX serving specifically. 26.3k GitHub stars on the modular/modular monorepo (covers MAX+Mojo) is the only signal; capped per stars_fallback and placed at 2 given no verified production-deployment count and a still-maturing serving ecosystem vs the established OSS engines.", "sources": [ { "url": "https://github.com/modular/modular", "shows": "26.3k stars on modular/modular (MAX + Mojo)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "500+ open-model coverage out of the box, OpenAI-compatible serving, multi-vendor GPU support (NVIDIA/AMD/Apple) + CPU, custom-kernel authoring in Mojo, cloud deployment; designed as high-performance inference + hardware-abstraction layer", "confidence": "medium", "note": "Broad multi-hardware coverage and a high-performance serving stack with custom-kernel programmability; no standardized MLPerf/AA inference-engine submission verified independently, so feature_matrix. Strong (C4) but not a demonstrated throughput-frontier-definer like the C5 anchors.", "sources": [ { "url": "https://docs.modular.com/max/", "shows": "500+ models, OpenAI-compatible endpoints, NVIDIA/AMD/Apple GPU + CPU, custom GPU kernels", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "MAX 26.3 (stable, released May 7, 2026; Mojo 1.0.0b1). High-performance inference-serving framework (500+ open models, OpenAI-compatible endpoints) on NVIDIA/AMD/Apple GPUs + CPU. Source largely public (github.com/modular/modular) but under the non-OSI Modular Community License." }, { "product": "Groq Inference", "org": "Groq", "type": "software", "description": "Inference service running on custom Language Processing Unit (LPU) hardware designed from the ground up for sequential token generation. Achieves industry-leading token throughput, widely benchmarked at 500-800 tokens/sec for Llama-class models, far exceeding GPU-based alternatives. Deterministic, SRAM-based architecture eliminates memory bottlenecks. Available via API with OpenAI-compatible endpoints. The company that put 'inference speed' in the mainstream conversation in early 2024.", "openness": { "score": 1, "class": "closed", "components": "engine:proprietary(LPU stack);source:closed;access:managed-cloud-API-only;license:Proprietary", "confidence": "high", "note": "Proprietary cloud inference engine + custom LPU silicon, API-only; per recipe, cloud inference engines (Groq) are closed.", "sources": [ { "url": "https://groq.com/", "shows": "GroqCloud proprietary managed inference on LPU hardware, API access via console, OpenAI-compatible", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Vendor reports '3m developers and teams' on GroqCloud; named enterprise users incl. Dropbox, Vercel, Canva, Robinhood, Volkswagen, Workday, Ramp. Developer-count is vendor-self-reported (reported_traction), placing it in the 1-10M band.", "sources": [ { "url": "https://groq.com/", "shows": "'3m developers and teams' use Groq; named customers Dropbox/Vercel/Canva/Robinhood/Workday/Ramp", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:ArtificialAnalysis(provider-speed)", "value": "Groq LPU is among the top tokens/sec providers on the Artificial Analysis provider leaderboard (purpose-built inference silicon); listed among the fastest-class providers for open models, though Cerebras leads on gpt-oss-120b. Speed/latency frontier among inference engines.", "confidence": "medium", "note": "Capability for an inference engine = throughput/latency frontier (the MLPerf analog here is the Artificial Analysis provider-speed leaderboard). Groq LPU is a recognized speed-frontier provider class; C5 alongside the open-engine anchors. NOTE: could not pull Groq's exact t/s figure for gpt-oss-120b on the AA page this run (Groq present but not in the displayed top-5); score leans on its established speed-leadership positioning -> medium confidence.", "sources": [ { "url": "https://artificialanalysis.ai/models/gpt-oss-120b/providers", "shows": "Artificial Analysis provider speed leaderboard for gpt-oss-120b (Groq listed among 22 providers; Cerebras top at 1,776.8 t/s)", "accessed": "2026-06-04" }, { "url": "https://groq.com/", "shows": "LPU purpose-built for inference; customer reports 7.41x chat-speed gain", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": false, "version_note": "GroqCloud proprietary cloud inference on Groq LPU (Language Processing Unit) hardware; OpenAI-compatible API, serves open models (gpt-oss, Llama, etc.). Closed/managed-only service. Confirmed live June 2026 (groq.com)." }, { "product": "Cerebras Inference", "org": "Cerebras Systems", "type": "software", "description": "Inference service running on the Cerebras Wafer-Scale Engine (WSE-3), a single chip containing 4 trillion transistors with 44GB of on-chip SRAM, eliminating off-chip memory bottlenecks entirely. Achieves extremely high throughput on large models by keeping the entire model on a single wafer. Available via API. Positioned as the fastest inference for large models where the entire model fits on one wafer. Raised $4.3B+, filed for IPO.", "openness": { "score": 1, "class": "closed", "components": "engine:proprietary(WSE wafer-scale stack);source:closed;access:managed-cloud-API-only;license:Proprietary", "confidence": "high", "note": "Proprietary cloud inference on custom wafer-scale hardware, API-only; closed per recipe (cloud inference engines).", "sources": [ { "url": "https://www.cerebras.ai/inference", "shows": "proprietary cloud inference on Wafer-Scale Engine, OpenAI-compatible API, serves GLM/gpt-oss/Llama Scout", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "No developer/user count disclosed on the inference page; named users GSK, Perplexity, AlphaSense, Meta (Llama partnership), DeepLearning.AI. Placed at 3 on reported enterprise traction (named-deployment signal) without a verified usage figure; smaller disclosed footprint than Groq's 3M-developer claim.", "sources": [ { "url": "https://www.cerebras.ai/inference", "shows": "named users GSK, Perplexity, AlphaSense, Meta, DeepLearning.AI; no user-count figure", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:ArtificialAnalysis(provider-speed)", "value": "Fastest provider on the Artificial Analysis gpt-oss-120b provider leaderboard at ~1,726 output tokens/sec (next-fastest SambaNova ~692.5 t/s, ~2.5x; Fireworks ~658.8 t/s 3rd); vendor claims up to 15x faster than NVIDIA GPUs and >2,000 t/s on Llama 4 Scout.", "confidence": "high", "note": "Throughput frontier-definer in this category: #1 by output speed on the AA provider leaderboard for gpt-oss-120b (~1,726 t/s, ~2.5x next-fastest SambaNova). Clear C5 (the AA provider-speed leaderboard is the MLPerf analog for inference engines). Live figure drifts run-to-run (~1,726-1,777 t/s); #1 ranking stable.", "sources": [ { "url": "https://artificialanalysis.ai/models/gpt-oss-120b/providers", "shows": "Cerebras #1 output speed ~1,726 t/s for gpt-oss-120b; next-fastest SambaNova ~692.5 t/s (~2.5x)", "accessed": "2026-06-04" }, { "url": "https://www.cerebras.ai/inference", "shows": "up to 15x faster than NVIDIA GPUs; >2,000 t/s on Llama 4 Scout", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Cerebras Inference: proprietary cloud inference on Wafer-Scale Engine (WSE); OpenAI-compatible API, serves open models (GLM 4.7, gpt-oss-120b, Llama 4 Scout). Closed/managed service. Confirmed live June 2026 (cerebras.ai)." }, { "product": "Fireworks Inference", "org": "Fireworks AI", "type": "software", "description": "High-performance inference platform built on a proprietary FireAttention engine optimized for multi-tenant model serving on commodity GPUs. Specializes in efficiently serving many models and LoRA adapters simultaneously with minimal overhead. Supports open models (Llama, Mixtral, Qwen) and function calling. Founded by ex-Meta PyTorch team members. Competitive pricing and low latency have made it a popular choice for startups building on open models.", "openness": { "score": 1, "class": "closed", "components": "engine:proprietary(FireAttention kernels);source:closed;access:managed-cloud-API-only;license:Proprietary", "confidence": "high", "note": "Proprietary FireAttention inference engine offered only as a managed cloud service; closed per recipe.", "sources": [ { "url": "https://fireworks.ai/", "shows": "managed cloud inference platform serving open models; engine not released publicly", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Strong named-customer roster including Cursor, Vercel, Notion, Sourcegraph, UiPath, Quora, Cresta, StackBlitz; serves top open models for many high-traffic AI products. No raw user count disclosed; reported_traction. Breadth of high-volume production customers supports the 1-10M-equivalent band.", "sources": [ { "url": "https://fireworks.ai/", "shows": "customers Cursor, Vercel, Notion, Sourcegraph, UiPath, Quora, Cresta, StackBlitz; case-study latency/throughput gains", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:ArtificialAnalysis(provider-speed)", "value": "3rd-fastest provider for gpt-oss-120b on the Artificial Analysis leaderboard at 658.8 output tokens/sec (top GPU-based provider; behind wafer/LPU-class Cerebras 1,726.0 and SambaNova 692.5). Customer case studies: Notion ~2s -> 350ms latency, Quora 3x speedup. (AA is a living leaderboard; figures re-verified 2026-06-04.)", "confidence": "medium", "note": "Top-tier GPU-based inference (FireAttention) and fastest GPU provider on the AA gpt-oss-120b board (658.8 t/s, 3rd overall), but below the wafer/LPU speed-frontier (Cerebras). C4 -> strong but not the throughput frontier-definer that gets a 5.", "sources": [ { "url": "https://artificialanalysis.ai/models/gpt-oss-120b/providers", "shows": "Fireworks 658.8 t/s (3rd by output speed) for gpt-oss-120b; Cerebras 1,726.0, SambaNova 692.5", "accessed": "2026-06-04" }, { "url": "https://fireworks.ai/", "shows": "Notion 2s->350ms, Quora 3x speedup case studies", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Fireworks AI cloud inference (proprietary FireAttention kernels) on NVIDIA GPUs; OpenAI-compatible API, serves many open models (DeepSeek V3.2/V4-Pro, Qwen 3.6, GLM-5.1, Kimi K2.6, Gemma 4, Llama, etc.). Closed/managed service. Confirmed live June 2026 (fireworks.ai)." }, { "product": "Together Inference Engine", "org": "Together AI", "type": "software", "description": "Proprietary inference engine optimized for serving open source models at scale with Flash Attention, speculative decoding, and custom CUDA kernels. Supports a wide catalog of open models with fast switching. Also offers a dedicated endpoints product for reserved GPU capacity. Founded by researchers from Stanford, Berkeley, and CMU. Differentiated by combining inference speed with strong research output (RedPajama, Together MoE research).", "openness": { "score": 1, "class": "closed", "components": "engine:proprietary(Together Inference Engine);source:closed(though Together publishes OSS research e.g. FlashAttention/ThunderKittens separately);access:managed-cloud-API-only;license:Proprietary", "confidence": "high", "note": "The inference engine/service itself is proprietary, API-only; closed per recipe. (Together open sources research components like FlashAttention separately, but those are not the serving engine.)", "sources": [ { "url": "https://www.together.ai/", "shows": "proprietary serverless/dedicated cloud inference for open models; engine not released; OSS research published separately", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Broad named-customer roster (Cursor, Cohere, DeepMind, ElevenLabs, Salesforce, Mozilla, Decagon, ~30+ orgs); widely used serverless inference for open models. No raw developer/user count disclosed; reported_traction at the 1-10M-equivalent band on enterprise breadth.", "sources": [ { "url": "https://www.together.ai/", "shows": "named customers Cursor, Cohere, DeepMind, ElevenLabs, Salesforce, Mozilla, Decagon + ~30 orgs", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Research-optimized GPU inference (FlashAttention-4 up to 1.3x cuDNN on Blackwell, ATLAS accelerators up to 4x LLM inference, vendor 2x-faster claim); serverless + batch + dedicated serving across broad open-model catalog", "confidence": "low", "note": "Strong GPU-based inference with leading-research kernels, but no Together figure available on the AA gpt-oss-120b provider speed board this run (only TTFT ~4.19s shown, not output t/s), so capability rests on vendor-reported research speedups -> feature_matrix, low confidence. C4: strong GPU provider, not a demonstrated throughput frontier-definer.", "sources": [ { "url": "https://www.together.ai/", "shows": "FlashAttention-4 1.3x cuDNN on Blackwell, ATLAS up to 4x LLM inference, 2x-faster claim", "accessed": "2026-06-04" }, { "url": "https://artificialanalysis.ai/models/gpt-oss-120b/providers", "shows": "Together.ai present on provider board (TTFT ~4.19s); output t/s not in displayed ranking", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Together AI Inference Engine: proprietary cloud inference (serverless/dedicated) for open models, OpenAI-compatible; built on Together's research stack (FlashAttention, ThunderKittens, ATLAS accelerators). Closed/managed service. Confirmed live June 2026 (together.ai)." }, { "product": "AWS Neuron", "org": "Amazon Web Services", "type": "software", "description": "Proprietary SDK and runtime for running inference on AWS's custom Inferentia2 and Trainium chips. The Neuron Compiler converts models from PyTorch/JAX into optimized executables for AWS custom silicon. Offers significant cost savings vs GPU instances for supported model architectures. Integrated with SageMaker and EC2. AWS's play to break NVIDIA GPU dependency for inference workloads, custom silicon with a proprietary software stack.", "openness": { "score": 1, "class": "closed", "components": "license:mixed/proprietary-core;runtime:proprietary(Neuron Runtime+Compiler not open sourced);open-glue:Linux kernel driver(Apache-2.0),vLLM-Neuron integration(open, in vLLM org),NKI library, samples(MIT-0);hardware-lock:AWS-Inferentia/Trainium-only", "confidence": "high", "note": "Headline inference path (Neuron Compiler + Neuron Runtime) is proprietary AWS, only runnable on AWS silicon; surrounding glue (driver, vLLM integration, NKI, samples) is open but does not make the engine open_source. Scored closed.", "sources": [ { "url": "https://github.com/aws-neuron/aws-neuron-sdk", "shows": "SDK repo lists samples/docs (MIT-0/Unknown) only; runtime+compiler not in an OSI-licensed repo; 2.30.0 release ~May 22 2026", "accessed": "2026-06-04" }, { "url": "https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/what-is-neuron.html", "shows": "open components (driver Apache-2.0, vLLM integration) named; Compiler/Runtime absent from open list; vLLM-V1 inference on Inferentia/Trainium", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Used across AWS Inf2/Trn fleet and integrated with HuggingFace/vLLM/PyTorch; no clean download or user count published. Level 3 reflects reported AWS-scale traction rather than a verified usage_volume figure.", "sources": [ { "url": "https://aws.amazon.com/ai/machine-learning/neuron/", "shows": "Neuron positioned as production SDK for Inferentia/Trainium with framework integrations", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "vLLM V1 API on Trainium/Inferentia with Expert Parallelism, disaggregated inference, speculative decoding; native PyTorch + JAX; compiler/runtime/profiler stack", "confidence": "medium", "note": "Strong large-scale LLM inference feature set on AWS silicon (EP, disaggregated serving, spec decode). No MLPerf Inference figure surfaced for Neuron this run, so feature_matrix not benchmark. Not a frontier C5 (hardware-locked, narrower than vLLM/SGLang/TensorRT-LLM coverage).", "sources": [ { "url": "https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/what-is-neuron.html", "shows": "vLLM V1 with EP, disaggregated inference, speculative decoding on Trainium/Inferentia", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Neuron SDK 2.30.0 (released ~May 22, 2026; 86 releases on GitHub). Serves LLM/DL inference on AWS Inferentia/Trainium accelerators (Inf2/Trn instances). Confirmed live June 2026." }, { "product": "Google Cloud TPU Inference", "org": "Google Cloud", "type": "software", "description": "Proprietary inference runtime for Google's Tensor Processing Units (TPUs), now in v5e/v6e generations. The Pathways system enables efficient model serving across TPU pods with automatic model partitioning. Powers Google's own production inference for Gemini and internal models. Available via Vertex AI and GKE for external customers. The most mature custom-silicon inference stack, developed over 8+ years of TPU generations.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary;runtime:Pathways(Google-internal distributed runtime, not open sourced);hardware-lock:Google TPU-only;managed:Google Cloud/GKE;note:JetStream(separate, Apache-2.0 OSS) is the open serving frontend but the scored product is the proprietary Pathways/TPU runtime", "confidence": "high", "note": "The named product (Pathways / TPU Runtime managed inference) is proprietary, TPU-locked, and cloud-delivered. JetStream (open) is a distinct artifact and not what this SKU is; scored closed.", "sources": [ { "url": "https://cloud.google.com/blog/products/compute/ai-hypercomputer-inference-updates-for-google-cloud-tpu-and-gpu", "shows": "Pathways runtime offered as managed Google Cloud component (not labeled open source); JetStream described as the open source engine separately", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Powers Google internal + Cloud TPU inference via AI Hypercomputer/GKE; no standalone usage figure for the Pathways runtime offering. Level 3 = reported managed-cloud traction.", "sources": [ { "url": "https://cloud.google.com/blog/products/compute/ai-hypercomputer-inference-updates-for-google-cloud-tpu-and-gpu", "shows": "Pathways/JetStream managed TPU inference on AI Hypercomputer, GKE-integrated, 3-yr CUD pricing", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "JetStream+Pathways: 1703 tok/s on Llama 3.1 405B on Trillium (multi-host), ~3x inference/$ vs TPU v5e; multi-host disaggregated serving", "confidence": "medium", "note": "High-throughput multi-host TPU inference (1703 tok/s Llama 405B on Trillium, primary Google figure). Strong but TPU-locked and not an independent MLPerf submission for the runtime; C4, below the open frontier engines' breadth.", "sources": [ { "url": "https://cloud.google.com/blog/products/compute/ai-hypercomputer-inference-updates-for-google-cloud-tpu-and-gpu", "shows": "1703 tok/s Llama 3.1 405B on Trillium, ~3x inference/$ vs v5e", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Pathways runtime (Google DeepMind distributed runtime) made available on Google Cloud and integrated into JetStream for multi-host/disaggregated TPU serving (2026, AI Hypercomputer). Proprietary managed cloud offering on Trillium v6 / Ironwood TPUs. Confirmed live June 2026." }, { "product": "Apple Core ML Runtime", "org": "Apple", "type": "software", "description": "On-device inference runtime that executes models across Apple's heterogeneous compute (CPU, GPU, and Neural Engine) on iPhone, iPad, Mac, and Vision Pro. The compiler automatically partitions model graphs across available hardware for optimal latency and power efficiency. Powers Siri, on-device autocorrect, image recognition, and Apple Intelligence features. The most widely deployed on-device inference runtime by install base (2B+ active Apple devices).", "openness": { "score": 1, "class": "closed", "components": "runtime:proprietary(Apple system framework, closed);conversion-tooling:coremltools(BSD-3-Clause, OSI, open);model-format:.mlmodel/.mlpackage(documented);hardware:Apple-silicon-only(Neural Engine)", "confidence": "high", "note": "The inference runtime itself is a closed Apple framework; only coremltools (model conversion) is open BSD-3. Headline (the runtime) is closed; scored closed with open-tooling note.", "sources": [ { "url": "https://developer.apple.com/documentation/coreml", "shows": "Core ML is Apple's proprietary on-device ML inference framework; runtime closed, runs models on Apple hardware", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Core ML is the system on-device inference runtime shipped on every modern iPhone/iPad/Mac (>1B active Apple devices) and used by a large share of App Store ML features. Mass-market reach attributed to the Apple device install base the runtime ships on, not a developer download count.", "sources": [ { "url": "https://developer.apple.com/documentation/coreml", "shows": "Core ML runs on-device across iOS/macOS/watchOS/tvOS, the default on-device inference framework for Apple platforms", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "on-device inference across CPU/GPU/Neural Engine; broad model coverage via coremltools (PyTorch/TF conversion); quantization/compression; transformer + diffusion support", "confidence": "medium", "note": "Best-in-class on-device/edge inference runtime (Neural Engine acceleration, quantization, broad converted-model coverage) but constrained to Apple silicon and on-device scale; C4 within this category (frontier C5 reserved for the large-scale server engines vLLM/SGLang/TensorRT-LLM).", "sources": [ { "url": "https://developer.apple.com/documentation/coreml", "shows": "Neural Engine acceleration, on-device inference, model optimization features", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": false, "version_note": "Apple Core ML on-device inference runtime (iOS/macOS/watchOS/tvOS, Neural Engine + GPU + CPU). Proprietary system framework; companion coremltools (conversion) is open (BSD-3-Clause). Confirmed live June 2026 via Apple Developer docs." }, { "product": "SambaNova Cloud", "org": "SambaNova Systems", "type": "software", "description": "Inference service running on custom Reconfigurable Dataflow Architecture (RDA) chips, the SN40L processor with 1TB+ on-chip memory. Designed for large models that don't fit in GPU memory without tensor parallelism. Offers both cloud API and on-premise appliance (DataScale) for enterprise deployments. Competitive with Groq and Cerebras on throughput for large models. Raised $1.5B+, primarily targeting enterprise and government customers.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary;software-stack:closed(SambaFlow compiler/runtime not open);hardware:SN40L-RDU-only;delivery:managed-cloud-API", "confidence": "high", "note": "Proprietary managed cloud inference on SambaNova's own RDU silicon; no open source software disclosed. Scored closed.", "sources": [ { "url": "https://sambanova.ai/blog/sn40l-chip-best-inference-solution", "shows": "SambaNova Cloud is a hosted proprietary inference service on SN40L; no OSS components disclosed; serves Llama 3.1 models", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Marketed as fast inference cloud with free tier; integrated by tools (Vellum) and benchmarked by third parties, but a niche specialized provider vs the hyperscalers. No verified user count; level 3 = reported traction.", "sources": [ { "url": "https://sambanova.ai/blog/sn40l-chip-best-inference-solution", "shows": "public SambaNova Cloud with free try-it inference; positioned as commercial inference provider", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Llama 3.1 8B ~1,042 tok/s, 70B ~457 tok/s, 405B ~129 tok/s at full (16-bit) precision on SN40L; >1000 tok/s single-node milestone", "confidence": "medium", "note": "Among the fastest per-request token/s in market on its silicon (full-precision 405B at 129 tok/s, 8B >1000 tok/s). Vendor/3rd-party throughput figures; specialized hardware, narrower model/feature breadth than the open frontier engines, so C4 not C5.", "sources": [ { "url": "https://sambanova.ai/blog/sn40l-chip-best-inference-solution", "shows": "Llama 3.1 405B 129 tok/s, 70B 457 tok/s, 8B 1042 tok/s at 16-bit on SN40L", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "SambaNova Cloud, proprietary managed inference service on SN40L RDU hardware (cloud.sambanova.ai). Serves Llama / open models at high token/s. Confirmed as live commercial cloud offering June 2026." }, { "product": "Qualcomm AI Engine Direct", "org": "Qualcomm", "type": "software", "description": "On-device inference runtime for Qualcomm's Snapdragon processors targeting mobile, automotive, IoT, and PC. Provides unified API across Qualcomm's Hexagon NPU, Adreno GPU, and Kryo CPU with automatic hardware dispatch for optimal performance and power. Powers on-device AI in billions of Android phones and Copilot+ PCs. Supports transformer models with quantization down to INT4. The dominant inference runtime for Android on-device AI workloads.", "openness": { "score": 1, "class": "closed", "components": "core-sdk:proprietary(Qualcomm AI Runtime SDK, separate EULA download from Qualcomm software center);open-wrapper:quic/ai-engine-direct-helper(BSD-3, OSI, only a convenience API layer);hardware-lock:Qualcomm Snapdragon NPU/HTP", "confidence": "high", "note": "The QNN runtime/SDK itself is proprietary (Qualcomm download + license); the open BSD-3 helper merely wraps it and requires the proprietary libraries. Headline engine closed; scored closed.", "sources": [ { "url": "https://github.com/quic/ai-engine-direct-helper", "shows": "BSD-3 helper that 'is an extension of the Qualcomm AI Runtime SDK'; underlying QNN SDK is a separate proprietary download required to run inference on NPU/HTP", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "QNN is the on-device inference backend for the broad Snapdragon install base and is wired into LiteRT (Google AI Edge) and ExecuTorch as the Qualcomm NPU backend, giving it wide on-device reach. No clean developer/usage count; level 4 reflects Snapdragon-ecosystem distribution (reported), not a measured figure.", "sources": [ { "url": "https://github.com/quic/ai-engine-direct-helper", "shows": "tooling for running models on Snapdragon NPU/HTP across Windows-on-Snapdragon and Linux ARM64", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "unified API across Hexagon HTP/NPU + Adreno GPU + Kryo CPU; AOT + on-device compilation; LiteRT CompiledModel + ExecuTorch backends; power-efficient on-device LLM/CV inference", "confidence": "medium", "note": "Leading mobile/edge on-device inference runtime (HTP acceleration, multi-accelerator, AOT compile) but Snapdragon-locked and edge-scale; C4 within this category, below the server frontier engines.", "sources": [ { "url": "https://github.com/quic/ai-engine-direct-helper", "shows": "runs inference on NPU/HTP; multi-accelerator Qualcomm stack", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Qualcomm AI Engine Direct SDK (aka Qualcomm AI Runtime / QNN) for on-device inference on Snapdragon NPU/HTP (Hexagon), GPU (Adreno), CPU (Kryo). Distributed as a proprietary SDK (separate Qualcomm download); open source helper wrapper (quic/ai-engine-direct-helper, BSD-3) exists. Integrated as backend for LiteRT and ExecuTorch. Confirmed live June 2026." }, { "product": "vLLM", "org": "vLLM (UC Berkeley / vLLM team)", "type": "software", "description": "High-throughput LLM serving engine built on PagedAttention, which treats KV cache as virtual memory pages to near-eliminate memory waste during batched inference. Supports continuous batching, tensor parallelism, speculative decoding, and 40+ model architectures. The de facto standard for self-hosted production model serving, used by Anyscale, Databricks, and most open source inference deployments.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;governance:vLLM-project(community,PyTorch-ecosystem adjacent)", "confidence": "high", "note": "license:Apache-2.0(OSI);source:public;governance:vLLM-project(community,PyTorch-ecosystem adjacent)", "sources": [ { "url": "https://github.com/vllm-project/vllm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.programming-helper.com/tech/vllm-2026-open-source-llm-inference-engine", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~3M+ PyPI downloads/month as of 2026; de facto standard open inference engine with enterprise adoption (Amazon, Roblox, Stripe). 81.9k GitHub stars corroborates but is not the primary basis. Level 4 (1-10M) on download volume; could exceed if cumulative installs counted.", "sources": [ { "url": "https://pypistats.org/packages/vllm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://pypi.org/project/vllm/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://awesomeagents.ai/tools/best-open-source-llm-inference-servers-2026/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:MLPerf", "value": null, "confidence": "high", "note": "Reference PagedAttention implementation; used in MLPerf Inference submissions (Red Hat OpenShift AI on GPT-J-6b and llama-2-70b). Broad model/hardware coverage (200+ architectures) per repo. Top-tier among open inference engines.", "sources": [ { "url": "https://www.redhat.com/en/blog/accelerating-generative-ai-adoption-red-hat-openshift-ai-achieves-impressive-results-mlperf-inference-benchmarks-vllm-runtime", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/vllm-project/vllm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "v0.22.0, released May 29, 2026 (GitHub releases). Confirmed to exist as of June 2026." }, { "product": "SGLang", "org": "SGLang (LMSYS / UC Berkeley)", "type": "software", "description": "High-performance LLM/VLM serving engine built around RadixAttention for automatic prefix-KV reuse, with structured-generation, speculative decoding, and disaggregated-prefill support. Deployed on 400K+ GPUs worldwide and serving trillions of tokens/day; xAI uses it to serve Grok 3, Microsoft Azure to serve DeepSeek R1 on AMD, and adopters include AMD, NVIDIA, AWS, Oracle Cloud, LinkedIn, and Cursor. Joined the PyTorch ecosystem in March 2025; 28K GitHub stars with 450+ contributors.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;governance:LMSYS/PyTorch-ecosystem(community)", "confidence": "high", "note": "license:Apache-2.0(OSI);source:public;governance:LMSYS/PyTorch-ecosystem(community)", "sources": [ { "url": "https://github.com/sgl-project/sglang", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Reported generating trillions of tokens/day in production across 400,000+ GPUs; enterprise users include xAI, NVIDIA, AMD, LinkedIn, Cursor, Oracle/Google/Azure/AWS clouds. Used as rollout backend for frontier-model training (verl, slime, etc.). 28.9k stars secondary. Level 4 reflects very large but operator-concentrated deployment footprint.", "sources": [ { "url": "https://github.com/sgl-project/sglang", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://inference.net/content/sglang-complete-guide/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "RadixAttention, structured generation, multi-step pipelines; benchmarked as top-tier vs vLLM/TensorRT-LLM on H100 in 2026 comparisons; 25x perf improvement reported on NVIDIA GB300 NVL72 (Feb 2026). No formal MLPerf submission confirmed for SGLang itself, so basis is feature_matrix plus third-party benchmarks rather than a standardized suite.", "sources": [ { "url": "https://www.spheron.network/blog/vllm-vs-tensorrt-llm-vs-sglang-benchmarks/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/sgl-project/sglang", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "v0.5.12.post1, released May 26, 2026 (GitHub). Hosted under LMSYS / PyTorch ecosystem. Confirmed to exist as of June 2026." }, { "product": "TensorRT-LLM", "org": "NVIDIA", "type": "software", "description": "NVIDIA's optimized LLM inference library providing maximum throughput on NVIDIA GPUs through kernel fusion, in-flight batching, FP8/INT4 quantization, and multi-GPU tensor parallelism. Python API for defining and compiling models into optimized TensorRT engines. Used by major cloud providers and inference services as the backend for NVIDIA GPU deployments. Source code is Apache-2.0, but the value proposition is tightly coupled to NVIDIA hardware and CUDA ecosystem.", "openness": { "score": 4, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;caveat:runs only on NVIDIA GPUs (vendor-hardware lock, not a license restriction)", "confidence": "high", "note": "license:Apache-2.0(OSI);source:public;caveat:runs only on NVIDIA GPUs (vendor-hardware lock, not a license restriction)", "sources": [ { "url": "https://github.com/NVIDIA/TensorRT-LLM", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Production inference optimizer/runtime for NVIDIA GPUs; used in MLPerf submissions (HGX H100 GPT-J ~3x gains). Described in 2026 as the 'specialist's tool', narrower adoption than vLLM/SGLang due to compilation overhead and NVIDIA-only scope. ~13.8k stars. Level 3 reflects significant but more specialized usage; no clean user/download count verified.", "sources": [ { "url": "https://github.com/NVIDIA/TensorRT-LLM", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.yottalabs.ai/post/best-llm-inference-engines-in-2026-vllm-tensorrt-llm-tgi-and-sglang-compared", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://theaiengineer.substack.com/p/vllm-vs-ollama-vs-sglang-vs-tensorrt", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:MLPerf", "value": null, "confidence": "high", "note": "Maximum NVIDIA-hardware performance; featured in MLPerf Inference (HGX H100 ~3x GPT-J gain attributed to TensorRT-LLM). Top raw throughput/latency on NVIDIA silicon among 2026 contenders, at the cost of compilation overhead.", "sources": [ { "url": "https://www.hpcwire.com/2024/03/28/mlperf-inference-4-0-results-showcase-genai-nvidia-still-dominates/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.spheron.network/blog/vllm-vs-tensorrt-llm-vs-sglang-benchmarks/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Actively maintained; repo last updated 2026-06-01 with ongoing releases (AutoDeploy Q2/Q3 milestone). Confirmed to exist as of June 2026." }, { "product": "LocalAI", "org": "Mudler (LocalAI)", "type": "software", "description": "Self-hosted, OpenAI-compatible inference engine that runs LLMs, vision, image, audio, and video models on commodity hardware (CPU or any GPU). A modular multi-backend architecture (36+ backends incl. llama.cpp, vLLM, transformers, whisper.cpp, diffusers) behind one unified API, with built-in agents, RAG, and MCP.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/mudler/LocalAI", "shows": "MIT, ~46.9k stars, multi-backend OpenAI-compatible API", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~47k stars; no clean download telemetry for a self-hosted server - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/mudler/LocalAI", "shows": "~46.9k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "text/vision/voice/image/video models; 36+ backends; OpenAI-compatible API; CPU-only capable; built-in agents + RAG + MCP", "confidence": "high", "note": "Widest model-type and backend coverage of the inference-engine peers.", "sources": [ { "url": "https://github.com/mudler/LocalAI", "shows": "feature set: multi-backend, multimodal, agents/MCP", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT, no feature-gated core. ~47k stars. Broadest model-type coverage of the local inference engines." }, { "product": "ds4", "org": "antirez (Salvatore Sanfilippo)", "type": "software", "description": "Native local LLM inference engine by antirez (Salvatore Sanfilippo), optimized first for DeepSeek V4 Flash (with DeepSeek V4 PRO support on very-high-memory machines). Self-contained C/CUDA/Objective-C engine built on llama.cpp/GGML with DeepSeek-specific loading, prompt rendering, tool calling, and KV state streaming (RAM and on-disk SSD). Ships an HTTP server with OpenAI-compatible endpoints and an integrated coding agent, across Metal (Apple Silicon), NVIDIA CUDA/DGX Spark, and AMD Strix Halo (ROCm) backends.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no feature-gated core;built on ggml/llama.cpp", "confidence": "high", "note": "MIT-licensed native inference engine; full C/CUDA/Obj-C source public, no proprietary tier.", "sources": [ { "url": "https://github.com/antirez/ds4/blob/main/LICENSE", "shows": "MIT License text, (c) 2026 The ds4.c authors; (c) 2023-2026 The ggml authors", "accessed": "2026-06-19" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~14.6k GitHub stars six weeks after launch, but driven heavily by antirez's profile; no package distribution and no install/usage count, so treated as a low-confidence star-proxy pending warehouse signal.", "sources": [ { "url": "https://github.com/antirez/ds4", "shows": "~14.6k stars, 332 commits, 0 releases, created May 2026", "accessed": "2026-06-19" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Backends: Metal/CUDA/ROCm; DeepSeek V4 Flash/PRO specialization; OpenAI-compatible HTTP server; RAM + SSD KV streaming; integrated coding agent. No MLPerf submission.", "confidence": "medium", "note": "Specialized single-model-family engine rather than the breadth of llama.cpp (C4) or the datacenter-throughput frontier (C5); scored 3 for a focused, capable local runtime.", "sources": [ { "url": "https://github.com/antirez/ds4", "shows": "README documents Metal/CUDA/ROCm backends, OpenAI-compatible server, SSD KV streaming, integrated agent", "accessed": "2026-06-19" } ] }, "maturity": 2.5, "mature": false, "version_note": "MIT-licensed (LICENSE body, (c) 2026 The ds4.c authors + ggml authors). Created 6 May 2026, ~332 commits, last push 17 Jun 2026; ~14.6k stars, no tagged releases (built from source). Deliberately narrow 'one model at a time' design. Verified 2026-06-19." } ] }, "finetuning_code": { "label": "Fine-tuning code", "arc": "Model components", "layer": "model_components", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "TRL", "org": "Hugging Face", "type": "software", "description": "The de facto reference library for post-training transformers, supports SFT, DPO, GRPO, PPO, KTO, reward modeling, and other preference-optimization methods through unified Trainer APIs. Picked over alternatives because it sits at the center of the HF ecosystem (transformers, datasets, accelerate, PEFT) and ships canonical reference implementations that papers and other frameworks build on. 18.4K GitHub stars, 482 contributors, ~400 commits in 90 days; extremely active.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(huggingface/trl);governance:Hugging Face;no feature-gated core;managed-tier:none", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), full source public, no proprietary core or managed training SaaS gate.", "sources": [ { "url": "https://github.com/huggingface/trl/blob/main/LICENSE", "shows": "Apache License Version 2.0 text", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~3.81M PyPI downloads last month; standard library for RLHF/post-training in the HF stack, integrated with Transformers/Accelerate and used across labs and open-model post-training pipelines.", "sources": [ { "url": "https://pypistats.org/packages/trl", "shows": "Downloads last month: 3,809,112", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: SFT, DPO/IPO/KTO/ORPO, PPO, GRPO, reward modeling; parallelism via HF Accelerate (DeepSpeed/FSDP); precision BF16/FP8 via Accelerate; broad method coverage but relies on Accelerate for scale rather than native frontier-scale parallelism like Megatron-LM (the C5 anchor).", "confidence": "high", "note": "Broadest post-training method coverage among the open libraries; one tier below the Megatron-LM frontier-scale-training anchor on demonstrated large-scale parallelism.", "sources": [ { "url": "https://github.com/huggingface/trl/blob/main/LICENSE", "shows": "huggingface/trl repo (TRL post-training library) confirmed live", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "TRL (Transformer Reinforcement Learning) post-training library; repo huggingface/trl active, LICENSE confirmed live June 2026. Supplies SFT, DPO/IPO/KTO/ORPO, PPO, GRPO trainers built on HF Transformers/Accelerate; the de-facto reference post-training/RLHF library in the HF ecosystem." }, { "product": "PEFT", "org": "Hugging Face", "type": "software", "description": "State-of-the-art Parameter-Efficient Fine-Tuning library implementing LoRA, QLoRA, DoRA, adapters, prompt tuning, prefix tuning, and IA3 as a drop-in wrapper around transformers models. Picked because it's the underlying adapter engine that TRL, Axolotl, Unsloth, and most other frameworks compose on top of; touching PEFT is unavoidable in the modern fine-tuning stack. 21.2K GitHub stars, 300 contributors, 116 commits in 90 days.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(huggingface/peft);governance:Hugging Face;no feature-gated core;managed-tier:none", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), full source public, no proprietary tier.", "sources": [ { "url": "https://github.com/huggingface/peft/blob/main/LICENSE", "shows": "Apache License Version 2.0 text", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 10.28M PyPI downloads in the last 30 days; the canonical adapter/LoRA library and a dependency of TRL and most LoRA fine-tuning stacks. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. llama-index ~10.18M, pydantic-ai ~31M at level 5); PEFT clears it. Conservative note: at ~10.3M it sits just above the floor, so the bump is borderline and rests on volume plus its dependency-graph centrality in the HF fine-tuning stack.", "sources": [ { "url": "https://pypistats.org/api/packages/peft/recent", "shows": "last_month downloads = 10,281,112", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: LoRA/QLoRA/AdaLoRA/LoHa/LoKr/IA3/prefix-prompt-P-tuning (adapter family); precision: 4/8-bit quant + BF16; parallelism inherited from Accelerate. Deep within parameter-efficient methods but scope is adapters/PEFT only (no full-RLHF-PPO orchestration like TRL/OpenRLHF, no frontier-scale pretraining like Megatron-LM).", "confidence": "high", "note": "Reference-grade depth on parameter-efficient methods; narrower method scope than full-stack trainers, so C4 not C5.", "sources": [ { "url": "https://github.com/huggingface/peft/blob/main/LICENSE", "shows": "huggingface/peft repo (PEFT adapter library) confirmed live", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "PEFT (Parameter-Efficient Fine-Tuning) library; repo huggingface/peft, LICENSE confirmed live June 2026. Reference implementation of adapter methods (LoRA, QLoRA, AdaLoRA, IA3, prefix/prompt tuning) integrated across Transformers/Diffusers/Accelerate." }, { "product": "Axolotl", "org": "Axolotl AI", "type": "software", "description": "YAML-config-driven fine-tuning framework, wraps TRL, PEFT, DeepSpeed, FSDP, and bitsandbytes behind a single config file so users can run SFT, LoRA, QLoRA, full fine-tunes, and DPO across 100+ model families without writing training loops. Picked for serious multi-GPU production fine-tuning where reproducibility via config matters and where Unsloth's single-GPU optimizations don't apply. 12K GitHub stars, 240 contributors, 185 commits in 90 days; the de facto tool for hobbyist and indie fine-tuners shipping models on HF Hub.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(axolotl-ai-cloud/axolotl);no managed SaaS tier (cloud-template integrations only: RunPod/Modal/etc.)", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0); no proprietary managed training service, only deployment templates for third-party clouds.", "sources": [ { "url": "https://github.com/axolotl-ai-cloud/axolotl/blob/main/LICENSE", "shows": "Apache License Version 2.0 text", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~22.3K PyPI downloads last month; ~12K GitHub stars. Popular config-driven FT tool in the open-model community but download volume is modest vs TRL/PEFT/Unsloth; level 2 on usage_volume, stars corroborate only.", "sources": [ { "url": "https://pypistats.org/packages/axolotl", "shows": "Downloads last month: 22,336", "accessed": "2026-06-04" }, { "url": "https://github.com/axolotl-ai-cloud/axolotl", "shows": "~12,000+ GitHub stars (corroborating, not basis)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: full FT, LoRA, QLoRA, GPTQ, QAT, DPO/IPO/KTO/ORPO, GRPO/GDPO, reward modeling; parallelism: FSDP1/FSDP2, DeepSpeed, multi-node Torchrun/Ray, Sequence Parallelism + ND Parallelism (CP+TP+FSDP); precision: BF16/4-8bit. Broadest single-tool method+parallelism matrix in this batch.", "confidence": "high", "note": "Very wide method and parallelism coverage (ND parallelism, multi-node), strong C4; below Megatron-LM frontier-scale-training anchor on demonstrated extreme scale.", "sources": [ { "url": "https://github.com/axolotl-ai-cloud/axolotl", "shows": "documented methods (LoRA/QLoRA/DPO/GRPO/...) and parallelism (FSDP/DeepSpeed/SP/ND, multi-node Ray)", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Axolotl fine-tuning framework (axolotl-ai-cloud/axolotl), confirmed live June 2026 (Apache-2.0). Config-driven post-training: full FT, LoRA/QLoRA/GPTQ/QAT, DPO/IPO/KTO/ORPO, GRPO/GDPO, reward modeling; multi-GPU FSDP1/FSDP2/DeepSpeed, multi-node via Torchrun/Ray, plus SP and ND parallelism." }, { "product": "LLaMA-Factory", "org": "", "type": "software", "description": "LLaMA-Factory (hiyouga/LLaMA-Factory), Apache-2.0, confirmed live June 2026 (~71.9K stars). Unified efficient fine-tuning of 100+ LLMs/VLMs (ACL 2024); CLI + web UI; LoRA/QLoRA/full FT and preference/RL methods.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(hiyouga/LLaMA-Factory);CLI+WebUI;no proprietary managed tier", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), full source public including the web UI; no feature-gated core.", "sources": [ { "url": "https://github.com/hiyouga/LLaMA-Factory", "shows": "Apache-2.0 license, unified fine-tuning framework", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "~234.6K PyPI (llamafactory) downloads last month plus very large GitHub adoption (~71.9K stars, 8.8K forks) and named users (Amazon, NVIDIA, Alibaba/Aliyun). Many users run from source/WebUI rather than pip, so pip undercounts; level 3 on download volume + named-deployment breadth, stars corroborate only.", "sources": [ { "url": "https://pypistats.org/packages/llamafactory", "shows": "Downloads last month: 234,631", "accessed": "2026-06-04" }, { "url": "https://github.com/hiyouga/LLaMA-Factory", "shows": "~71.9K stars; named users Amazon/NVIDIA/Alibaba/Aliyun", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: full FT, LoRA, QLoRA, plus preference/RL tuning; 100+ LLM/VLM architectures; CLI + web UI; multimodal (VLM) support. Broad model coverage and a no-code WebUI distinguish it; parallelism present but headline scale below Megatron-LM anchor.", "confidence": "medium", "note": "Strong breadth (100+ models, VLMs, no-code UI) puts it at C4; lacks the demonstrated frontier-scale parallelism of the Megatron-LM C5 anchor.", "sources": [ { "url": "https://github.com/hiyouga/LLaMA-Factory", "shows": "100+ LLMs/VLMs, LoRA/QLoRA/full FT, CLI+WebUI", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "LLaMA-Factory (hiyouga/LLaMA-Factory), Apache-2.0, confirmed live June 2026 (~71.9K stars). Unified efficient fine-tuning of 100+ LLMs/VLMs (ACL 2024); CLI + web UI; LoRA/QLoRA/full FT and preference/RL methods." }, { "product": "Unsloth", "org": "Unsloth AI", "type": "software", "description": "Performance-optimized fine-tuning library that rewrites kernels in Triton/CUDA to deliver 2-5x faster training with 60-80% less memory on a single GPU, makes 7B/13B LoRA fine-tuning feasible on a consumer RTX 4090. Picked when hardware is constrained and the workload fits one GPU; Unsloth's kernels also slot under TRL trainers so it complements rather than replaces the broader ecosystem. 65K GitHub stars and ~1500 commits in 90 days makes it the fastest-moving repo in the category; widely promoted on r/LocalLLaMA as the default for local fine-tuning.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI, core unsloth pkg)+AGPL-3.0(OSI, optional Unsloth Studio UI);source:public(unslothai/unsloth);no proprietary managed-training SaaS gate", "confidence": "high", "note": "OSI-licensed throughout (Apache-2.0 core, AGPL-3.0 for the optional Studio UI); both are OSI licenses so class is open_source, not open_core.", "sources": [ { "url": "https://github.com/unslothai/unsloth", "shows": "Apache-2.0 core + AGPL-3.0 Studio UI; fine-tuning framework", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~2.43M PyPI downloads last month; widely used for low-VRAM/consumer-GPU fine-tuning of open models; ~65.8K stars corroborate. Level 4 on download volume.", "sources": [ { "url": "https://pypistats.org/packages/unsloth", "shows": "Downloads last month: 2,432,077", "accessed": "2026-06-04" }, { "url": "https://github.com/unslothai/unsloth", "shows": "~65.8K GitHub stars (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: full FT, LoRA/QLoRA, RL (GRPO, FP8 RL), pretraining, embedding FT; custom Triton kernels delivering ~2x faster / up to 70% less VRAM, MoE training ~12x faster / 35% less VRAM; 500+ supported models; multiple quant formats. Single-node memory/speed optimization is its frontier edge; less focused on multi-node frontier-scale parallelism than Megatron-LM.", "confidence": "high", "note": "Best-in-class single-/consumer-GPU efficiency (kernel-level), broad method coverage; C4, not the frontier multi-node-scale anchor.", "sources": [ { "url": "https://github.com/unslothai/unsloth", "shows": "Triton-kernel 2x speed / 70% VRAM claims, GRPO/FP8 RL, 500+ models", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Unsloth (unslothai/unsloth), confirmed live June 2026 (~65.8K stars). Memory/speed-optimized fine-tuning via custom Triton kernels; SFT, RL (GRPO), pretraining, quantization; claims up to 2x faster / 70% less VRAM. Core Apache-2.0; optional Unsloth Studio UI under AGPL-3.0." }, { "product": "torchtune", "org": "PyTorch Foundation (Linux Foundation)", "type": "software", "description": "torchtune (pytorch/torchtune), BSD-3-Clause, confirmed live June 2026 but DEPRECATED; repo states 'torchtune is no longer actively maintained; development wound down in 2025.' Provided SFT, knowledge distillation, DPO/PPO/GRPO, QAT; single/multi-device/multi-node; FSDP2; HF Hub + W&B integration.", "openness": { "score": 5, "class": "open_source", "components": "license:BSD-3-Clause(OSI);source:public(pytorch/torchtune);governance:PyTorch Foundation/Meta;no managed tier", "confidence": "high", "note": "Fully OSI-licensed (BSD-3-Clause), full source public; openness unaffected by the project being wound down.", "sources": [ { "url": "https://github.com/pytorch/torchtune", "shows": "BSD-3-Clause license; PyTorch post-training library", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "~370K PyPI downloads last month, but the project is officially deprecated (development wound down in 2025), so usage is on a declining trajectory; ~5.8K stars. Level 3 on current download volume with the stale caveat flagged.", "sources": [ { "url": "https://pypistats.org/packages/torchtune", "shows": "Downloads last month: 370,048", "accessed": "2026-06-04" }, { "url": "https://github.com/pytorch/torchtune", "shows": "'no longer actively maintained; development wound down in 2025'; ~5.8K stars", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: SFT, knowledge distillation, DPO/PPO/GRPO, QAT; parallelism: single-device, multi-device, multi-node, FSDP2; memory-efficiency focus; integrations (HF Hub, W&B). Capability frozen at 2025 wind-down; no FP8/FP4 or newer-method additions; behind actively-developed peers and the Megatron-LM anchor.", "confidence": "medium", "note": "Competent native-PyTorch method/parallelism coverage but development halted in 2025, so it no longer tracks the moving frontier, C3.", "sources": [ { "url": "https://github.com/pytorch/torchtune", "shows": "SFT/distillation/DPO/PPO/GRPO/QAT, FSDP2, single/multi-device/multi-node", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "torchtune (pytorch/torchtune), BSD-3-Clause, confirmed live June 2026 but DEPRECATED; repo states 'torchtune is no longer actively maintained; development wound down in 2025.' Provided SFT, knowledge distillation, DPO/PPO/GRPO, QAT; single/multi-device/multi-node; FSDP2; HF Hub + W&B integration." }, { "product": "OpenRLHF", "org": "OpenRLHF", "type": "software", "description": "Ray-based distributed RLHF/agentic-RL framework that distributes Actor, Reward, Reference, and Critic models across GPUs for full-scale 70B+ training. Picked over TRL when training scale exceeds a single node and the workload is RL-heavy (PPO, DAPO, REINFORCE++, GRPO, KTO), combines Ray + vLLM for high-throughput rollouts with DeepSpeed ZeRO-3 for memory-efficient training. 9.5K GitHub stars, 87 contributors, published research backing (arxiv 2405.11143); used by labs scaling preference optimization.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(OpenRLHF/OpenRLHF);no managed tier;built on Ray+vLLM+DeepSpeed", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), full source public, no proprietary tier.", "sources": [ { "url": "https://github.com/OpenRLHF/OpenRLHF", "shows": "Apache-2.0 license; production-ready RLHF framework", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "medium", "note": "~9.6K GitHub stars; named adopters listed in repo (Google, ByteDance, Tencent, Alibaba, Baidu, China Telecom, Vivo, Allen AI, NexusFlow, Jülich). Specialist RLHF tool used by research/training teams rather than a mass-download library (no headline PyPI figure surfaced); level 2 on named-traction, not stars.", "sources": [ { "url": "https://github.com/OpenRLHF/OpenRLHF", "shows": "~9.6K stars; named users Google/ByteDance/Tencent/Alibaba/Baidu/Allen AI etc.", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: PPO, REINFORCE++, GRPO, RLOO (RLHF-focused); architecture: Ray scheduling + vLLM-accelerated rollout + DeepSpeed; multi-turn agent RL; scales to 70B+ across multiple GPUs. Deep, production-grade distributed RLHF specialization; narrower than general FT tools but strong on large-scale RL.", "confidence": "medium", "note": "Strong distributed-RLHF capability (Ray+vLLM+DeepSpeed, 70B+), C4; below the Megatron-LM frontier-scale anchor and method scope is RL-centric rather than full-spectrum.", "sources": [ { "url": "https://github.com/OpenRLHF/OpenRLHF", "shows": "PPO/REINFORCE++/GRPO/RLOO, Ray+vLLM+DeepSpeed, 70B+ scale", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "OpenRLHF (OpenRLHF/OpenRLHF), Apache-2.0, confirmed live June 2026 (~9.6K stars). Self-described 'first high-performance production-ready open source RLHF framework'; PPO, REINFORCE++, GRPO, RLOO; Ray scheduling + vLLM rollout + DeepSpeed; scales to 70B+ models; named users incl. Google, ByteDance, Tencent, Alibaba, Baidu, Allen AI." }, { "product": "verl", "org": "ByteDance Seed / Volcano Engine", "type": "software", "description": "HybridFlow flexible and efficient RL post-training framework developed by ByteDance Seed MLSys and HKU, purpose-built for RLHF and reasoning RL on math and code tasks. Picked when teams need state-of-the-art GRPO/PPO at frontier scale; verl shipped early support for DeepSeek-R1-style reasoning RL recipes and has become a reference implementation in the post-DeepSeek wave. 21.5K GitHub stars, 618 contributors, ~420 commits in 90 days, one of the fastest-growing repos in the category.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;governance:community(ByteDance-Seed origin);no feature-gated core", "confidence": "high", "note": "Fully OSI-licensed; full source public, no managed-tier gating of the library itself.", "sources": [ { "url": "https://github.com/volcengine/verl", "shows": "Apache-2.0 license; v0.8.0 (Jun 1 2026); RL post-training framework", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "~96.7k PyPI downloads last month (primary). De facto standard high-throughput RL post-training lib; used to train Seed-Thinking-v1.5 and Doubao, and adopted/contributed by Qwen team, Moonshot/Kimi, NVIDIA research, Microsoft Research, Amazon, LinkedIn, Xiaomi, Baidu and many labs. Level 4 reflects broad and growing developer-tool usage at the high end of the 100K-1M band trending toward heavier use.", "sources": [ { "url": "https://pypistats.org/packages/verl", "shows": "~96,716 PyPI downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/volcengine/verl", "shows": "adopter list incl. ByteDance, Qwen, Moonshot, NVIDIA research, Microsoft Research, Amazon, LinkedIn", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Methods: PPO, GRPO, RLOO, ReMax, REINFORCE++, GSPO, PRIME, DAPO, DrGRPO, plus SFT/DPO; HybridFlow hybrid-controller; multiple FSDP/Megatron backends; SGLang/vLLM rollout; used to train frontier reasoning models at scale.", "confidence": "high", "note": "Frontier-tier within finetuning_code: broadest modern RL-algorithm coverage and demonstrated use training production frontier reasoning models. Calibrated alongside Megatron-LM (C5) as a scale/method definer for this category.", "sources": [ { "url": "https://github.com/volcengine/verl", "shows": "PPO/GRPO/RLOO/GSPO/PRIME/DAPO/DrGRPO methods; HybridFlow; used for Seed-Thinking-v1.5 / Doubao", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "verl (Volcano Engine RL / HybridFlow), v0.8.0 released June 1 2026. Initiated by ByteDance Seed, now community-maintained. RL post-training library (takes existing weights and adapts via RLHF/RL). Confirmed live on GitHub June 2026." }, { "product": "SWIFT", "org": "ModelScope (Alibaba)", "type": "software", "description": "Alibaba ModelScope's unified PEFT/full-parameter CPT/SFT/DPO/GRPO toolkit covering 600+ LLMs and 300+ MLLMs (Qwen3, DeepSeek-R1, GLM, Llama4, InternVL, etc.). Picked over LLaMA-Factory when working in the ModelScope ecosystem or when broad multimodal model coverage matters. SWIFT often ships day-zero recipes for Chinese-origin models. 14.2K GitHub stars, 147 contributors, 335 commits in 90 days; published as AAAI 2025.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;maintainer:ModelScope(Alibaba);no feature-gated core", "confidence": "high", "note": "Fully OSI-licensed open source training/deploy framework; source public.", "sources": [ { "url": "https://github.com/modelscope/ms-swift", "shows": "Apache-2.0 license; v4.2.3 (May 31 2026); ModelScope/Alibaba maintainer", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "~110k PyPI downloads last month (primary). Broad coverage (600+ text + 300+ multimodal models incl. Qwen3, DeepSeek, Llama, GLM) and strong uptake in the Chinese/ModelScope ecosystem. Lower bound of the 100K-1M band; level 4 on sustained monthly download volume.", "sources": [ { "url": "https://pypistats.org/packages/ms-swift", "shows": "~110,083 PyPI downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/modelscope/ms-swift", "shows": "600+ text models, 300+ multimodal models supported", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: full FT, LoRA/QLoRA/DoRA, SFT, GRPO family (GRPO/DAPO/GSPO/SAPO/CISPO/CHORD/RLOO/Reinforce++), preference (DPO/KTO/RM/CPO/SimPO/ORPO), pretraining; 600+ models; integrated quantization/eval/deploy.", "confidence": "high", "note": "Very broad method + model coverage and an end-to-end train/eval/deploy pipeline. Scored 4 (not 5) vs verl/Megatron: it is a comprehensive practitioner toolkit rather than the demonstrated large-scale frontier-training definer.", "sources": [ { "url": "https://github.com/modelscope/ms-swift", "shows": "LoRA/QLoRA/DoRA, SFT, GRPO family, DPO/KTO/CPO/SimPO/ORPO, pretraining across 600+/300+ models", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "ms-swift (SWIFT) by ModelScope/Alibaba, v4.2.3 (patch) released May 31 2026. Fine-tuning + deployment framework covering 600+ LLMs and 300+ multimodal models. Confirmed live on GitHub June 2026." }, { "product": "LitGPT", "org": "Lightning AI", "type": "software", "description": "20+ from-scratch implementations of high-performance open LLMs (Llama, Mistral, Gemma, Phi, Qwen) with recipes for pretrain, finetune (LoRA, QLoRA, full), and deploy at scale. Picked by teams that want a clean, hackable, single-file-per-model codebase optimized for Lightning Fabric and FSDP, rather than the abstraction layers of transformers/TRL. 13.4K GitHub stars, 132 contributors; pace has slowed (14 commits in 90 days) but the library remains a popular teaching/research baseline.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;maintainer:Lightning-AI;no feature-gated core", "confidence": "high", "note": "Apache-2.0, full source public; the library is unrestricted (Lightning's commercial offerings are separate products).", "sources": [ { "url": "https://github.com/Lightning-AI/litgpt", "shows": "Apache-2.0 license; v0.5.12 (Dec 18 2025); pretrain/finetune/deploy recipes", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~14.2k PyPI downloads last month (primary); 13.4k GitHub stars (corroborating only). Used in research (TinyLlama, NeurIPS 2023 LLM Efficiency Challenge starter kit). Modest but real recurring usage places it at the low end of medium; level 3 on monthly download volume, not stars.", "sources": [ { "url": "https://pypistats.org/packages/litgpt", "shows": "~14,230 PyPI downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/Lightning-AI/litgpt", "shows": "13.4k stars; TinyLlama / NeurIPS efficiency challenge usage", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: full finetuning, LoRA, QLoRA, Adapter/Adapter v2, pretraining + continued pretraining; Flash Attention, FSDP, fp4/8/16/32 quantization; 1-1000+ GPU/TPU. No PPO/GRPO/DPO RL post-training.", "confidence": "medium", "note": "Solid PEFT + pretraining recipes and clean from-scratch implementations, but narrower than verl/ms-swift/NeMo-RL (no RLHF/preference methods) and oriented to readable recipes over frontier-scale demonstration. Mid-tier within the category.", "sources": [ { "url": "https://github.com/Lightning-AI/litgpt", "shows": "full FT, LoRA/QLoRA, Adapter/Adapter v2, pretraining; FSDP, fp4-32 quant, 1-1000+ GPUs", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "LitGPT by Lightning AI, v0.5.12 released Dec 18 2025 (~6mo old at scoring; acceptable freshness). From-scratch LLM implementations with pretrain/finetune/deploy recipes for 20+ models. Confirmed live on GitHub June 2026." }, { "product": "NeMo-RL", "org": "NVIDIA", "type": "software", "description": "NVIDIA's scalable open source toolkit for RL post-training, successor to NeMo-Aligner. Supports DPO, PPO, GRPO, and reward modeling on Megatron-LM and FSDP backends with first-class multi-node H100/B200 scaling. Picked when teams are already on the NVIDIA NeMo stack and need RL post-training that scales to thousands of GPUs. 1.6K GitHub stars but 181 commits in 90 days and a growing contributor base, well-resourced; pairs with NeMo Customizer (managed) for enterprise users.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;maintainer:NVIDIA;no feature-gated core (NVIDIA-GPU oriented, not a license restriction)", "confidence": "high", "note": "Apache-2.0, full source public; GPU orientation is a hardware practicality, not a license restriction.", "sources": [ { "url": "https://github.com/NVIDIA-NeMo/RL", "shows": "Apache-2.0 license; v0.6.0 (Apr 30 2026); post-training library", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "<10K", "signal_type": "stars_fallback", "confidence": "medium", "note": "No PyPI download or named-user volume located this run; only ~1.7k GitHub stars (last-resort signal, caps at level 3). Young (v0.6.0, Apr 2026) and narrower than the broader NeMo framework. Level 2 on stars_fallback; honest signal is limited.", "sources": [ { "url": "https://github.com/NVIDIA-NeMo/RL", "shows": "~1.7k GitHub stars; v0.6.0 Apr 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Methods: GRPO, GSPO, DAPO, GDPO, SFT (+LoRA), DPO (+LoRA), on-policy distillation, reward modeling; backends DTensor (TP/SP/PP/CP/FSDP2) and Megatron Core with 6D parallelism (TP/PP/CP/SP/EP/FSDP); SGLang backend, speculative decoding, long-context training.", "confidence": "high", "note": "Frontier-tier scale/parallelism within finetuning_code: Megatron-Core backend with 6D parallelism and modern RL/preference method coverage. Capability (does it train at scale, well?) is high even though adoption is still small; calibrated as C5 alongside Megatron-LM and verl.", "sources": [ { "url": "https://github.com/NVIDIA-NeMo/RL", "shows": "GRPO/GSPO/DAPO/GDPO/SFT/DPO/distillation; DTensor + Megatron Core 6D parallelism; SGLang backend", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "NVIDIA NeMo-RL, v0.6.0 released Apr 30 2026. Scalable post-training library for LLMs/VLMs (small-scale to multi-node). Confirmed live on GitHub June 2026." }, { "product": "OpenAI Fine-Tuning API", "org": "OpenAI", "type": "software", "description": "Managed fine-tuning for GPT-4o, GPT-4o-mini, GPT-4.1, and o-series via the OpenAI API; supports supervised fine-tuning, DPO (added late 2024), and vision fine-tuning. Picked when the use case demands GPT-4-tier quality at lower latency/cost on a narrow task; the only way to fine-tune OpenAI's frontier weights. Training priced at ~$3/M tokens for GPT-4o-mini and GPT-4o (GPT-4.1 family ~$0.80-3/M); models served at standard inference rates.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-code:closed;runs-on:OpenAI-platform-only;license:Proprietary(paid API service)", "confidence": "high", "note": "Proprietary managed fine-tuning service, no source, runs only on OpenAI's platform against OpenAI's closed models.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/model-optimization", "shows": "proprietary paid fine-tuning service; SFT/DPO/RFT; gpt-4.1/4o/o4-mini tunable", "accessed": "2026-06-04" }, { "url": "https://developers.openai.com/api/docs/deprecations", "shows": "fine-tuning platform wind-down: new users blocked May 7 2026; all job creation ends Jan 6 2027; inference persists until base deprecation; existing users still create SFT/DPO (gpt-4.1) and RFT (o4-mini) jobs", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed standalone fine-tuning-API usage figure located this run (developer count / jobs run / tokens). The OpenAI platform broadly serves ~1B+ ChatGPT WAU, but that surface is not the fine-tuning API and cannot be honestly attributed to this SKU. Declining to assign a level rather than borrow the umbrella surface number.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/model-optimization", "shows": "fine-tuning offered via paid API; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods exposed: SFT, vision fine-tuning, DPO (preference), reinforcement fine-tuning (RFT) for reasoning models. Tunes frontier closed models (gpt-4.1 family, gpt-4o, o4-mini). Managed end-to-end (data validation, training, hosting, eval).", "confidence": "medium", "note": "Strong managed method coverage (SFT+DPO+RFT) over frontier base models, but black-box: no parallelism/precision/scale knobs exposed, no MLPerf-Training basis. Scored 4 on method breadth + frontier base quality; not 5 since the user cannot control training scale/internals.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/model-optimization", "shows": "SFT, vision FT, DPO, RFT across gpt-4.1/4o/o4-mini", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "OpenAI model-optimization / fine-tuning API. Methods: SFT, vision fine-tuning, DPO, reinforcement fine-tuning (RFT, reasoning models only). Tunable models incl. gpt-4.1 / -mini / -nano (SFT,DPO), gpt-4o (vision), o4-mini (RFT). Proprietary managed service. FRESHNESS: OpenAI is winding down this fine-tuning platform (per primary deprecations page): new/never-before users blocked from creating jobs as of May 7 2026; orgs inactive 60d lose job creation Jul 2 2026; ALL customers lose ability to create new fine-tuning jobs Jan 6 2027. Inference on existing fine-tuned models persists until base-model deprecation. Product still exists and functions for existing users at scoring (June 2026) but is on a sunset path." }, { "product": "Amazon Bedrock Custom Models / Fine-Tuning", "org": "Amazon Web Services", "type": "software", "description": "Managed fine-tuning for Anthropic Claude 3 Haiku, Meta Llama, Cohere Command, Amazon Titan, and Amazon Nova models inside Bedrock, the only fully managed service offering fine-tuning of Claude weights. Picked when teams are already on AWS and need governance, VPC isolation, and Provisioned Throughput serving for the resulting custom model. GA since mid-2024; usage requires Provisioned Throughput commitment for inference.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-code:closed;runs-on:AWS-Bedrock-only;license:Proprietary(managed AWS service, token+storage billed)", "confidence": "high", "note": "Proprietary managed customization service inside AWS Bedrock, no source, runs only on AWS against Bedrock-hosted foundation models.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/custom-models.html", "shows": "supervised fine-tuning, reinforcement fine-tuning, distillation; managed AWS service; token/storage pricing", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed usage figure for the Bedrock customization feature located this run (jobs run / customers using fine-tuning). Bedrock overall is widely used, but no honest signal specific to the custom-models/fine-tuning capability was found; declining to assign a level.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/custom-models.html", "shows": "managed customization feature; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: supervised fine-tuning, reinforcement fine-tuning (custom Lambda reward functions, real-time metrics), distillation (teacher->student). Continued pre-training also offered on select models.", "confidence": "medium", "note": "Solid managed method set (SFT + RFT + distillation), but a black-box service with no exposed parallelism/precision/scale controls and no benchmark basis. Scored 3: capable managed FT across a model catalog, below the RFT-on-frontier-reasoning depth of the OpenAI/Vertex offerings and far below the OSS scale-frontier libs.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/custom-models.html", "shows": "SFT, reinforcement fine-tuning with Lambda reward funcs, distillation", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Amazon Bedrock model customization (docs live June 2026). Methods: supervised fine-tuning, reinforcement fine-tuning (Lambda-defined reward functions), and distillation. Customizes Bedrock foundation models. Proprietary managed AWS service." }, { "product": "Vertex AI Tuning", "org": "Google Cloud", "type": "software", "description": "Managed supervised fine-tuning and preference tuning for Gemini 2.0/2.5 Flash and Pro inside Google Cloud Vertex AI, covers text, image, audio, and document tuning with as few as 100 labeled examples. Picked when teams need Gemini-tier quality customized to a domain and want GCP-native data residency, IAM, and Vertex pipeline integration. The only path to fine-tune Gemini weights.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-code:closed;runs-on:Google-Cloud-Vertex-AI-only;license:Proprietary(managed GCP service)", "confidence": "high", "note": "Proprietary managed tuning service inside Vertex AI; no source, runs only on Google Cloud against closed Gemini models.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/tune-models", "shows": "supervised fine-tuning + preference tuning for Gemini; managed GCP service", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed usage figure for the Vertex Gemini tuning feature located this run. The Gemini app surface has 750M+ MAU but that is not the Vertex tuning service and cannot be attributed to this SKU; declining to assign a level rather than borrow the surface number.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/tune-models", "shows": "managed Gemini tuning feature; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: supervised fine-tuning and preference tuning (DPO-style) for Gemini; multimodal tuning across text/document/image/audio/video plus function-calling tuning; checkpoint management.", "confidence": "medium", "note": "Broad managed multimodal tuning (SFT + preference + function-calling, across modalities) over frontier closed Gemini models. Black-box (no scale/precision controls, no benchmark basis), so capped below the OSS scale definers; scored 4 on method+modality breadth, comparable to the OpenAI FT API.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/tune-models", "shows": "SFT + preference tuning, multimodal + function-calling tuning for Gemini", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Google Cloud Vertex AI tuning for Gemini (docs live June 2026). Methods: supervised fine-tuning and preference tuning, across text/document/image/audio/video and function-calling. Proprietary managed Google Cloud service." }, { "product": "Mosaic AI Model Training", "org": "Databricks", "type": "software", "description": "Databricks' managed full-fine-tune and continued-pretraining service for open-weight models (Llama 3, Mistral, DBRX) built on the proprietary MosaicML training stack, claimed up to 2x faster than naive open source. Picked when training data lives in Unity Catalog and teams want Lakehouse-native fine-tuning with full weight ownership (resulting model registers to UC). Distinct from competitors in that customers get the actual weight checkpoint, not just a hosted endpoint.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-code:closed;runs-on:Databricks-workspace-only(AWS regions);license:Proprietary(managed service); base weights:Llama-Community-License(non-OSI)", "confidence": "high", "note": "Proprietary managed FT service running only inside a Databricks workspace; no source. (Databricks' separate LLM Foundry / Composer libs are OSS but are not this product.)", "sources": [ { "url": "https://docs.databricks.com/aws/en/large-language-models/foundation-model-training/", "shows": "managed FT service via databricks_genai SDK; deprecated, removal Aug 14 2026; Llama-only base models", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed usage figure for the FMFT feature located this run (jobs run / customers fine-tuning). Databricks overall is widely used, but no honest signal specific to this FT capability was found; declining to assign a level. Feature is also being sunset (removal Aug 14 2026), which further depresses any forward adoption.", "sources": [ { "url": "https://docs.databricks.com/aws/en/large-language-models/foundation-model-training/", "shows": "managed FT feature; deprecation notice; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Methods: chat-completion FT, instruction FT, continued pre-training. No exposed LoRA/QLoRA/DPO/RLHF or preference methods; no parallelism/precision/scale knobs. Base models limited to the Meta Llama family.", "confidence": "medium", "note": "Narrow managed method set (SFT + continued pretraining only, no preference/RL methods) over a single base-model family, and a black-box service. Scored 2: below the OpenAI/Vertex/Together managed offerings (SFT+DPO+RFT / 100B+ models) and far below the OSS scale definers. Being deprecated also caps forward capability investment.", "sources": [ { "url": "https://docs.databricks.com/aws/en/large-language-models/foundation-model-training/", "shows": "chat-completion FT, instruction FT, continued pre-training; Llama-only", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Databricks Foundation Model Fine-Tuning (now Databricks Model Training), accessed via the databricks_genai SDK. Confirmed live in docs June 2026 BUT marked deprecated with scheduled removal Aug 14 2026. Methods: chat-completion FT, instruction FT, continued pre-training. Base models: Meta Llama 3.1/3.2/3.3 family only. Proprietary managed Databricks service." }, { "product": "Azure OpenAI Fine-Tuning", "org": "Microsoft Azure", "type": "software", "description": "Azure-managed fine-tuning of GPT-4o, GPT-4o-mini, GPT-4.1, and GPT-3.5 with enterprise controls (VNet, private endpoints, RBAC, content filters) that the raw OpenAI API doesn't expose. Picked by enterprises that already standardize on Azure for governance and data residency. Same underlying training methods as OpenAI's API (SFT, DPO) but billed through Azure and deployed to Azure-managed endpoints.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-code:closed;runs-on:Azure-AI-Foundry-only;license:Proprietary(managed Azure service)", "confidence": "high", "note": "Proprietary managed FT service inside Azure AI Foundry; no source, runs only on Azure against (mostly) closed OpenAI models.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/fine-tuning", "shows": "managed FT service; SFT/DPO/RFT; gpt-4o/4.1/o4-mini/gpt-5 tunable; Azure-only", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed standalone usage figure for the Azure OpenAI fine-tuning feature located this run (jobs run / customers tuning). Azure OpenAI overall is widely deployed in enterprise, but no honest signal specific to the FT SKU was found; declining to assign a level rather than borrow the umbrella surface number.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/fine-tuning", "shows": "GA managed FT feature; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: SFT, DPO (preference), RFT (model-grader reward signals) across the gpt-4o/gpt-4.1/o4-mini/gpt-5 frontier line, plus preview SFT on Ministral/Qwen/Llama/gpt-oss. Managed end-to-end (data validation, per-epoch checkpoints, auto-deploy); Standard/Global/Developer training tiers.", "confidence": "medium", "note": "Strong managed method coverage (SFT+DPO+RFT) over frontier closed base models plus an open-model preview lane, but black-box: no parallelism/precision/scale controls, no MLPerf-Training basis. Scored 4 on method breadth + frontier base quality, calibrated equal to the OpenAI FT API (C4) it wraps; not 5 since the user cannot control training internals/scale.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/fine-tuning", "shows": "SFT, DPO, RFT across gpt-4o/4.1/o4-mini/gpt-5 + Qwen/Llama/gpt-oss preview", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Azure OpenAI fine-tuning via Azure AI Foundry (docs live, GA, June 2026). Methods: SFT, DPO, RFT (reinforcement fine-tuning). Tunable: gpt-4o-mini/gpt-4o (SFT+DPO), gpt-4.1/-mini/-nano (SFT+DPO), o4-mini (RFT), gpt-5 (RFT, gated/invite). Preview: Ministral-3B, Qwen-32B, Llama-3.3-70B-Instruct, gpt-oss-20b (SFT). Proprietary managed Microsoft Azure service." }, { "product": "Together Fine-Tuning", "org": "Together AI", "type": "software", "description": "Managed fine-tuning across 50+ open-weight models (Llama, Mistral, Qwen, DeepSeek) supporting LoRA, full-parameter SFT, tool-call/reasoning-trace/VLM tuning, and models up to 1T params; resulting checkpoints served at base-model inference rates. Differentiates from Fireworks with full fine-tuning (not just LoRA) and broader catalog. Together AI raised $305M Series B at $3.3B valuation (Feb 2025), hit $1B ARR by Feb 2026, with Salesforce, Zoom, and The Washington Post among 450K+ developer accounts.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-pipeline:closed;runs-on:Together-cloud-only;license:Proprietary(managed service, per-token training pricing); base weights:open-models(varies)", "confidence": "high", "note": "The fine-tuning service/pipeline is proprietary and runs only on Together's cloud; no source. (It tunes open-weight base models, but the offering itself is closed.)", "sources": [ { "url": "https://docs.together.ai/docs/fine-tuning-overview", "shows": "managed FT service (LoRA/full/preference/vision); data-prep to hosting; proprietary cloud", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed standalone usage figure for the Together fine-tuning feature located this run (jobs run / customers tuning). Together AI raised a $305M Series B and serves substantial inference traffic, and a vendor case study cites a customer moving to daily iteration / 77%->87% accuracy, but no hard per-SKU FT usage number was found; declining to assign a level rather than rely on a single anecdote.", "sources": [ { "url": "https://docs.together.ai/docs/fine-tuning-overview", "shows": "managed FT feature; no aggregate usage figures disclosed", "accessed": "2026-06-04" }, { "url": "https://www.together.ai/fine-tuning", "shows": "FT product page; case-study traction (daily iteration, 77%->87% accuracy)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: LoRA + full fine-tuning, preference (DPO), function-calling, reasoning, vision-language FT; demonstrated tuning of 100B+ open models (DeepSeek-V3, Qwen3-235B). Managed end-to-end (prep -> train -> dedicated-endpoint serve).", "confidence": "medium", "note": "Broad managed method coverage (LoRA + full FT + preference + multimodal) over large open models incl. demonstrated 100B+/235B-scale tuning - the widest open-model scale among the managed services here. Black-box (limited scale/precision control), no MLPerf-Training basis, so capped below OSS scale definers; scored 4, on par with the OpenAI/Azure FT APIs.", "sources": [ { "url": "https://docs.together.ai/docs/fine-tuning-overview", "shows": "LoRA, full FT, preference, function-calling, reasoning, vision-language FT", "accessed": "2026-06-04" }, { "url": "https://www.together.ai/fine-tuning", "shows": "fine-tune 100B+ models incl. DeepSeek-V3 and Qwen3-235B", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Together AI managed fine-tuning (docs live June 2026). Methods: LoRA, full fine-tuning, preference (DPO), function-calling, reasoning, and vision-language FT. Can fine-tune 100B+ open models incl. DeepSeek-V3 and Qwen3-235B; handles data prep -> training -> dedicated-endpoint hosting. Proprietary managed service over open base models." }, { "product": "Fireworks Fine-Tuning", "org": "Fireworks AI", "type": "software", "description": "Managed LoRA fine-tuning for popular open-weight models (Llama, Mistral, Qwen, DeepSeek) with served checkpoints at base-model inference rates and tight integration to Fireworks' high-throughput inference engine. Picked when latency-sensitive production serving matters as much as the fine-tune itself; Fireworks consistently benchmarks at the low end of TTFT for hosted open-weight models. Pricing from $0.50/M training tokens for models up to 16B.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-pipeline:closed;runs-on:Fireworks-cloud-only;license:Proprietary(managed service); base weights:open-models(varies)", "confidence": "high", "note": "The managed FT service/pipeline is proprietary and runs only on Fireworks' cloud; no source. (Fireworks' separate reward-kit RFT toolkit is OSS but was not confirmed as part of this product this run.)", "sources": [ { "url": "https://docs.fireworks.ai/fine-tuning/fine-tuning-models", "shows": "managed FT service (SFT/DPO/RFT); LoRA outputs; dedicated-deployment serving; proprietary cloud", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed standalone usage figure for the Fireworks fine-tuning feature located this run (jobs run / customers tuning). Fireworks serves notable inference volume, but no honest signal specific to the FT SKU was found; declining to assign a level.", "sources": [ { "url": "https://docs.fireworks.ai/fine-tuning/fine-tuning-models", "shows": "managed FT feature; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: SFT, DPO (preference), RFT (reinforcement fine-tuning). LoRA-adapter outputs with warm-start continued training; serves resulting adapters on dedicated deployments. Open base-model catalog.", "confidence": "medium", "note": "Strong managed method coverage (SFT+DPO+RFT, incl. reinforcement FT) over open base models. Black-box (LoRA-output oriented, limited scale/precision control), no MLPerf-Training basis, so capped below OSS scale definers; scored 4, on par with the other RFT-capable managed services. Slightly narrower demonstrated scale than Together (no confirmed 100B+ full-FT claim), but matching RFT depth keeps it at 4.", "sources": [ { "url": "https://docs.fireworks.ai/fine-tuning/fine-tuning-models", "shows": "SFT, DPO, RFT; LoRA outputs + warm-start", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Fireworks AI managed fine-tuning (docs live June 2026). Methods: SFT, DPO, RFT (reinforcement fine-tuning). Output produces LoRA models; warm-start continued training supported; fine-tuned models served on on-demand (dedicated) deployments only. Proprietary managed service over open base models." }, { "product": "Mistral Fine-Tuning API", "org": "Mistral AI", "type": "software", "description": "Managed LoRA fine-tuning of Mistral 7B, Mistral Small, and other Mistral models via La Plateforme, plus an open source companion (mistral-finetune) for self-hosted training. Picked when teams want Mistral's European data residency story or open-weight portability between managed and self-hosted training. API priced at ~$2/M training tokens; the only managed offering for Mistral's commercial weights.", "openness": { "score": 1, "class": "closed", "components": "source:closed;training-pipeline:closed;runs-on:La-Plateforme-only;license:Proprietary(managed service, per-token + storage pricing); base weights:Mistral-models(open-weight variants exist)", "confidence": "high", "note": "The hosted FT API on La Plateforme is a proprietary managed service (LoRA adapters under the hood); no source for the service. Mistral separately ships an OSS mistral-finetune SDK for self-hosting, but that is a distinct product from this managed API. Verified 2026-06-04: the FT API remains a live production service on La Plateforme (changelog + platform docs); only the legacy docs guide was relocated to /resources/deprecated/, which carries a deprecation banner but does not retire the API.", "sources": [ { "url": "https://mistral.ai/news/customization", "shows": "serverless FT on La Plateforme; LoRA adapters under the hood; managed proprietary service; separate OSS mistral-finetune SDK", "accessed": "2026-06-04" }, { "url": "https://docs.mistral.ai/resources/deprecated/finetuning", "shows": "legacy fine-tuning DOCS guide moved to /resources/deprecated/ (Classifier Factory + text/vision SFT documented); corroborates that only the docs guide moved while the managed FT API on La Plateforme remains a live production service", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed standalone usage figure for the Mistral fine-tuning API located this run (jobs run / customers tuning). La Plateforme is a widely used developer platform, but no honest per-SKU FT signal was found; declining to assign a level.", "sources": [ { "url": "https://mistral.ai/news/customization", "shows": "managed FT feature on La Plateforme; no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: LoRA-adapter supervised fine-tuning (text & vision) + Classifier Factory for classification models. Tunable base models limited to Mistral's own (Small/Medium tier). No exposed full-FT/DPO/RFT or scale/precision knobs in the managed API.", "confidence": "medium", "note": "Narrower managed method set than the SFT+DPO+RFT services - LoRA SFT + classifier factory only, over Mistral's own models, black-box. Scored 3: a real, capable managed LoRA-FT service but below the preference/RL-capable OpenAI/Azure/Together/Fireworks offerings (C4) and far below the OSS scale definers.", "sources": [ { "url": "https://mistral.ai/news/customization", "shows": "LoRA SFT (text & vision) + Classifier Factory; Mistral Small/Medium base", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Mistral serverless fine-tuning on La Plateforme - confirmed live/available in 2026 (not deprecated; only the legacy docs guide moved to /resources/deprecated/). Methods: LoRA-adapter SFT (text & vision) + Classifier Factory. Tunable: Mistral Small / Medium (originally 7B + Small). $4 min job fee + $2/mo storage per model. Proprietary managed service. (Separate mistral-finetune OSS SDK exists for self-hosting - not this product.)" }, { "product": "Predibase", "org": "Predibase", "type": "software", "description": "Predibase managed fine-tuning/serving platform, acquired by Rubrik June 25 2025 (now an AI Agent Operations platform under Rubrik). Pioneered the first end-to-end Reinforcement Fine-Tuning (RFT) platform for LLMs. Methods: SFT, RFT (GRPO-style), continued pretraining; fine-tunes open source models; LoRA eXchange (LoRAX) OSS serving + turbo serving engine. Proprietary managed SaaS/VPC platform (OSS LoRAX + Ludwig are separate components). Confirmed live June 2026.", "openness": { "score": 2, "class": "open_core", "components": "core-platform:closed(proprietary post-training stack + turbo serving);OSS-components:LoRAX(open source multi-LoRA serving)+Ludwig(open source);runs-on:Predibase-SaaS-or-customer-VPC;license:Proprietary(managed) with Apache-licensed LoRAX/Ludwig adjuncts", "confidence": "high", "note": "Open-core: the founders created and maintain OSS LoRAX (multi-LoRA serving) and Ludwig, but the Predibase fine-tuning/RFT platform itself is proprietary managed SaaS. Scored 2 (open-core leaning closed) because the FT product - the thing being scored - is the proprietary platform, with OSS only at the serving/declarative-ML edges.", "sources": [ { "url": "https://docs.predibase.com/", "shows": "managed platform to fine-tune & serve open source LLMs; SFT/RFT/continued-pretraining; built by LoRAX creators; SaaS/VPC", "accessed": "2026-06-04" }, { "url": "https://www.rubrik.com/company/newsroom/press-releases/25/rubrik-to-acquire-predibase-to-accelerate-agentic-ai-adoption", "shows": "Rubrik acquisition (Jun 25 2025); proprietary post-training stack + turbo serving + OSS LoRA eXchange", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "low", "note": "No hard per-platform usage figure (jobs / customers) disclosed this run. Reported traction: enterprise post-training/agent platform now distributed under Rubrik (acquisition valued $100M-$500M per press), founder-pedigree (Google/Uber), and a widely-adopted OSS serving layer (LoRAX). Placed at 2 on reported enterprise traction, not a verified usage_volume number; low confidence pending hard data.", "sources": [ { "url": "https://techcrunch.com/2025/06/25/rubrik-acquires-predibase-to-accelerate-adoption-of-ai-agents/", "shows": "Rubrik acquires Predibase (Jun 25 2025) to scale enterprise AI-agent/fine-tuning adoption", "accessed": "2026-06-04" }, { "url": "https://docs.predibase.com/", "shows": "production private serverless endpoints; SaaS/VPC enterprise deployment", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: supervised fine-tuning, reinforcement fine-tuning (first end-to-end RFT platform for LLMs, GRPO-style; learns from outcomes/feedback without heavy data prep), continued pretraining; LoRA + LoRAX multi-adapter serving (hundreds of fine-tuned models per GPU); turbo serving (~2x). Tunes open source base models.", "confidence": "medium", "note": "Strong managed method depth - notably the pioneer of end-to-end RFT for LLMs - plus efficient multi-LoRA serving over open models. Scored 4: matches the RFT-capable managed services (OpenAI/Azure/Fireworks at C4) and adds best-in-class multi-adapter serving, but is still a black-box managed platform without exposed large-scale training internals, so below the OSS scale-frontier definers (Megatron-LM C5).", "sources": [ { "url": "https://www.rubrik.com/blog/ai/25/introducing-reinforcement-fine-tuning-on-predibase", "shows": "first end-to-end reinforcement fine-tuning platform for LLMs (per Rubrik; corroborated by acquisition coverage)", "accessed": "2026-06-04" }, { "url": "https://docs.predibase.com/", "shows": "SFT, RFT, continued pretraining; LoRAX multi-adapter serving; turbo serving 2x", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Predibase managed fine-tuning/serving platform, acquired by Rubrik June 25 2025 (now an AI Agent Operations platform under Rubrik). Pioneered the first end-to-end Reinforcement Fine-Tuning (RFT) platform for LLMs. Methods: SFT, RFT (GRPO-style), continued pretraining; fine-tunes open source models; LoRA eXchange (LoRAX) OSS serving + turbo serving engine. Proprietary managed SaaS/VPC platform (OSS LoRAX + Ludwig are separate components). Confirmed live June 2026." }, { "product": "OpenPipe", "org": "OpenPipe", "type": "software", "description": "OpenPipe ships the open source Agent Reinforcement Trainer (ART, Apache-2.0, ~9.7K stars) alongside a closed managed prompt-logging and fine-tuning platform, an open-core stack for SFT and RL post-training of agents. Acquired by CoreWeave in September 2025 to anchor their RL-for-agents capability alongside Weights & Biases; ART continues active development as the open library, with the platform serving as the commercial wrapper.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI, OpenPipe Inc.; some vendored code LGPL-3.0);source:public;methods:RL(GRPO)+SFT trajectory training;managed-tier:OpenPipe hosted training/serving SaaS on top of the OSS lib", "confidence": "high", "note": "Core ART library is OSI-licensed (Apache-2.0) and fully public; OpenPipe's revenue product is a managed training/hosting platform layered on top, so open_core.", "sources": [ { "url": "https://github.com/OpenPipe/ART/blob/main/LICENSE", "shows": "Apache-2.0 license text (OpenPipe, Inc. 2025; LGPL-3.0 notice for vendored portions)", "accessed": "2026-06-04" }, { "url": "https://github.com/OpenPipe/ART", "shows": "ART = open source GRPO RL trainer; v0.5.17 release; active dev", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "openpipe-art PyPI ~47.7k downloads last month; 9.9k GitHub stars (corroborating only). Specialist RL-fine-tuning audience; 10K-100K monthly-download band.", "sources": [ { "url": "https://pypistats.org/packages/openpipe-art", "shows": "~47,684 downloads last month for openpipe-art", "accessed": "2026-06-04" }, { "url": "https://github.com/OpenPipe/ART", "shows": "9.9k stars (secondary corroboration)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: multi-step agentic RL via GRPO + SFT; integrates with vLLM/Unsloth backends; supports Qwen, Llama, GPT-OSS. Specialist agent-RL focus, not broad-method or large-scale-parallelism training infra.", "confidence": "medium", "note": "Capable, focused agent-RL trainer; mid-tier vs full training stacks (no TP/PP/EP at Megatron scale). Feature-matrix basis per recipe; no MLPerf-Training submission.", "sources": [ { "url": "https://github.com/OpenPipe/ART", "shows": "GRPO RL trainer, vLLM/Unsloth integration, Qwen/Llama/GPT-OSS support", "accessed": "2026-06-04" } ] }, "maturity": 2.5, "mature": false, "version_note": "Scored on ART (Agent Reinforcement Trainer), OpenPipe's open source fine-tuning/RL library (GRPO). v0.5.17 (Mar 13, 2026), confirmed live on GitHub June 2026. OpenPipe also runs a managed training/hosting platform on top of the OSS library." }, { "product": "Lamini", "org": "AMD", "type": "software", "description": "Enterprise fine-tuning stack acquired by AMD at Advancing AI 2025 (June 2025) and rolled into AMD's ROCm/AI software organization; the former CEO Sharon Zhou is now VP of AI at AMD. Picked when teams running on AMD Instinct MI300/MI325 accelerators want a first-party fine-tuning path optimized for ROCm; Lamini's Mixture of Memory Experts (MoME) technique trains thousands of LoRA-style memory experts per model to drive ~95% factual recall with ~10x fewer hallucinations than vanilla SFT. The discrete lamini.ai SaaS has been wound down; the technology now anchors AMD's enterprise fine-tuning story alongside ROCm 6.x and AMD's growing data-center AI revenue ramp.", "openness": { "score": 1, "class": "closed", "components": "engine:closed(hosted memory-tuning/fine-tuning service on Lamini/AMD servers);client-sdk:open(Apache-2.0, lamini-ai/lamini);data:closed;training-code:closed;access:API-key-gated", "confidence": "high", "note": "Proprietary managed fine-tuning service; the only open artifact is a thin API client SDK, so the product itself is closed.", "sources": [ { "url": "https://github.com/lamini-ai/lamini", "shows": "repo is the Lamini API client/SDK (Apache-2.0); engine runs on hosted service, API credentials required", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No verified user/usage/download figure for the hosted platform; post-AMD-acquisition integration into AMD's AI stack obscures any standalone signal. Declining to assign a level rather than infer from press.", "sources": [ { "url": "https://github.com/lamini-ai/lamini", "shows": "hosted-service model, no published usage figures on the primary channel", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "feature_matrix", "value": null, "confidence": "low", "note": "Vendor markets memory-tuning (claimed ~95% factual accuracy, 32x compression) but these are unverified vendor claims with no independent benchmark; closed engine is not assessable on the recipe's feature matrix. Not scoring rather than fabricate.", "sources": [ { "url": "https://github.com/lamini-ai/lamini", "shows": "client SDK only; engine internals/benchmarks not independently verifiable", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Lamini enterprise LLM fine-tuning/inference platform; acquired by AMD June 2025 (founders incl. Sharon Zhou joined AMD). The fine-tuning/memory-tuning engine runs on Lamini's hosted servers; only the Python client SDK (github.com/lamini-ai/lamini, Apache-2.0) is open. Confirmed via repo + coverage June 2026." }, { "product": "Anyscale Fine-Tuning", "org": "Anyscale", "type": "software", "description": "Ray-based managed fine-tuning platform that wraps LLMForge and ray-llm to run SFT, continued pretraining, LoRA, and preference optimization (DPO, KTO, ORPO, PPO) across Llama-3, Mistral, Mixtral, and other open-weight families, then deploys the resulting checkpoints with LoRA multiplexing on the same cluster. Picked when teams want the gravity of the Ray ecosystem (Anyscale's founders created Ray at UC Berkeley's RISELab in 2019) but as a managed offering rather than self-hosted infra. Anyscale raised a $100M Series C at a $1B valuation in 2021 and has reached ~$281M total raised; Ray itself powers training and serving workloads at Uber, Pinterest, Spotify, OpenAI, Cohere, and Instacart, with Anyscale capturing the managed-platform tier above it.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(Anyscale-platform-only);source:not-public;built-on:Ray+DeepSpeed+HF-Accelerate(OSS deps) but LLMForge itself not redistributable;methods:LoRA+full-FT;status:deprecated", "confidence": "high", "note": "LLMForge was a proprietary Anyscale-platform-only library (not on GitHub); now deprecated in favor of OSS Axolotl/LLaMA-Factory. Closed, and effectively end-of-life.", "sources": [ { "url": "https://www.anyscale.com/library/llmforge", "shows": "LLMForge deprecated; migrate to LLaMA-Factory; Anyscale-only library", "accessed": "2026-06-04" }, { "url": "https://docs.anyscale.com/llms/finetuning/intro/", "shows": "fine-tuning now via OSS libs on Ray/Anyscale, LLMForge consolidation", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Anyscale-platform-only and now deprecated; no standalone download/user signal exists and the product is being retired. Null rather than inferred.", "sources": [ { "url": "https://www.anyscale.com/library/llmforge", "shows": "deprecation notice, no usage figures, migration path away from LLMForge", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Methods: LoRA + full-parameter FT; YAML config; scaling via Ray Train/Ray Data + DeepSpeed/Accelerate. No advanced quantization/algorithm coverage; Anyscale itself cites Axolotl/LLaMA-Factory as providing enhanced functionality, the reason for deprecation.", "confidence": "medium", "note": "Modest method/feature coverage relative to live OSS peers; Anyscale's own deprecation rationale (Axolotl/LLaMA-Factory have more features) caps this low.", "sources": [ { "url": "https://www.anyscale.com/library/llmforge", "shows": "LoRA + full FT; OSS alternatives offer quantization/advanced algorithms", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "LLMForge = Ray library for LLM fine-tuning, available only on the Anyscale managed platform (not a public OSS project). As of 2026 Anyscale has DEPRECATED LLMForge and consolidated around open source Axolotl / LLaMA-Factory with native Ray support. Confirmed via Anyscale docs June 2026." }, { "product": "Snowflake Cortex Fine-Tuning", "org": "Snowflake", "type": "software", "description": "Serverless fine-tuning function inside Snowflake Cortex AI for Llama 3 (8B/70B), Llama 3.1 (8B/70B), Mistral 7B, and Mixtral 8x7B. Training and inference both run on Snowflake compute without exporting data to a third-party platform. Picked when training data already lives in Snowflake tables and the value of in-perimeter governance (Unity-style RBAC, audit trails, region pinning across AWS US-West-2/US-East-1/EU-Central-1 and Azure East US 2) outweighs catalog breadth. GA on 2025-02-07; billed per-token in Snowflake credits with no Provisioned Throughput required to serve the tuned model, a differentiator versus Bedrock.", "openness": { "score": 1, "class": "closed", "components": "service:fully-managed-proprietary(in-Snowflake);source:closed;method:PEFT(LoRA-class, not specified);base-models:fixed allowlist;access:Snowflake-account+credits;data:user-data stays in account but engine closed", "confidence": "high", "note": "Proprietary fully-managed SaaS fine-tuning feature; no source, no self-host. Closed.", "sources": [ { "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-finetuning", "shows": "fully managed proprietary PEFT service, fixed base-model list, credit-based pricing, GA", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No published usage/customer-count figure for the fine-tuning feature specifically; Snowflake has a large data-platform install base but no honest signal for this SKU's fine-tuning adoption. Null.", "sources": [ { "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-finetuning", "shows": "GA service with usage-history view but no disclosed adoption numbers", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Method: PEFT (single LoRA-class technique) on a fixed allowlist of 6 base models (Llama 3/3.1, Mistral/Mixtral); no full FT, no RLHF/DPO/GRPO, no user-chosen models/scale. Convenience-first managed feature, narrow method/model coverage.", "confidence": "medium", "note": "Narrow capability vs the recipe's feature matrix (single PEFT method, fixed older base models, no advanced RL methods). Integrated convenience, not a frontier trainer.", "sources": [ { "url": "https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-finetuning", "shows": "PEFT only, 6 fixed base models, no full-FT/RL methods listed", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Snowflake Cortex Fine-tuning, fully managed, generally-available PEFT service inside Snowflake. Fine-tunes a fixed set of base models (llama3-8b/70b, llama3.1-8b/70b, mistral-7b, mixtral-8x7b). Token-priced. Confirmed live via Snowflake docs June 2026." }, { "product": "Atropos", "org": "Nous Research", "type": "software", "description": "Language-model reinforcement-learning environments framework for collecting and evaluating LLM trajectories across diverse environments. Nous positions it as part of the post-training stack, and the GitHub repo has 1.2K stars, 362 forks, and active 2026 development, which makes it a meaningful open source training primitive.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;scope:async RL-environments framework + trajectory API; trainer integrations:Axolotl/Tinker; no feature-gated core", "confidence": "high", "note": "Fully OSI-licensed (MIT) and public, no managed/closed core. Fits finetuning_code as the RL-training-environment layer (RLHF/RLAIF/GRPO).", "sources": [ { "url": "https://github.com/NousResearch/atropos", "shows": "MIT license; LLM RL-environments framework; v0.4.0 (Mar 10 2026); Axolotl/Tinker trainer integrations", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "stars_fallback", "confidence": "low", "note": "No PyPI/download or named-user count found on the primary channel; only 1.3k GitHub stars as a signal (stars_fallback => capped <=3, placed at 1). Early/niche research adoption.", "sources": [ { "url": "https://github.com/NousResearch/atropos", "shows": "1.3k stars; no published download/user figures", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Methods: RL training environments (RLHF/RLAIF, GRPO-style) with async-first microservice design; broad environment coverage (dataset eval GSM8K/MMLU, games, code execution, multimodal); OpenAI-compatible endpoints; debugging tooling (view-run/process). Trainer-agnostic via Axolotl/Tinker.", "confidence": "medium", "note": "Capable, well-architected RL-environments layer; mid-tier as a fine-tuning enabler (it orchestrates RL rather than implementing large-scale training itself). Vendor-reported gains (4.6x parallel-task, 2.5x finance) not independently benchmarked.", "sources": [ { "url": "https://github.com/NousResearch/atropos", "shows": "async RL-environments framework, diverse environments, RLHF/RLAIF support, debugging tools", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "Atropos, Nous Research LLM RL-environments framework for collecting/evaluating trajectories and driving RL training (RLHF/RLAIF, GRPO-style). v0.4.0 (Mar 10, 2026), confirmed live on GitHub June 2026. Integrates with trainers Axolotl and Tinker. Borderline: it is the RL-environment/orchestration layer that feeds a fine-tuning trainer." }, { "product": "Tinker", "org": "Thinking Machines Lab", "type": "software", "description": "Managed training API for researchers that lets teams control the training loop while Thinking Machines handles the infrastructure. It exposes four core primitives (forward_backward, optim_step, sample, and save_state) and supports frontier open-weight models including Qwen, GPT-OSS, Llama, DeepSeek, and Nemotron. Scale signal: Thinking Machines announced a multi-year, gigawatt-scale strategic partnership with NVIDIA and said NVIDIA made a significant investment to support the company’s long-term growth.", "openness": { "score": 1, "class": "closed", "components": "engine:closed(hosted distributed-training service on TML infra);cookbook:open(Apache-2.0, thinking-machines-lab/tinker-cookbook);api:public-but-proprietary(forward_backward/optim_step/sample/save_state);method:LoRA(+RL via sample);access:paid+account-gated", "confidence": "high", "note": "Core training service is a proprietary hosted API; only the example cookbook is OSI-licensed, so the product is closed. Not open_core since no self-hostable engine exists.", "sources": [ { "url": "https://thinkingmachines.ai/tinker/", "shows": "paid managed LoRA training API, usage pricing, supported models 1B-550B", "accessed": "2026-06-04" }, { "url": "https://github.com/thinking-machines-lab/tinker-cookbook/blob/main/LICENSE", "shows": "Apache-2.0 license for the (only) open component, the cookbook", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "low", "note": "Early named research adopters (Princeton Goedel, Stanford Rotskoff, Berkeley SkyRL, Redwood Research) and moved from private beta to paid GA; no disclosed user/usage count. Reported research traction => conservative level 2.", "sources": [ { "url": "https://thinkingmachines.ai/tinker/", "shows": "paid GA service with named research adopters; no published user count", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Methods: LoRA training + sampling for RL/eval; broad model breadth (1B-550B incl. large MoE like Qwen3-235B-A22B, DeepSeek, Nemotron); abstracts distributed multi-GPU training behind 4-function API. Largest demonstrated: 550B-class MoE fine-tuning.", "confidence": "medium", "note": "Strong fine-tuning capability (frontier-scale model breadth and managed distributed training) but LoRA-centric (not full FT / full RL stack), so 4 not 5. Feature-matrix basis; no MLPerf-Training submission.", "sources": [ { "url": "https://thinkingmachines.ai/tinker/", "shows": "LoRA training across Qwen/Llama/DeepSeek/GPT-OSS/Moonshot/Nemotron, 1B-550B, MoE support", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Tinker, Thinking Machines Lab's managed LoRA fine-tuning/training API (announced Oct 1 2025; now paid, usage-priced per-million-tokens + $0.10/GB-mo storage). Fine-tunes open-weight models 1B-550B (Qwen, Llama, DeepSeek, GPT-OSS, Moonshot, Nemotron incl. Qwen3-235B-A22B). Only the tinker-cookbook (Apache-2.0) is open; the training engine is a hosted proprietary service. Confirmed live June 2026." }, { "product": "Megatron-LM", "org": "NVIDIA", "type": "software", "description": "Megatron Core 0.17.1, released May 28, 2026 (GitHub releases). Companion Megatron-Bridge 0.4.0 (Apr 2026). Confirmed to exist as of June 2026.", "openness": { "score": 5, "class": "open_source", "components": "license:BSD-3-Clause(OSI, NVIDIA core)+Apache-2.0/MIT(OSI, vendored components);source:public;restriction:attribution/change-notice only", "confidence": "high", "note": "license:BSD-3-Clause(OSI, NVIDIA core)+Apache-2.0/MIT(OSI, vendored components);source:public;restriction:attribution/change-notice only", "sources": [ { "url": "https://github.com/NVIDIA/Megatron-LM/blob/main/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/NVIDIA/Megatron-LM", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Foundational large-scale training library; directly inspired/underpins Colossal-AI, HuggingFace Accelerate, NVIDIA NeMo. Used to train models 2B-462B params across thousands of GPUs. User base is concentrated among large-model training teams (smaller absolute headcount than inference tools). 16.6k stars. Level 3 reflects specialist-but-influential reach; no clean download count verified.", "sources": [ { "url": "https://github.com/NVIDIA/Megatron-LM", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://developer.nvidia.com/megatron-core", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "State-of-the-art parallelism (TP/PP/DP/EP/CP), mixed precision FP16/BF16/FP8/FP4; benchmarked training of 462B-param model on 6144 H100 GPUs. Reference-grade large-scale transformer training infra. No MLPerf-Training submission verified for the library itself, so basis is feature_matrix plus NVIDIA-reported scaling benchmarks.", "sources": [ { "url": "https://github.com/NVIDIA/Megatron-LM", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://developer.nvidia.com/megatron-core", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Megatron Core 0.17.1, released May 28, 2026 (GitHub releases). Companion Megatron-Bridge 0.4.0 (Apr 2026). Confirmed to exist as of June 2026." } ] }, "evaluation_code": { "label": "Evaluation code", "arc": "Model components", "layer": "model_components", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "SWE-bench", "org": "Princeton NLP", "type": "software", "description": "Docker-based evaluation harness that benchmarks coding agents on 2,294 real GitHub issues from 12 popular Python repositories, producing pass-rate scores against held-out tests. Picked because it is the de-facto industry standard for measuring software-engineering agent capability. Every frontier lab (Anthropic, OpenAI, Google) reports SWE-bench Verified scores. ~5k stars on GitHub and Epoch AI maintains an authoritative leaderboard. Note: OpenAI's April 2026 audit flagged that 59.4% of the hardest unsolved problems in SWE-bench Verified had flawed test cases; they now prefer the harder SWE-bench Pro and SWE-bench Bash variants for frontier evaluation.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(GitHub);harness+inference+eval code public;no feature-gated core", "confidence": "high", "note": "MIT-licensed harness, full source public; OSI -> open_source.", "sources": [ { "url": "https://pypi.org/project/swebench/", "shows": "official SWE-bench package, MIT license, v4.1.0", "accessed": "2026-06-04" }, { "url": "https://github.com/princeton-nlp/SWE-bench", "shows": "MIT license, public harness source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "SWE-bench Verified is the de-facto-standard agentic-coding eval cited in essentially every 2025-26 frontier release report (GPT-5 74.9%, DeepSeek-V4-Pro 80.6%, Qwen3.6, Kimi K2.6) -- the top adoption signal for a harness per recipe. pypistats reports ~31.5M swebench downloads/mo but that figure is almost certainly CI/benchmark-runner-inflated and not human installs, so used as corroboration only, not the headline; level set on standard-status, capped at 4.", "sources": [ { "url": "https://pypistats.org/packages/swebench", "shows": "~31.5M monthly PyPI downloads (likely CI-inflated; corroboration only)", "accessed": "2026-06-04" }, { "url": "https://github.com/princeton-nlp/SWE-bench", "shows": "ICLR 2024 oral; harness used across model evaluations", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "agentic-coding eval standard: real GitHub-issue patch generation + test-based verification; Verified/Lite/Multimodal/Pro splits; broad model-backend support; reference implementation for the most-cited coding benchmark", "confidence": "high", "note": "Frontier-definer within evaluation_code for agentic/coding eval -- the reference harness for the benchmark every lab reports against.", "sources": [ { "url": "https://github.com/princeton-nlp/SWE-bench", "shows": "harness runs models on real GitHub issues with test-based verification; multimodal split", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "SWE-bench evaluation harness. PyPI `swebench` v4.1.0 (Sep 11 2025); repo migrated to github.com/SWE-bench/SWE-bench, last headline feature SWE-bench Multimodal (Jan 2025). Confirmed live June 2026. Scoring the HARNESS (the software that runs models against GitHub-issue tasks); the underlying dataset belongs in benchmark_eval_data." }, { "product": "lm-evaluation-harness", "org": "EleutherAI", "type": "software", "description": "Unified framework for evaluating language models on 200+ academic NLP benchmarks (MMLU, ARC, GSM8K, HellaSwag, etc.) with a single config-driven CLI. Picked as the canonical reference harness; it powers the Hugging Face Open LLM Leaderboard and is the standard tool for reporting reproducible benchmark numbers in papers. 12.6k GitHub stars, 392 contributors, 58 commits in last 90 days.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(GitHub);no feature-gated core", "confidence": "high", "note": "MIT-licensed, fully public; OSI -> open_source.", "sources": [ { "url": "https://github.com/EleutherAI/lm-evaluation-harness", "shows": "MIT license, public source, v0.4.12 (May 2026)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.57M monthly PyPI downloads (lm-eval); backs HuggingFace Open LLM Leaderboard and is used in hundreds of papers and model release reports -- the strongest 'cited as standard' signal in the category. Headline = PyPI primary source; standard-status corroborates.", "sources": [ { "url": "https://pypistats.org/packages/lm-eval", "shows": "~1,571,581 monthly PyPI downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/EleutherAI/lm-evaluation-harness", "shows": "backend for HF Open LLM Leaderboard; used in hundreds of papers", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "60+ standard academic benchmarks with hundreds of subtasks; broadest model-backend breadth (HF Transformers, vLLM, OpenAI, Anthropic, SGLang, NeMo, Megatron-LM, local servers); reproducibility/standardization (powers Open LLM Leaderboard)", "confidence": "high", "note": "Frontier-definer within evaluation_code -- widest task + backend coverage and the standardization reference harness for LLM eval.", "sources": [ { "url": "https://github.com/EleutherAI/lm-evaluation-harness", "shows": "60+ benchmarks, multi-backend support, Open LLM Leaderboard backend", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "lm-eval v0.4.12 (May 11 2026); confirmed live June 2026. Backend for HuggingFace Open LLM Leaderboard. The de-facto-standard model-eval harness." }, { "product": "BigCodeBench", "org": "BigCode Project", "type": "software", "description": "Evaluation harness for code generation models on 1,140 practical programming tasks requiring use of multiple libraries and tools, scored with branch-coverage tests. Picked over HumanEval because it tests real-world library use rather than self-contained algorithmic puzzles. Hosts the BigCode Models Leaderboard; 500+ GitHub stars on the harness, plus 1k+ stars on the sister bigcode-evaluation-harness for code-completion models.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);execution harness + leaderboard public", "confidence": "high", "note": "Apache-2.0, fully public; OSI -> open_source.", "sources": [ { "url": "https://github.com/bigcode-project/bigcodebench", "shows": "Apache-2.0 license, public harness, 1140 tasks", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "medium", "note": "Narrow, specialist code-gen benchmark: 503 GitHub stars, active HF leaderboard with 163 models evaluated. No clean PyPI/download figure for the harness; no published install volume. stars_fallback -> capped at level 3, set to 2 on the modest star + leaderboard footprint.", "sources": [ { "url": "https://github.com/bigcode-project/bigcodebench", "shows": "503 stars; leaderboard with 163 models", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard", "shows": "active leaderboard ranking evaluated models", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "1140 function-level code-gen tasks with diverse library calls + complex instructions; two splits (Complete/Instruct); execution-based scoring; single-domain (code generation) vs multi-benchmark harnesses", "confidence": "medium", "note": "Solid, rigorous code-gen eval but single-benchmark scope -- below the broad multi-task harnesses (lm-eval, OpenCompass) on coverage breadth.", "sources": [ { "url": "https://github.com/bigcode-project/bigcodebench", "shows": "1140 tasks, Complete/Instruct splits, execution-based eval", "accessed": "2026-06-04" } ] }, "maturity": 2.5, "mature": false, "version_note": "BigCodeBench v0.2.4 (Mar 2 2025); 1140 software-engineering programming tasks (Complete + Instruct splits); active HF leaderboard, 163 models evaluated. Confirmed live June 2026. Borderline: it is primarily a single benchmark+its harness; the dataset itself maps to benchmark_eval_data, but the runnable harness fits here." }, { "product": "OpenCompass", "org": "OpenCompass Community", "type": "software", "description": "Comprehensive evaluation platform supporting 100+ datasets across Llama, Mistral, GPT-4, Claude, Qwen, GLM and more, with subjective (arena-style) and objective scoring modes. Picked over lm-eval-harness for Chinese-language and multimodal coverage; backed by Shanghai AI Lab and updated weekly. 7k+ GitHub stars, 162 contributors, 25 commits in last 90 days.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);no feature-gated core", "confidence": "high", "note": "Apache-2.0, fully public; OSI -> open_source.", "sources": [ { "url": "https://github.com/open-compass/opencompass", "shows": "Apache-2.0 license, public source, v0.5.2 (Feb 2026)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Widely used eval framework, especially in the China/HF ecosystem; 7.1k GitHub stars and broad dataset+backend coverage. No clean PyPI/download headline located this run; level 3 on reported_traction (framework distribution + star footprint), not on stars alone past the cap.", "sources": [ { "url": "https://github.com/open-compass/opencompass", "shows": "7.1k stars; 100+ datasets, multi-backend eval framework", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "100+ datasets (70+ with ~400k questions); 20+ HF models pre-supported + OpenAI/Gemini/Claude API backends; broad multi-task coverage and standardized reporting", "confidence": "medium", "note": "Very broad multi-benchmark coverage and backend breadth -- just below lm-eval as the standardization reference; strong but not the single de-facto global standard.", "sources": [ { "url": "https://github.com/open-compass/opencompass", "shows": "100+ datasets, 20+ HF + API model backends", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "OpenCompass v0.5.2 (Feb 14 2026). 100+ datasets / 70+ with ~400k questions; 20+ HF models + API models (OpenAI/Gemini/Claude). Confirmed live June 2026." }, { "product": "DeepEval", "org": "Confident AI", "type": "software", "description": "Pytest-style evaluation framework for LLM apps with 40+ built-in metrics including G-Eval, hallucination, faithfulness and RAG-specific scores. Picked for CI/CD integration; drops into existing test suites and gates merges on metric thresholds. 15.6k stars, 263 contributors, 561 commits in last 90 days, the most active open source eval framework on GitHub.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI, OSS framework);source:public(GitHub);commercial-tier:Confident AI managed eval/observability cloud on top", "confidence": "high", "note": "Apache-2.0 OSS core + proprietary Confident AI cloud -> open_core (recipe names Confident AI as the open_core exemplar); the library itself is OSI.", "sources": [ { "url": "https://github.com/confident-ai/deepeval", "shows": "Apache-2.0 license; Confident AI commercial cloud platform", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~4.63M monthly PyPI downloads (deepeval); 15.9k stars, used-by 1.3k projects. Headline = PyPI primary source; download volume places it firmly in the 1-10M band.", "sources": [ { "url": "https://pypistats.org/packages/deepeval", "shows": "~4,628,371 monthly PyPI downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/confident-ai/deepeval", "shows": "15.9k stars; used by 1.3k projects", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "broad LLM-app eval metrics (G-Eval LLM-as-judge, answer relevancy, faithfulness, task completion, hallucination), RAG + agentic + conversational coverage, pytest-style harness, agentic-eval support", "confidence": "high", "note": "Strong, broad app/RAG/agentic eval coverage with LLM-as-judge -- top-tier for product-eval, just below the academic-benchmark standards (lm-eval) on benchmark-standardization breadth.", "sources": [ { "url": "https://github.com/confident-ai/deepeval", "shows": "G-Eval, RAG/agentic/conversational metrics, pytest-style eval", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "DeepEval v4.0.5 ('Opus 4.8 Day-0 Support', May 28 2026). OSS eval framework (Apache-2.0) with commercial Confident AI cloud platform. Confirmed live June 2026." }, { "product": "promptfoo", "org": "promptfoo", "type": "software", "description": "promptfoo v0.121.14 (Jun 2 2026). CLI + library for LLM eval and red-teaming; now part of OpenAI but remains MIT-licensed open source. Confirmed live June 2026.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(GitHub);no feature-gated core observed (acquired by OpenAI, stated to remain MIT/open source)", "confidence": "high", "note": "MIT-licensed, fully public; OSI -> open_source. OpenAI ownership noted but does not change the OSS license at scoring time.", "sources": [ { "url": "https://github.com/promptfoo/promptfoo", "shows": "MIT license; 'remains open source and MIT licensed' under OpenAI; v0.121.14", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.18M npm downloads/mo (May 4-Jun 2 2026); 21.9k stars; project states it powers LLM apps serving 10M+ users in production. Headline = npm registry primary source; 1-10M band.", "sources": [ { "url": "https://api.npmjs.org/downloads/point/last-month/promptfoo", "shows": "1,176,340 monthly npm downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/promptfoo/promptfoo", "shows": "21.9k stars; '10M+ users in production' claim", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "broad eval + security: prompt/model testing, side-by-side model comparison, red-teaming/vulnerability scanning, CI/CD integration, caching/live-reload; agentic + app eval plus a security dimension peers lack", "confidence": "high", "note": "Very broad eval coverage plus a distinctive red-teaming/security axis; top-tier for application eval, below the academic-benchmark standardization references on raw benchmark coverage.", "sources": [ { "url": "https://github.com/promptfoo/promptfoo", "shows": "eval + red-teaming + model comparison + CI/CD features", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "promptfoo v0.121.14 (Jun 2 2026). CLI + library for LLM eval and red-teaming; now part of OpenAI but remains MIT-licensed open source. Confirmed live June 2026." }, { "product": "OpenAI Evals", "org": "OpenAI", "type": "software", "description": "Framework for evaluating LLMs and LLM systems plus a registry of community-contributed benchmarks, published as the original 2023 reference implementation for eval-driven development. Picked because it is the canonical starting point referenced in OpenAI's own docs and many derivative harnesses. 18.5k stars but development has effectively slowed (2 commits in last 90 days) as work migrated to the closed OpenAI Dashboard Evals API.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(full framework + registry);per-dataset-licenses:mixed(CC/CC0/Apache for bundled data); no feature-gated core", "confidence": "high", "note": "Framework code is fully OSI (MIT); the included eval datasets carry their own open licenses, which is standard for a harness.", "sources": [ { "url": "https://github.com/openai/evals/blob/main/LICENSE.md", "shows": "MIT License text (copyright 2023 OpenAI)", "accessed": "2026-06-04" }, { "url": "https://github.com/openai/evals", "shows": "open framework for evaluating LLMs with open benchmark registry; MIT badge; 18.6k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No PyPI/download count available for this repo (it is run from source, not pip-installed); 18.6k GitHub stars + 3k forks is the only honest signal, so capped at level 3 per the stars-fallback rule. Historically influential as one of the first open eval frameworks but largely superseded by the hosted Evals API and by newer harnesses.", "sources": [ { "url": "https://github.com/openai/evals", "shows": "18.6k stars, 3k forks, 691 commits", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Open benchmark registry + custom-eval authoring (completion-fn + eval templates); model-graded and exact/fuzzy-match graders. Coverage breadth modest vs lm-eval-harness/HELM; no built-in agentic-eval harness; maintenance has slowed.", "confidence": "medium", "note": "Solid general-purpose authoring framework but not frontier on task coverage, backend breadth, or agentic-eval support relative to category leaders.", "sources": [ { "url": "https://github.com/openai/evals", "shows": "framework + registry of benchmarks, custom eval authoring, model-graded grading", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "openai/evals GitHub repo (MIT, 18.6k stars, ~691 commits). Open-source framework for evaluating LLMs with a community benchmark registry. Confirmed live on GitHub June 2026; bundled eval datasets carry their own per-dataset licenses (CC/Apache/CC0)." }, { "product": "simple-evals", "org": "OpenAI", "type": "software", "description": "OpenAI's lightweight reference harness used to produce the model cards for GPT-4o, o1, and successors. Includes MMLU, MATH, GPQA, MGSM, DROP, HumanEval and SimpleQA. Picked when researchers want the exact numbers OpenAI reports rather than community variants. 4.5k stars, used as the authoritative reproduction script for GPT scorecards.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(reference eval implementations); no managed tier", "confidence": "high", "note": "Fully OSI (MIT); purely a reference-implementation library, no gated core.", "sources": [ { "url": "https://github.com/openai/simple-evals", "shows": "MIT license; lightweight eval library; deprecation notice (July 2025); ~4.5k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Run from source (no pip distribution), so no download signal; ~4.5k stars + 490 forks is the only measure, capped at <=3 by the stars rule and lowered to 2 because the project is deprecated (frozen July 2025) and used mainly as a reference for a few specific benchmarks (HealthBench/BrowseComp/SimpleQA) rather than as a live harness.", "sources": [ { "url": "https://github.com/openai/simple-evals", "shows": "~4.5k stars, 490 forks; deprecation notice", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "~9 bundled benchmarks, zero-shot/CoT only; no plugin task system, no agentic-eval support, no model-backend abstraction beyond direct API calls; deprecated/frozen.", "confidence": "medium", "note": "Deliberately minimal reference harness, now frozen; narrow coverage and feature set vs category leaders (HELM, Inspect AI, lm-eval-harness).", "sources": [ { "url": "https://github.com/openai/simple-evals", "shows": "bundled benchmark list (MMLU/MATH/GPQA/HumanEval/HealthBench/etc.); zero-shot CoT approach", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "openai/simple-evals GitHub repo (MIT, ~4.5k stars). Lightweight zero-shot/CoT eval library bundling MMLU, MATH, GPQA, DROP, MGSM, HumanEval, SimpleQA, BrowseComp, HealthBench. DEPRECATED July 2025; no longer updated for new models/benchmarks; only HealthBench/BrowseComp/SimpleQA reference impls maintained. Confirmed live (deprecated) June 2026." }, { "product": "OpenAI Evals API & Dashboard", "org": "OpenAI", "type": "software", "description": "Managed evaluation product inside the OpenAI Platform, Evals API plus a Dashboard UI that lets developers define eval datasets, run them against any OpenAI model, grade with custom or built-in graders, and track regressions across versions. Picked by teams already on OpenAI who want the 'measure → improve → ship' loop without standing up infra. Pricing rolls into standard OpenAI API token usage.", "openness": { "score": 1, "class": "closed", "components": "license:Proprietary(SaaS, OpenAI-platform-only);source:closed;managed-only(no self-host); being deprecated Nov 2026", "confidence": "high", "note": "Closed hosted SaaS bound to the OpenAI platform; not the MIT-licensed openai/evals repo (that is scored separately).", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/evals", "shows": "hosted proprietary Evals API + dashboard on OpenAI platform; sunset dates (read-only Oct 31 2026, shutdown Nov 30 2026)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "OpenAI publishes no usage figures for the Evals API/dashboard specifically, and the product is being sunset, so there is no honest adoption signal to report.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/evals", "shows": "no usage/customer figures; product on sunset path", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Criteria-based grading (string-match + custom + model graders), JSONL/inline datasets with schema validation, runs vs Chat Completions/Responses, async with webhooks, dashboard + token/cost reporting. Locked to OpenAI models; no multi-backend support; no standardized academic benchmark suite; being deprecated.", "confidence": "medium", "note": "Convenient managed eval surface for OpenAI-model users, but narrow (single-provider, no benchmark library) and on a deprecation path, not a frontier harness.", "sources": [ { "url": "https://developers.openai.com/api/docs/guides/evals", "shows": "grading methods, dataset upload, run execution vs OpenAI APIs, dashboard + cost analysis", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Hosted, proprietary evaluation product on the OpenAI platform (API + web dashboard), distinct from the open source openai/evals repo. Lets developers define grading criteria, upload JSONL test sets, run against Chat Completions/Responses APIs, and inspect pass/fail + cost metrics. BEING SUNSET: dashboard read-only 31 Oct 2026, full shutdown 30 Nov 2026 (OpenAI recommends Datasets as replacement). Confirmed live June 2026 via OpenAI developer docs." }, { "product": "Ragas", "org": "Exploding Gradients", "type": "software", "description": "Evaluation framework purpose-built for Retrieval-Augmented Generation pipelines with 30+ metrics covering retrieval quality (context precision/recall), generation faithfulness, and answer relevance. Picked when the system under test is RAG; Ragas decomposes pipeline failure into retrieval vs. generation buckets in a way general harnesses do not. 14k stars, 240 contributors.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(full metrics + test-gen library); core un-gated. (Vendor also offers a hosted app.ragas.io tier; not required for the library.)", "confidence": "high", "note": "Core library is fully OSI (Apache-2.0) with no feature-gated functionality; could be argued open_core given the hosted cloud product, but the eval library itself is complete and open, so scored open_source.", "sources": [ { "url": "https://github.com/explodinggradients/ragas", "shows": "Apache-2.0 license; LLM-app eval metrics + test-set generation; v0.4.3 (Jan 2026); 14.2k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.51M PyPI downloads in the last 30 days (pypistats), de facto standard for RAG/LLM-app evaluation, widely integrated (LangChain, LlamaIndex, observability platforms). Stars (14.2k) corroborate. Solidly in the 1M-10M monthly band.", "sources": [ { "url": "https://pypistats.org/packages/ragas", "shows": "1,513,553 downloads in the last month", "accessed": "2026-06-04" }, { "url": "https://github.com/explodinggradients/ragas", "shows": "14.2k stars; LangChain/observability integrations", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Broad RAG/LLM-app metric suite (faithfulness, context precision/recall, answer relevancy, custom DiscreteMetric/Aspect Critique), automatic synthetic test-data generation, model-agnostic LLM-as-judge, framework + observability integrations. Specialized for application/RAG eval rather than academic-benchmark model eval (different sub-domain from HELM/lm-eval-harness).", "confidence": "medium", "note": "Frontier within the application/RAG-eval niche (metric breadth + test generation), but not a general academic-benchmark model harness; scored 4 not 5 because coverage is domain-scoped.", "sources": [ { "url": "https://github.com/explodinggradients/ragas", "shows": "objective metrics (LLM-based + traditional), Aspect Critique, automatic test-data generation, integrations", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "explodinggradients/ragas, v0.4.3 (released 13 Jan 2026), Apache-2.0, ~14.2k stars. Toolkit for evaluating LLM/RAG applications: LLM-based + traditional metrics, synthetic test-set generation, LangChain/observability integrations. Confirmed live on GitHub + PyPI June 2026. Note: maintainer Exploding Gradients also runs a commercial app.ragas.io cloud tier (open-core), though the core library is fully OSI." }, { "product": "Inspect AI", "org": "UK AI Security Institute", "type": "software", "description": "Frontier-AI evaluation framework with one interface over OpenAI, Anthropic, Google, xAI, Bedrock and local vLLM/Ollama backends, plus a collection of 200+ pre-built evals (Inspect Evals) co-maintained with Arcadia Impact and the Vector Institute. Picked by AISI, EU AI Office, and many frontier labs as the standard for safety/dangerous-capability evaluations. 2.1k stars but extremely active, 1,333 commits in last 90 days, 227 contributors.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(full framework, 200+ evals);governance:public-sector(UK AISI)+Meridian Labs; no managed/gated tier", "confidence": "high", "note": "Fully OSI (MIT), government-backed, no proprietary core, full pipeline open.", "sources": [ { "url": "https://github.com/UKGovernmentBEIS/inspect_ai", "shows": "MIT license; framework for LLM evals with 200+ pre-built evals; UK AISI maintainer; 2.2k stars, 5,886 commits", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "high", "note": "Headline signal (per recipe: adoption by frontier labs / cited in release-grade safety work): adopted as the evaluation framework of choice by major labs incl. Anthropic, Google DeepMind, and xAI/Grok, and powers nearly all of UK AISI's automated frontier-model evaluations. Corroborated by ~9.9M PyPI downloads in the last 30 days (pepy.tech) / 10.5M (pypistats), though that figure is inflated by CI/dependency traffic, so lab adoption is treated as the primary basis. Placed at level 4, not 5, because the human-operator base (eval engineers at labs/AISI) is far smaller than mass-market scale.", "sources": [ { "url": "https://www.aisi.gov.uk/blog/our-first-year", "shows": "AISI runs thousands of frontier-model evaluations on Inspect; broad lab adoption", "accessed": "2026-06-04" }, { "url": "https://inspect.aisi.org.uk/", "shows": "framework for frontier AI evaluations by UK AISI + Meridian Labs; adopted by major labs", "accessed": "2026-06-04" }, { "url": "https://pepy.tech/projects/inspect-ai", "shows": "87.89M total / 9,856,617 last-30-day PyPI downloads (corroboration)", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "200+ pre-built evals; first-class agentic-eval support (tool use, multi-turn dialog, agent tasks); model-graded scoring; broadest model-backend abstraction in the category (OpenAI/Anthropic/Google/Groq/Mistral/xAI/Bedrock/Azure/Together/Cloudflare + local vLLM/Ollama/llama-cpp); reproducibility tooling for large-scale frontier evals.", "confidence": "high", "note": "Frontier-defining for this category: combines the broadest backend coverage, agentic-eval support, and de-facto-standard status among frontier-safety labs. One of the 1-2 leaders, so scored 5.", "sources": [ { "url": "https://inspect.aisi.org.uk/models.html", "shows": "single interface over OpenAI/Anthropic/Google/xAI/Bedrock/Azure/local vLLM/Ollama/llama-cpp", "accessed": "2026-06-04" }, { "url": "https://github.com/UKGovernmentBEIS/inspect_ai", "shows": "200+ pre-built evals; tool use, multi-turn, model-graded eval components", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "UKGovernmentBEIS/inspect_ai (UK AI Security Institute + Meridian Labs), MIT, ~2.2k stars, ~5,886 commits, very actively developed. Framework for frontier LLM/agent evaluations: 200+ pre-built evals, tool use, multi-turn, model-graded scoring, one interface over OpenAI/Anthropic/Google/xAI/Bedrock/Azure/local vLLM/Ollama. Confirmed live June 2026." }, { "product": "HELM", "org": "Stanford CRFM", "type": "software", "description": "Holistic Evaluation of Language Models, Python framework for reproducible, transparent evaluation of foundation models including LLMs and multimodal models, organized around scenarios, adapters, and metrics. Picked for academic-grade reproducibility and breadth (accuracy, robustness, fairness, bias, toxicity). 2.8k stars, 128 contributors, 151 commits in last 90 days; Stanford has announced HELM will enter maintenance mode June 2026.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(framework + standardized datasets + web UI + leaderboard code);governance:Stanford CRFM(academic); no gated core", "confidence": "high", "note": "Fully OSI (Apache-2.0); framework, standardized data formats, web UI, and leaderboards all public.", "sources": [ { "url": "https://github.com/stanford-crfm/helm", "shows": "Apache-2.0 license; holistic eval framework with standardized datasets, web UI, leaderboards; v0.5.16; 2.8k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Headline signal is influence/standard-setting, not install volume: HELM is a widely-cited standardized public leaderboard from Stanford CRFM spanning multiple modalities (capabilities/safety/vision/medical), referenced across the research community. PyPI distribution (crfm-helm) is only ~3,142 downloads/month; most usage is via the hosted leaderboard and academic citation rather than pip, so download volume understates reach. Level 3 reflects meaningful research/standard adoption short of mass production deployment.", "sources": [ { "url": "https://github.com/stanford-crfm/helm", "shows": "multiple public leaderboards (HELM Capabilities/Safety/VHELM/HEIM/MedHELM); Stanford CRFM standardized benchmark", "accessed": "2026-06-04" }, { "url": "https://pypistats.org/packages/crfm-helm", "shows": "3,142 PyPI downloads last month (low because usage is leaderboard/citation-driven, not pip)", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Broadest standardized benchmark/task coverage in the category: multiple holistic leaderboards across text (Capabilities, Safety, ToRR table-reasoning), vision-language (VHELM), text-to-image (HEIM), medical (MedHELM), audio, and enterprise; unified model access across OpenAI/Anthropic/Google/etc.; metrics beyond accuracy; standardized reproducible format + web UI.", "confidence": "high", "note": "Frontier-defining on coverage and standardization, the holistic multi-modality leaderboard standard. One of the 1-2 category leaders, scored 5. (Weaker than Inspect AI on agentic-eval support, but broader on benchmark/modality coverage.)", "sources": [ { "url": "https://github.com/stanford-crfm/helm", "shows": "standardized datasets, unified model access, metrics beyond accuracy, web UI, multi-modal leaderboards (MMLU-Pro/GPQA/IFEval/WildBench)", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "stanford-crfm/helm, v0.5.16 (released 30 Apr 2026), Apache-2.0, ~2.8k stars. Standardized holistic evaluation framework + public leaderboards: HELM Capabilities, HELM Safety, VHELM (vision-language), HEIM (text-to-image), MedHELM, audio-LM, enterprise benchmarks; supports MMLU-Pro, GPQA, IFEval, WildBench. Confirmed live June 2026." }, { "product": "lighteval", "org": "Hugging Face", "type": "software", "description": "Lightweight successor to the eval harness HF used internally; supports vLLM, transformers, TGI and OpenAI-compatible backends with a focus on speed and easy custom metric authoring. Picked when teams want a fast, hackable runner that integrates cleanly with the HF ecosystem (datasets, accelerate). 2.4k stars, 120 contributors, 17 commits in last 90 days; powers parts of the Open LLM Leaderboard v2.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(github.com/huggingface/lighteval);no feature-gated core; multi-backend (Accelerate/vLLM/SGLang/TGI/LiteLLM/inspect-ai); 1000+ tasks", "confidence": "high", "note": "Fully OSI-licensed (MIT), full source public, no managed/commercial tier gating the eval engine.", "sources": [ { "url": "https://github.com/huggingface/lighteval", "shows": "MIT license, public source, v0.13.0 (Nov 2025), 1000+ tasks, multi-backend support", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~37,239 PyPI downloads in the last month (pypistats). Powers the HF Open LLM Leaderboard and is the reproducible eval suite behind it; meaningful in the eval/research community but a smaller install base than general-purpose harnesses. 2.4k GitHub stars corroborate only. Headline = PyPI volume (primary).", "sources": [ { "url": "https://pypistats.org/packages/lighteval", "shows": "~37,239 lighteval downloads in the last month", "accessed": "2026-06-04" }, { "url": "https://github.com/huggingface/lighteval", "shows": "stated as the toolkit powering the HF Open LLM Leaderboard; 2.4k stars (corroboration)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "task/benchmark coverage: 1000+ tasks (MMLU, GPQA, GSM8K, MATH, LiveCodeBench, IFEval, MT-Bench, HELM, multilingual); model-backend breadth: Accelerate/vLLM/SGLang/Nanotron/TGI/LiteLLM/HF Inference Endpoints/custom; reproducibility: standardized suite reproducing the HF Open LLM Leaderboard; agentic-eval: via inspect-ai backend", "confidence": "high", "note": "Strong across the recipe's feature-matrix dimensions (coverage, backend breadth, reproducibility, some agentic support). Not a 5; lm-evaluation-harness remains the broader de-facto standard cited in more frontier release reports; lighteval is the HF-leaderboard-tied counterpart.", "sources": [ { "url": "https://github.com/huggingface/lighteval", "shows": "1000+ tasks, 8+ model backends, reproduces Open LLM Leaderboard evals", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "lighteval v0.13.0 (24 Nov 2025), latest GitHub release; confirmed live on GitHub June 2026. All-in-one LLM eval toolkit from HF's Leaderboard & Evals team; powers the HF Open LLM Leaderboard. Columbia extension (model.code.evaluation) caveat applies." }, { "product": "OSWorld", "org": "XLang AI Lab", "type": "software", "description": "Scalable, real-computer environment for benchmarking multimodal agents on 369 open-ended tasks across Ubuntu, Windows and macOS; agents drive a virtual desktop and are scored by execution-based test scripts. Picked because it is the leading harness for computer-use agents (Claude Computer Use, OpenAI Operator both publish OSWorld scores). 2.9k stars, 86 contributors, 55 commits in last 90 days.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(github.com/xlang-ai/OSWorld);execution-based eval harness + 369-task environment public; multi-platform (VMware/VirtualBox/Docker/AWS)", "confidence": "high", "note": "Apache-2.0, full harness + environment public. Scored as the evaluation harness; the bundled 369-task set is also openly redistributable.", "sources": [ { "url": "https://github.com/xlang-ai/OSWorld", "shows": "Apache-2.0 license, public source, computer-use agent eval harness", "accessed": "2026-06-04" }, { "url": "https://os-world.github.io/", "shows": "369 tasks (361 excl. Google Drive), real-computer execution-based evaluation", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Top adoption signal for a harness = cited as the standard in frontier release reports: OSWorld scores reported in OpenAI (CUA 38.1%, GPT-5.4 75.0%), Anthropic (Claude Sonnet 4.5 61.4%, Sonnet 4.6 72.5%, Opus 4.6 72.7%) computer-use launches through 2026. Reach band is directional. The harness is run by a concentrated set of frontier labs/agent teams rather than a mass install base. 2.9k stars corroborate only.", "sources": [ { "url": "https://www.anthropic.com/news/claude-sonnet-4-6", "shows": "Claude Sonnet 4.6 OSWorld score reported in official model release (standard-citation signal)", "accessed": "2026-06-04" }, { "url": "https://os-world.github.io/", "shows": "best-model vs human baselines; ongoing updates with new LLMs/VLMs", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "benchmark/task coverage: 369 real-computer tasks across web/desktop/OS-file-IO/multi-app workflows; agentic-eval support: execution-based (not multiple-choice) evaluation of full computer-use agent trajectories; model-backend breadth: any VLM/LLM agent (Operator, GPT, Gemini, Claude, UI-TARS, Agent-S, Qwen, CogAgent baselines); reproducibility: multi-platform virtualization (VMware/VirtualBox/Docker/AWS)", "confidence": "high", "note": "Frontier-defining within evaluation_code for the computer-use/agentic axis. Execution-based, real-environment agent evaluation is the hardest eval modality and OSWorld is its reference harness. A 5 here is for the category's frontier-definers; OSWorld qualifies for agentic eval.", "sources": [ { "url": "https://os-world.github.io/", "shows": "369 execution-based real-computer tasks, multi-OS, agent-trajectory evaluation", "accessed": "2026-06-04" }, { "url": "https://github.com/xlang-ai/OSWorld", "shows": "harness + baselines for Operator/GPT/Gemini/Claude/UI-TARS/Agent-S", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "OSWorld: scalable real-computer environment + execution-based evaluation harness for multimodal/computer-use agents (NeurIPS 2024). Apache-2.0. 369 tasks (361 excl. 8 problematic Google Drive tasks). De-facto standard for computer-use agents (GPT-5.4 75.0%, Claude Opus 4.6 72.7%, Claude Sonnet 4.6 72.5% on OSWorld-Verified cited in 2026 releases). Confirmed live June 2026. Dual artifact: ships both the task set and the runner; registry placed it in evaluation_code (the harness); note the dataset overlaps benchmark_eval_data." }, { "product": "Chatbot Arena", "org": "LMArena", "type": "software", "description": "Hosted human-preference LLM evaluation platform; rebranded LMSYS Chatbot Arena -> LMArena -> Arena (arena.ai; lmarena.ai 301-redirects to arena.ai as of June 2026). The PRODUCT is the hosted leaderboard/battle platform (closed). Its historical OSS code is FastChat (Apache-2.0) but stale (v0.2.36, Feb 2024); auxiliary OSS eval tooling Arena-Hard-Auto (Apache-2.0) is maintained. Confirmed live June 2026.", "openness": { "score": 2, "class": "open_core", "components": "platform:closed(hosted arena.ai, proprietary live leaderboard + private vote data); reference-code:FastChat Apache-2.0 (powers Chatbot Arena but stale, last release Feb 2024, 39.5k stars); aux-OSS:Arena-Hard-Auto Apache-2.0 (automatic LLM eval, highest correlation to LMArena); ranking methodology + crowd-vote dataset not fully open", "confidence": "medium", "note": "Scored as the hosted platform: the live evaluation product, its vote stream, and current ranking pipeline are closed; substantial OSS heritage (FastChat) and an OSS automatic-eval companion exist, so open_core not closed. The OSS code is stale relative to the operating platform.", "sources": [ { "url": "https://github.com/lm-sys/FastChat", "shows": "Apache-2.0; 'FastChat powers Chatbot Arena (lmarena.ai), serving over 10M chat requests for 70+ LLMs'; latest release v0.2.36 Feb 2024", "accessed": "2026-06-04" }, { "url": "https://github.com/lmarena/arena-hard-auto", "shows": "Apache-2.0 automatic LLM eval tool, highest correlation to LMArena", "accessed": "2026-06-04" }, { "url": "https://arena.ai/", "shows": "hosted 'Official AI Ranking & LLM Leaderboard' with Battle Mode; conversations disclosed to providers (proprietary platform)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "FastChat README (primary) states Chatbot Arena has served 'over 10 million chat requests for 70+ LLMs'; Arena Elo is the most-cited human-preference leaderboard in frontier model releases (LMArena Elo appears across vendor launch posts). Headline = the 10M+ chat-request figure (primary, FastChat repo). Banded 1M-10M conservatively on request volume; could be higher cumulatively.", "sources": [ { "url": "https://github.com/lm-sys/FastChat", "shows": "'serving over 10 million chat requests for 70+ LLMs'", "accessed": "2026-06-04" }, { "url": "https://arena.ai/", "shows": "hosted live leaderboard / battle platform in active operation June 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "eval modality: blind pairwise human-preference battles -> Elo/Bradley-Terry ranking (the reference human-eval methodology); coverage: 70+ LLMs, multimodal/vision/coding arenas; agentic-eval: limited (preference-based, not task-execution); reproducibility: Arena-Hard-Auto provides an automatic-judge approximation", "confidence": "medium", "note": "Reference platform for human-preference evaluation and the de-facto Elo standard cited everywhere; not a 5 because as eval CODE it is largely closed/hosted and narrower than execution-based harnesses on agentic/task coverage. Capability assessed on its evaluation methodology breadth.", "sources": [ { "url": "https://github.com/lmarena/arena-hard-auto", "shows": "500 SWE/math queries + 250 creative prompts, GPT-4.1/Gemini-2.5 judges, highest correlation to LMArena", "accessed": "2026-06-04" }, { "url": "https://arena.ai/", "shows": "battle-mode pairwise comparison across many models", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Hosted human-preference LLM evaluation platform; rebranded LMSYS Chatbot Arena -> LMArena -> Arena (arena.ai; lmarena.ai 301-redirects to arena.ai as of June 2026). The PRODUCT is the hosted leaderboard/battle platform (closed). Its historical OSS code is FastChat (Apache-2.0) but stale (v0.2.36, Feb 2024); auxiliary OSS eval tooling Arena-Hard-Auto (Apache-2.0) is maintained. Confirmed live June 2026." }, { "product": "Patronus Evaluation Platform", "org": "Patronus AI", "type": "software", "description": "Managed evaluation platform built around proprietary judge models, Lynx (8B hallucination detector), GLIDER (3.8B explainable rubric judge) and Percival (agent execution-trace analyzer). Picked by enterprises needing managed eval models rather than spinning up their own LLM-as-judge. Raised $20M; usage-based pricing $10-$20 per 1,000 evaluator API calls.", "openness": { "score": 2, "class": "open_core", "components": "platform:closed(hosted SaaS; remote Judge/Lynx/Glider evaluators run on Patronus infrastructure); client-SDKs:open(patronus-py public on GitHub, v0.1.25, but only 7 stars and wires to the hosted API); eval engine + proprietary evaluators not self-hostable", "confidence": "medium", "note": "Open-source SDK is a thin client to a closed hosted evaluation backend; the core scoring engine and managed evaluators are proprietary => open_core, low end (the OSS surface is a connector, not the product). License text not displayed on the SDK repo page (note).", "sources": [ { "url": "https://github.com/patronus-ai/patronus-py", "shows": "Python SDK whose remote evaluators 'run remotely on Patronus infrastructure'; v0.1.25 Jan 2026; 7 stars", "accessed": "2026-06-04" }, { "url": "https://docs.patronus.ai/docs", "shows": "hosted platform: built-in Lynx/Glider evaluators, experiments, production monitoring, SDK access to cloud service", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "No disclosed user/customer counts or usage volume found on the docs/homepage this run; open source SDK has only 7 GitHub stars (stars_fallback would cap at hobbyist and is not an honest signal for a hosted platform). Declining to assign a level rather than infer from marketing.", "sources": [ { "url": "https://github.com/patronus-ai/patronus-py", "shows": "7 stars on the public SDK (no honest usage figure for the hosted platform)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "benchmark/task coverage: RAG, agent eval, NLP, OWASP/safety metrics, red-teaming dataset generation; model-backend breadth: model-agnostic via SDK; agentic-eval support: yes (agent evaluation + Percival); proprietary evaluators: Lynx (hallucination), Glider; reproducibility: hosted/standardized but not independently reproducible (closed engine)", "confidence": "medium", "note": "Solid, broad commercial eval feature set (evaluators + experiments + monitoring + agent eval) but not frontier-defining within the category and the closed backend limits reproducibility. Mid-tier on the feature matrix.", "sources": [ { "url": "https://docs.patronus.ai/docs", "shows": "Lynx/Glider evaluators, RAG/agent/NLP/OWASP metrics, experiments, red-teaming, monitoring", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Patronus AI hosted GenAI evaluation/optimization platform ('Core Platform'); open source client SDKs (patronus-py, latest v0.1.25 Jan 2026; patronus-api-node) that connect to proprietary hosted infra. Built-in evaluators Lynx (hallucination), Glider; experiments, production monitoring/tracing, Percival, RL Envs. Confirmed live June 2026." }, { "product": "Scale Evaluation", "org": "Scale AI", "type": "software", "description": "Enterprise evaluation and red-teaming service combining expert human raters, in-house prompt engineers, and proprietary benchmark suites for frontier-model developers and enterprises. Picked because Scale is the third-party evaluator for the US AI Safety Institute and runs custom evals for major labs that cannot be reproduced with OSS tools. March 2026 launch of Scale Labs (expansion of the 2023 SEAL division) consolidates this offering.", "openness": { "score": 1, "class": "closed", "components": "platform:closed(proprietary hosted SaaS); datasets:closed(private, unpublished SEAL benchmarks kept private to prevent gaming/training contamination); methodology:partially-disclosed(leaderboard transparency) but no open eval code or self-hostable engine; human-eval workforce is a proprietary service", "confidence": "high", "note": "Proprietary by design; the deliberately-private benchmark datasets and hosted expert-eval workforce are the moat; no open source eval code. Closed.", "sources": [ { "url": "https://scale.com/genai-platform", "shows": "Scale GenAI Platform: automated + expert-human evaluation 'Trust Feedback Loop'; test against proprietary benchmark datasets", "accessed": "2026-06-04" }, { "url": "https://scale.com/blog/leaderboard", "shows": "SEAL Leaderboards use curated PRIVATE/unpublished datasets that 'cannot be gamed'", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "SEAL Leaderboards are widely cited and 'trusted by leading AI labs' as a third-party evaluator; SEAL Showdown reports 'millions of real-world conversations' across 100+ countries / 70 languages / 200 professional domains. No disclosed paying-customer count for the Scale Evaluation platform specifically. Level 3 reflects reported traction of the eval surface, not a verified user count; directional.", "sources": [ { "url": "https://scale.com/blog/showdown", "shows": "SEAL Showdown built on millions of real-world conversations, 100+ countries, 70 languages, 200 domains", "accessed": "2026-06-04" }, { "url": "https://scale.com/blog/leaderboard", "shows": "third-party evaluator trusted by leading AI labs", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "benchmark/task coverage: coding, instruction-following, math, multilingual, reasoning, agentic workflows, multimodal, safety/alignment; eval modalities: private curated benchmarks + adversarial vulnerability scans + human expert eval + real-preference (Showdown); reproducibility: NOT reproducible by design (private datasets); standardization: high (neutral third-party)", "confidence": "medium", "note": "Very broad professional eval coverage (multi-domain private benchmarks + expert workforce + adversarial red-teaming + real-preference), strong commercial capability. Not a 5: closed/non-reproducible and not a community-standard harness in the open-eval-code sense.", "sources": [ { "url": "https://scale.com/blog/leaderboard", "shows": "SEAL multi-domain expert-driven leaderboards (coding/IF/math/multilingual/reasoning/agentic/safety)", "accessed": "2026-06-04" }, { "url": "https://scale.com/genai-platform", "shows": "auto-generated eval datasets + proprietary benchmarks + adversarial vulnerability scans + expert human eval", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Scale AI 'Scale Evaluation' platform (the 'Evaluate AI' product line as of Oct 2025), part of Scale GenAI Platform. Proprietary hosted evaluation/monitoring with private SEAL benchmark datasets + expert human-eval workforce ('Trust Feedback Loop'). Public-facing SEAL Leaderboards + SEAL Showdown (millions of real-world conversations across 100+ countries). Confirmed live 2025-2026." }, { "product": "Artificial Analysis Intelligence Index", "org": "Artificial Analysis", "type": "software", "description": "Independent benchmarking service that runs 10+ academic and agentic evals (MMLU, GPQA, long-context, etc.) against every major model and publishes the Intelligence Index, a composite score with 95% CIs. Picked as the go-to neutral leaderboard for model selection: their Index v3 score is widely cited and covers 300+ models across 100+ providers. Raised a seed round via the Nat Friedman / Daniel Gross AIGrant program.", "openness": { "score": 2, "class": "open_core", "components": "platform:closed(proprietary benchmarking site + methodology); methodology:documented(public methodology page, per-eval breakdown, temp/token params); datasets:closed(maintains internal copies of all eval datasets); harness:partial-open(Stirrup MIT, used for the agentic GDPval-AA eval only, 413 stars); the full Intelligence Index harness is not open", "confidence": "high", "note": "Methodology is well-documented and one component harness (Stirrup) is MIT-OSS, but the Index itself (pipeline, internal dataset copies, grader config) is proprietary => open_core (closed-leaning with a documented methodology + one OSS sub-tool). Not pure closed given Stirrup + transparent methodology; not source_available since the core isn't published.", "sources": [ { "url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "shows": "v4.0.4 (Mar 2026), 10-eval composite, internal copies of all datasets, standardized params (proprietary pipeline + open Stirrup harness for GDPval-AA)", "accessed": "2026-06-04" }, { "url": "https://github.com/ArtificialAnalysis/Stirrup", "shows": "Stirrup MIT-licensed agent harness, 413 stars (the only open component)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "The AA Intelligence Index is one of the most-cited independent model-ranking standards in frontier release coverage and the flagship_scores set itself uses it as a capability basis (e.g. Grok 4.20, Gemini 3.5 Flash records). Widely referenced across vendor launches and tech press as the go-to cross-model index. No disclosed unique-visitor count; level 4 reflects its standard-citation reach across the industry, banded conservatively. Directional.", "sources": [ { "url": "https://artificialanalysis.ai/", "shows": "live Intelligence Index leaderboard positioned as the cross-model standard reference", "accessed": "2026-06-04" }, { "url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "shows": "independent standardized index cited as the reference methodology", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "benchmark/task coverage: 10-eval composite across Agents/Coding/General/Scientific (GDPval-AA, tau2-Telecom, Terminal-Bench Hard, SciCode, AA-LCR, AA-Omniscience, IFBench, HLE, GPQA Diamond, CritPt); model-backend breadth: 150+ models scored independently; agentic-eval support: yes (GDPval-AA via Stirrup, Terminal-Bench Hard, tau2); reproducibility: standardized params + internal dataset copies (contamination-resistant) but not externally reproducible", "confidence": "high", "note": "Frontier-grade composite coverage spanning agentic/coding/general/scientific with contamination-resistant internal datasets and standardized running. Strong capability. Not a 5: it is a curated index/methodology rather than an open community harness, and held-out/proprietary running limits external reproducibility.", "sources": [ { "url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "shows": "10-eval composite, 4 equal-weighted categories, standardized params, internal dataset copies", "accessed": "2026-06-04" }, { "url": "https://artificialanalysis.ai/", "shows": "150+ models ranked on the Intelligence Index", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Artificial Analysis Intelligence Index v4.0.4 (March 2026). Composite of 10 evals (GDPval-AA, tau2-Bench Telecom, Terminal-Bench Hard, SciCode, AA-LCR, AA-Omniscience, IFBench, Humanity's Last Exam, GPQA Diamond, CritPt) in 4 equal-weighted categories. Independent third-party benchmarking platform with published methodology; partial OSS (Stirrup agent harness, MIT, used for GDPval-AA) but the Index pipeline + internal dataset copies are proprietary. Confirmed live June 2026." }, { "product": "Vals AI", "org": "Vals AI", "type": "software", "description": "Domain-specialist benchmarking service that runs proprietary, held-out evals over finance, law, software, and healthcare tasks, including a private Canadian-court-case QA benchmark and the Vals Index. Picked by enterprises that need closed/private benchmarks because public benchmarks suffer from contamination. Publishes a leaderboard of all major frontier models scored on these private sets.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary;source:closed(SaaS-only,no public repo);benchmarks:proprietary/private-held-out(Vals Index on non-public datasets);eval-harness:closed", "confidence": "high", "note": "Closed third-party eval platform; benchmarks deliberately held private to prevent dataset leakage, eval infra not open sourced.", "sources": [ { "url": "https://www.vals.ai", "shows": "proprietary SaaS eval platform, 'proprietary benchmarks' require contact, no GitHub/self-host option", "accessed": "2026-06-04" }, { "url": "https://www.vals.ai/about", "shows": "runs evaluations in-house on private/held-out datasets as neutral third party", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Widely cited independent leaderboard authority: 24 models tracked on the Vals Index (updated 2026-06-04); benchmark figures (e.g. GLM-5.1) are referenced in the project's own flagship scoring sources and across third-party model coverage; partnered with Legaltech Hub for industry studies. No standalone user/traffic count published, so reported_traction; level 3 reflects broad referenced use as a benchmark authority rather than a measured usage figure.", "sources": [ { "url": "https://www.vals.ai/benchmarks", "shows": "live multi-domain leaderboards (Vals Index, Multimodal Index) tracking 24+ frontier models, updated June 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Coverage: 6+ domain verticals (finance/law/healthcare/coding/math/tax incl. TaxEval, CorpFin, MedCode, LegalBench); model-backend breadth = all major frontier labs; reproducibility via private held-out sets (anti-leakage); agentic-eval support (Finance Agent v2). Pointwise scoring + composite indices.", "confidence": "medium", "note": "Strong feature_matrix: leading independent/blind benchmark suite with deep vertical coverage and agentic-eval, but narrower than a general harness (no self-host, fixed proprietary task set). Not a frontier-definer (5) on standardization since methodology is closed; placed at 4.", "sources": [ { "url": "https://www.vals.ai", "shows": "Vals Index + Multimodal Index + domain benchmarks (finance/law/healthcare/coding/math/education) incl TaxEval, CorpFin, MedCode, LegalBench", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Vals AI independent LLM evaluation platform; Vals Index + Vals Multimodal Index + domain benchmarks (finance, law, healthcare, coding, math, tax). Verified live June 2026 (leaderboard updated 2026-06-04, includes Claude Opus 4.8, Qwen 3.7 Plus, MiniMax M3). SaaS-only, no public source repo." }, { "product": "Epoch AI Benchmarks", "org": "Epoch AI", "type": "software", "description": "Independent benchmarking lab that runs and maintains canonical leaderboards for high-stakes evals (SWE-bench Verified, FrontierMath, GPQA Diamond) and operates the FrontierMath private math benchmark. Picked when researchers want the authoritative scoreboard for a specific benchmark; Epoch's SWE-bench Verified leaderboard is the one frontier labs reference in announcements. Funded by Open Philanthropy and lab partners.", "openness": { "score": 2, "class": "source_available", "components": "dashboard:public(free to browse);eval-code:partial-open(several org repos MIT/Apache - SWE-bench docker registry, eci-public, MirrorCode-data);benchmark-data:mixed(FrontierMath gated/private to prevent contamination; some public);no-unified-OSI-harness-product", "confidence": "medium", "note": "Mixed-tier: public results dashboard plus some MIT/Apache-licensed eval components, but the headline benchmark (FrontierMath) is deliberately private and there is no single OSI-licensed end-to-end harness, closer to source_available than fully open_source.", "sources": [ { "url": "https://github.com/epoch-research", "shows": "public org repos incl SWE-bench docker registry, eci-public (MIT), MirrorCode-data; mostly MIT/Apache-2.0", "accessed": "2026-06-04" }, { "url": "https://epoch.ai/benchmarks", "shows": "public benchmark dashboard running FrontierMath/GPQA Diamond/SWE-bench Verified/SimpleQA Verified across frontier models", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Epoch's benchmark results (esp. FrontierMath) are a widely-referenced capability signal cited across frontier-model coverage and research; the public dashboard is a recognized trends source. No standalone usage/traffic count published, so reported_traction; level 3 reflects broad authoritative reference rather than a measured figure. No stars-only fallback used.", "sources": [ { "url": "https://epoch.ai/benchmarks", "shows": "public database of leading-model benchmark results positioned as authoritative AI-capability tracker", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Benchmark coverage: 7+ hard frontier benchmarks (FrontierMath Tiers 1-4, GPQA Diamond, SWE-bench Verified, SimpleQA Verified, MATH L5, OTIS Mock AIME, Chess Puzzles); model-backend breadth = all major frontier labs; reproducibility via contamination-resistant held-out math; agentic eval = coding (SWE-bench) but primarily static-task scoring.", "confidence": "medium", "note": "Strong feature_matrix on hard/frontier benchmark coverage and trusted standardized results, especially math. Not a general agentic-eval harness; held-out design limits self-serve reproducibility. Placed at 4, authoritative but not a frontier-defining general harness like lm-eval-harness.", "sources": [ { "url": "https://epoch.ai/benchmarks", "shows": "operated benchmarks list: FrontierMath, GPQA Diamond, SWE-bench Verified, SimpleQA Verified, MATH Level 5, OTIS Mock AIME, Chess Puzzles", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Epoch AI benchmark database + public dashboard scoring frontier models on hard tasks (FrontierMath, GPQA Diamond, SWE-bench Verified, SimpleQA Verified, MATH Level 5, OTIS Mock AIME, Chess Puzzles). Verified live June 2026; org github.com/epoch-research has MIT/Apache eval repos (SWE-bench docker registry, eci-public, MirrorCode-data), but FrontierMath dataset is held private." }, { "product": "Amazon Bedrock Evaluations", "org": "Amazon Web Services", "type": "software", "description": "Managed evaluation service inside Amazon Bedrock for scoring foundation models and RAG systems with three modes: automatic (BERTScore, F1, accuracy, robustness, toxicity), LLM-as-a-Judge (correctness, completeness, harmfulness, custom prompts), and human evaluation (internal workforce or AWS-managed reviewer team). Differentiates by being native to the Bedrock model catalog and Knowledge Bases; eval jobs run against any Bedrock-supported model, results tie directly into the model deployment pipeline, and comparative analysis across runs is built in. Bedrock model evaluation went GA April 2024; the LLM-as-a-Judge variant went GA March 2025. Bedrock as a platform serves tens of thousands of customers (per AWS, Dec 2024) including adidas, Intuit, Pfizer, Salesforce, Siemens, Thomson Reuters, and the NYSE.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(AWS managed service);source:closed;deployment:cloud-only(AWS-hosted eval jobs, S3 I/O);billing:pay-as-you-go(judge-model inference + AWS-managed human workforce)", "confidence": "high", "note": "Proprietary AWS managed service; no source, no self-host, runs as billed Bedrock evaluation jobs.", "sources": [ { "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation.html", "shows": "managed Bedrock service running automatic/judge/human eval jobs on AWS, CORS S3 I/O, CloudTrail events", "accessed": "2026-06-04" }, { "url": "https://aws.amazon.com/bedrock/evaluations/", "shows": "AWS-hosted feature, AWS-managed human evaluator workforce option, directs to pricing page", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "GA since 2025-03-20 and distributed inside Amazon Bedrock (a large enterprise AI surface), but AWS publishes no standalone usage count for the Evaluations feature specifically. Level 3 = reported traction via the Bedrock platform footprint, not a measured figure; low confidence because the signal is the parent surface, not the feature. No stars fallback (closed service).", "sources": [ { "url": "https://aws.amazon.com/blogs/machine-learning/evaluate-models-or-rag-systems-using-amazon-bedrock-evaluations-now-generally-available/", "shows": "GA announcement (2025-03-20) of model + RAG evaluations as a broadly available Bedrock feature", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Eval-type coverage: model eval (LLM-as-a-judge, programmatic BERTScore/F1, human), RAG eval (retrieval + retrieve-and-generate, citation precision/coverage); metric areas: quality (correctness/completeness/faithfulness), UX (helpfulness/coherence/relevance), instruction-compliance, safety (harmfulness/stereotyping/refusal); model-backend breadth = Bedrock + external/on-prem via BYO responses.", "confidence": "medium", "note": "Broad managed eval suite covering model + RAG + agent, judge/programmatic/human modes, and four metric families with BYO-responses portability. Comprehensive but a custom-dataset eval product, not a standardized public benchmark leaderboard; placed at 4.", "sources": [ { "url": "https://aws.amazon.com/bedrock/evaluations/", "shows": "model eval (LLM-as-judge/programmatic/human) + RAG eval (retrieval, retrieve-and-generate) with safety/quality metrics", "accessed": "2026-06-04" }, { "url": "https://aws.amazon.com/blogs/machine-learning/evaluate-models-or-rag-systems-using-amazon-bedrock-evaluations-now-generally-available/", "shows": "four metric areas (quality/UX/instruction-compliance/safety) and citation precision/coverage metrics", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "AWS managed evaluation feature within Amazon Bedrock; model eval (LLM-as-a-judge, programmatic incl BERTScore/F1, human), RAG eval (retrieval + retrieve-and-generate). RAG eval + LLM-as-a-judge GA 2025-03-20 (preview at re:Invent Dec 2024); 'bring your own inference responses' supports external/on-prem models. Verified live June 2026." }, { "product": "Vertex AI Gen AI Evaluation Service", "org": "Google Cloud", "type": "software", "description": "Managed evaluation service on Vertex AI (now part of Gemini Enterprise Agent Platform) that scores generative-AI outputs using three approaches: computation-based metrics (ROUGE, BLEU, exact match), LLM-as-judge with Gemini as the judge model (including adaptive rubrics that auto-generate per-prompt pass/fail tests), and AutoSxS pairwise pipelines that A/B test two models with a judge declaring a winner. Differentiates on tight integration with Vertex AI pipelines, Model Garden, Gemini tuning workflows, and LiteLLM for third-party models; eval datasets can be built from production logs or synthetic data and run across seven global regions. Used by Vertex AI customers including Wendy's, Wayfair, Verizon, Mercedes-Benz, and Etsy (per Google Cloud case studies) as the canonical eval surface for Gemini-based applications.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(GCP managed service);source:closed;deployment:cloud-only(Vertex AI pipelines/AutoSxS);billing:GCP managed-service pricing", "confidence": "high", "note": "Proprietary Google Cloud managed service; no source, no self-host, runs as Vertex AI eval pipelines.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/evaluation-overview", "shows": "managed GCP eval service (computation-based + autorater + AutoSxS) within Gemini Enterprise Agent Platform", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Managed GCP feature with no published standalone usage/customer count and no download or stars signal (closed cloud service). Unlike Bedrock Evaluations there was no clear GA-scale traction figure verifiable this run, so declining to assign a level rather than infer from the parent Vertex surface.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/evaluation-overview", "shows": "managed Vertex AI feature, no usage/adoption figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Eval-target coverage: models + agents + RAG; metric types: computation-based, model-based/autorater (managed rubric-based metrics + custom judge models), pointwise + pairwise (AutoSxS side-by-side); RAG grounding evaluation; model-backend breadth = Gemini + custom/external models.", "confidence": "medium", "note": "Broad managed eval suite: model/agent/RAG targets, both computation- and autorater-based metrics, and pointwise + pairwise (AutoSxS) comparison. Comparable scope to Bedrock Evaluations; a custom-dataset eval product rather than a public standardized leaderboard, so 4 not 5.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/evaluation-overview", "shows": "evaluates models/agents/RAG with computation-based + autorater metrics and pointwise/pairwise AutoSxS", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Google Cloud Vertex AI managed Gen AI evaluation service; evaluates models, agents, and RAG. Computation-based metrics, model-based/autorater (managed rubric-based + custom judge models), pointwise + pairwise (AutoSxS). Part of Gemini Enterprise Agent Platform. Verified live June 2026 (docs current)." }, { "product": "AlpacaEval", "org": "Tatsu Lab (Stanford)", "type": "software", "description": "Automatic LLM-judge evaluator for instruction-following models: an LLM annotator compares a candidate's outputs against a reference on a fixed instruction set to produce a win-rate and leaderboard. AlpacaEval 2.0 adds a length-controlled win-rate to debias for output length (raising correlation with Chatbot Arena to ~0.98).", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully Apache-2.0.", "sources": [ { "url": "https://github.com/tatsu-lab/alpaca_eval", "shows": "Apache-2.0, ~2.0k stars", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~2k stars - stars_fallback; widely used as the standard LLM-judge instruction-following eval.", "sources": [ { "url": "https://github.com/tatsu-lab/alpaca_eval", "shows": "~1,996 stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "LLM-as-judge win-rate; length-controlled metric; reference-model comparison; leaderboard", "confidence": "medium", "note": "De-facto standard for its niche with an influential LC metric; narrower than multi-task harnesses.", "sources": [ { "url": "https://github.com/tatsu-lab/alpaca_eval", "shows": "AlpacaEval 2.0 LC win-rate", "accessed": "2026-06-17" } ] }, "maturity": 3.5, "mature": false, "version_note": "Apache-2.0. ~2k stars. A full eval runs in ~5 min for under ~$10." }, { "product": "Open LLM Leaderboard", "org": "Hugging Face", "type": "software", "description": "Hugging Face's flagship public leaderboard for comparing open LLMs on a fixed, reproducible benchmark suite run on a shared cluster. v2 (2024) raised difficulty with IFEval, BBH, MATH-L5, GPQA, MuSR, and MMLU-Pro. It evaluated 13,000+ models and drew 2M+ visitors before being archived.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(HF-Space);backend:lm-eval-harness/lighteval(open);source:public", "confidence": "high", "note": "Open Apache-2.0 leaderboard built on open eval backends (now archived).", "sources": [ { "url": "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard", "shows": "Apache-2.0 Space, ~14k likes", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "reported_traction", "confidence": "high", "note": "ARCHIVED 2025-03-13. At peak: 13,000+ models evaluated, 2M+ unique visitors - historically the most influential open-model leaderboard.", "sources": [ { "url": "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard/discussions/1135", "shows": "retirement announcement, 2025-03-13", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "standardized 6-benchmark suite (v2: IFEval/BBH/MATH-L5/GPQA/MuSR/MMLU-Pro); shared-cluster reproducible runs; 13,000+ models", "confidence": "high", "note": "Historically the most visible open-model leaderboard; static rather than human-preference; now archived.", "sources": [ { "url": "https://huggingface.co/docs/leaderboards/en/open_llm_leaderboard/archive", "shows": "v2 benchmark suite, 2M visitors, archive", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "ARCHIVED/retired (announced 2025-03-13); HF redirected users to the OpenEvals hub. HF Space was Apache-2.0; backend used lm-evaluation-harness (v1) and lighteval (v2). Listed for historical significance." }, { "product": "Compar:IA", "org": "DINUM / beta.gouv.fr (French Government)", "type": "software", "description": "Compar:IA is an open source LLM comparison 'arena' created by the French Government (betagouv / DINUM with the Ministry of Culture). Users blind-compare responses from two anonymous models, vote on the better answer, then see the models' identities and environmental impact, raising awareness of model pluralism, bias, and carbon cost. Its core mission is to build large non-English (predominantly French) human-preference and alignment datasets; it is self-hostable.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI; verified LICENSE body);source:public;self-hostable", "confidence": "high", "note": "Permissive OSI license with full public source code.", "sources": [ { "url": "https://raw.githubusercontent.com/betagouv/ComparIA/develop/LICENSE", "shows": "LICENSE body 'Apache License, Version 2.0'", "accessed": "2026-06-22" }, { "url": "https://github.com/betagouv/ComparIA", "shows": "Apache-2.0 license badge, full source, betagouv ownership", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "stars_fallback", "confidence": "high", "note": "74 GitHub stars (capped at 3 per stars-fallback rule); the live service has collected 600k+ prompts and 250k+ votes.", "sources": [ { "url": "https://github.com/betagouv/ComparIA", "shows": "74 stars, 16 forks", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Blind pairwise model arena across 30+ models with preference voting, leaderboards, EcoLogits environmental tracking, multilingual deployment, and open dataset export.", "confidence": "high", "note": "Broad arena feature set comparable to LMArena plus environmental tracking and dataset generation.", "sources": [ { "url": "https://github.com/betagouv/ComparIA", "shows": "Feature list: model comparison, voting, EcoLogits, dataset publishing, self-hosting", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive OSI license with full public source code." }, { "product": "SimpleAudit", "org": "Simula Research Laboratory", "type": "software", "description": "SimpleAudit is a lightweight, local-first framework for multilingual auditing and red-teaming of AI systems via adversarial probing. Developed by Simula / SimulaMet (Norway) with the Norwegian Directorate of Health, it runs with open models locally (no API required) and ships scenario packs for safety, RAG, healthcare, and epistemic (broken-premise) testing. It is a verified Digital Public Good.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI; verified LICENSE body, copyright Simula);source:public;Python", "confidence": "high", "note": "Permissive OSI (MIT) license with full public source.", "sources": [ { "url": "https://raw.githubusercontent.com/kelkalot/simpleaudit/main/LICENSE", "shows": "LICENSE body 'MIT License', copyright Simula Research Laboratory", "accessed": "2026-06-22" }, { "url": "https://www.digitalpublicgoods.net/r/simpleaudit", "shows": "DPG profile confirming multilingual safety red-teaming framework", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "niche", "signal_type": "stars_fallback", "confidence": "medium", "note": "19 GitHub stars, 9 forks - early-stage; no PyPI download signal found.", "sources": [ { "url": "https://github.com/kelkalot/simpleaudit", "shows": "19 stars, 9 forks", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Multilingual adversarial probing with local/open-model support; scenario packs (safety, RAG, healthcare) plus a broken-premise evaluator.", "confidence": "high", "note": "Focused but functional red-teaming toolkit with a distinctive broken-premise mode.", "sources": [ { "url": "https://github.com/kelkalot/simpleaudit", "shows": "README: adversarial probing, local-first, scenario packs, broken-premise runner", "accessed": "2026-06-22" } ] }, "maturity": 2.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive OSI (MIT) license with full public source." } ] }, "benchmark_eval_data": { "label": "Benchmark / eval datasets", "arc": "Model components", "layer": "model_components", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "HumanEval", "org": "OpenAI", "type": "dataset", "description": "164 hand-written Python programming problems with function signatures, docstrings, and unit tests. The original code generation benchmark, introduced in the Codex paper (2021). Still the most widely cited code eval dataset; virtually every code model paper reports pass@k on HumanEval, making it the de facto baseline despite its small size and Python-only scope.", "openness": { "score": 5, "class": "open", "components": "license:MIT(OSI-style permissive, fully redistributable);redistributability:full(downloadable parquet, no gating);datasheet:yes(dataset card);size:164 problems/test split;contamination:hand-written to avoid GitHub-dump overlap (per design)", "confidence": "high", "note": "Fully open: MIT license, ungated, downloadable, with a dataset card. Textbook open-data case for this category.", "sources": [ { "url": "https://huggingface.co/datasets/openai/openai_humaneval", "shows": "MIT license; not gated; 164 problems; dataset card present; ~287k downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Foundational code-generation benchmark cited as a standard in code-model release reports for years (pass@k metric originated here). ~287k HF downloads/month on the canonical config alone, plus heavy mirrored/forked use; sustained 1M+/yr equivalent. The reference code-gen eval despite being partially saturated by frontier models.", "sources": [ { "url": "https://huggingface.co/datasets/openai/openai_humaneval", "shows": "~287k downloads last month on canonical HumanEval config", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "A dataset is not 'capable'. Worth noting it is now largely saturated/contaminated as a discriminator, but per recipe capability is n/a here.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "HumanEval (openai/openai_humaneval), 164 hand-written Python problems (prompt + canonical solution + unit tests), single test split, ~92 kB. From 'Evaluating LLMs Trained on Code' (Codex paper, arXiv:2107.03374). Verified live on HF June 2026 (~287k downloads last month)." }, { "product": "GSM8K", "org": "OpenAI", "type": "dataset", "description": "Grade School Math 8K: a benchmark of grade-school-level math word problems used to test chain-of-thought style reasoning and mathematical problem solving. It is a staple benchmark for evaluating small and frontier models on concise, single-answer math tasks.", "openness": { "score": 5, "class": "open", "components": "license:MIT(OSI-style permissive, fully redistributable);redistributability:full(downloadable parquet, ungated);datasheet:yes(dataset card);size:8.5K problems;splits:train/test public", "confidence": "high", "note": "Fully open: MIT, ungated, downloadable, with a dataset card. Clean open-data case.", "sources": [ { "url": "https://huggingface.co/datasets/openai/gsm8k", "shows": "MIT license; not gated; 8.5K problems (7473 train/1319 test); dataset card; ~948k downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "The standard grade-school math-reasoning benchmark, cited near-universally in LLM release reports for years. ~948k HF downloads/month on the canonical config (the highest in this batch) puts annualized usage comfortably into the 1-10M band.", "sources": [ { "url": "https://huggingface.co/datasets/openai/gsm8k", "shows": "~948k downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'. Now largely saturated by frontier models but per recipe capability is n/a.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "GSM8K (openai/gsm8k), 8.5K grade-school math word problems (7,473 train / 1,319 test), ~5.9 MB, 'main' + 'socratic' configs. From 'Training Verifiers to Solve Math Word Problems' (arXiv:2110.14168). Verified live on HF June 2026 (~948k downloads last month)." }, { "product": "SWE-bench Verified", "org": "Princeton NLP / OpenAI", "type": "dataset", "description": "500 human-verified instances drawn from the broader SWE-bench dataset, each a real GitHub issue paired with a test-verified solution patch. Created through collaboration between Princeton NLP and OpenAI to reduce noise in the original 2,294-instance set. The gold-standard benchmark for evaluating coding agents on realistic software engineering tasks. Leaderboard results drive agent development across the industry. As of April 2026, OpenAI's internal audit found 59.4% of the hardest unsolved problems had flawed test cases and that frontier models can reproduce gold patches verbatim. OpenAI now prefers SWE-bench Pro for headline coding evaluations.", "openness": { "score": 4, "class": "open", "components": "license:undeclared on HF dataset card (underlying SWE-bench harness repo is MIT);redistributability:full(downloadable parquet, ungated);datasheet:yes(dataset card + arXiv:2310.06770);held-out:no hidden split in Verified itself (the public 500-instance set is fully released; gold patches + tests included);data-source:real GitHub PR/issue pairs", "confidence": "medium", "note": "Effectively open (ungated, downloadable, full gold patches and tests, datasheet present) but the HF dataset card declares NO explicit license field, so the data-license class is inferred from the MIT harness repo and public redistribution rather than a stated data license. Scored 4 not 5 on that ambiguity. (Distinct from SWE-bench's broader/hidden variants which can carry held-out splits.)", "sources": [ { "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "shows": "500 instances, ~2.1 MB, not gated, dataset card present, gold patch + FAIL_TO_PASS/PASS_TO_PASS tests included, leaderboard swebench.com", "accessed": "2026-06-04" }, { "url": "https://github.com/SWE-bench/SWE-bench/blob/main/LICENSE", "shows": "MIT License for the SWE-bench harness", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified/blob/main/README.md", "shows": "dataset card metadata has no license field declared", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "The de-facto standard agentic-coding benchmark of 2025-2026: cited as the headline coding metric in nearly every frontier model release (GPT-5, Claude Opus, DeepSeek, Qwen, Kimi all report SWE-bench Verified). ~914k HF downloads/month plus a live, heavily-submitted leaderboard at swebench.com. Strongest 'citation-as-standard' signal in the batch.", "sources": [ { "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "shows": "~914k downloads last month; leaderboard at swebench.com", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'. Notably high contamination-resistance and discriminating power for coding agents (real repo state, executable tests), but per recipe capability is n/a; the quality proxy is noted, not numerically scored.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "SWE-bench Verified (princeton-nlp/SWE-bench_Verified), 500 human-validated GitHub issue-resolution task instances (~2.1 MB) curated by OpenAI from the SWE-bench test set. Leaderboard at swebench.com. Verified live on HF June 2026 (~914k downloads last month)." }, { "product": "CodeContests", "org": "Google DeepMind", "type": "dataset", "description": "A competitive programming dataset containing ~13,000 problems collected from Codeforces, CodeChef, and other contest platforms, each with problem descriptions, solutions in multiple languages, and test cases. Introduced in the AlphaCode paper (2022). Designed to test models on problems requiring algorithmic reasoning far beyond typical function-completion benchmarks, the hard end of code generation evaluation.", "openness": { "score": 5, "class": "open", "components": "license:CC-BY-4.0(open data license, attribution-only);redistributability:full(downloadable, ungated);datasheet:yes(comprehensive dataset card + arXiv:2203.07814);size:~4k problems w/ rich test cases;sources:Codeforces/AtCoder/CodeChef/Aizu/HackerEarth + Description2Code(MIT)+CodeNet(Apache-2.0)", "confidence": "high", "note": "Fully open: CC-BY-4.0, ungated, downloadable, with a thorough datasheet documenting provenance. Clean open-data case on the dataset-openness path.", "sources": [ { "url": "https://huggingface.co/datasets/deepmind/code_contests", "shows": "CC-BY-4.0 license; not gated; 3,760 train/117 val/165 test (4,044 total); dataset card present; ~56k downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "Well-known competitive-programming benchmark (the AlphaCode dataset) and a recurring reference for code-reasoning evals, but materially less ubiquitous than HumanEval/GSM8K/SWE-bench in current release reports. ~56k HF downloads/month (lowest in the batch) places it in the 100K-1M annualized band. Honest signal: solid research adoption, not headline-eval scale.", "sources": [ { "url": "https://huggingface.co/datasets/deepmind/code_contests", "shows": "~56k downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'. High difficulty / good discriminating power for code reasoning, but per recipe capability is n/a.", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "CodeContests (deepmind/code_contests), competitive-programming dataset built for AlphaCode (arXiv:2203.07814). 3,760 train / 117 valid / 165 test problems (4,044 total per HF viewer), ~7.6 GB, with public/private/generated test cases and correct+incorrect human solutions in multiple languages. Verified live on HF June 2026 (~56k downloads last month). [Verifier note: original record stated 13,328 train; HF dataset viewer shows 3,760 train / 4,044 total; corrected.]" }, { "product": "MBPP", "org": "Google Research", "type": "dataset", "description": "Mostly Basic Python Problems, 974 crowd-sourced Python programming tasks designed to be solvable by entry-level programmers, each with a task description, solution, and three automated test cases. Introduced alongside Google's program synthesis work (2021). The complement to HumanEval: where HumanEval tests function-level generation with detailed docstrings, MBPP tests natural-language-to-code translation at a broader difficulty range. Widely used as a standard second benchmark alongside HumanEval.", "openness": { "score": 5, "class": "open", "components": "license:CC-BY-4.0(open data license);redistributability:full(downloadable, ungated);datasheet:yes(extensive dataset card);size:~974 full / 427 sanitized problems w/ test_list;splits:train/test/validation/prompt public", "confidence": "high", "note": "Fully open: CC-BY-4.0, ungated, downloadable, with an extensive dataset card. Clean open-data case; the standard companion to HumanEval for basic Python code-gen eval.", "sources": [ { "url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "shows": "CC-BY-4.0 license; not gated; ~974 full/427 sanitized problems; dataset card present; ~198k downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Standard basic-Python code-generation benchmark, routinely paired with HumanEval and cited in code-model release reports. ~198k HF downloads/month on the canonical config; widely mirrored/forked. Annualized usage and citation-as-standard support a 4, just below the HumanEval/GSM8K tier.", "sources": [ { "url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "shows": "~198k downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'. Largely saturated by frontier models; per recipe capability is n/a.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "MBPP (google-research-datasets/mbpp), ~974 crowd-sourced basic Python problems (full) + 427 hand-verified (sanitized); splits train/test/validation/prompt. From 'Program Synthesis with LLMs' (Austin et al., arXiv:2108.07732). Verified live on HF June 2026 (~198k downloads last month)." }, { "product": "LiveCodeBench", "org": "LiveCodeBench Team", "type": "dataset", "description": "Continuously updated benchmark that scrapes fresh LeetCode/Codeforces/AtCoder problems after a model's training cutoff, ensuring contamination-free evaluation across code generation, self-repair, execution, and test-output prediction. v6 includes 1,000+ problems collected May 2023–2025. Per Artificial Analysis (May 2026), Gemini 3 Pro Preview leads at 91.7%, followed by Gemini 3 Flash Reasoning at 90.8% and DeepSeek V3.2 Speciale at 89.6%; DeepSeek V4-Pro-Max reached 93.5% on a later snapshot. ~870 GitHub stars but disproportionate influence as the trusted dynamic complement to HumanEval/MBPP.", "openness": { "score": 4, "class": "open", "components": "data:downloadable(HF parquet);license:cc(generic CC tag, specific variant unspecified on dataset card);datasheet:present(README documents versions/sources/structure);redistributable:yes;splits:public problems with hidden test cases bundled", "confidence": "medium", "note": "Open and downloadable with a datasheet, but the HF card declares only a generic 'cc' license (no specific CC variant), so the exact reuse terms are ambiguous. Scored open but not a clean CC-BY; capped at 4.", "sources": [ { "url": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "shows": "license:cc, 880 problems (release_v5), 71,393 monthly downloads, datasheet/README", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/datasets/livecodebench/code_generation_lite/blob/main/README.md", "shows": "YAML metadata license: cc", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "71,393 monthly HF downloads (primary). De facto standard contamination-free coding benchmark; LiveCodeBench appears as a headline coding metric in frontier model release reports (e.g. DeepSeek-V4-Pro reports LiveCodeBench Pass@1 93.5 in the frozen flagship set). 'Usage' here is downloads + benchmark citation, not end-users; placed at 4 on download volume plus release-report standard status, which is the top adoption signal for a benchmark.", "sources": [ { "url": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "shows": "71,393 monthly downloads", "accessed": "2026-06-04" }, { "url": "https://livecodebench.github.io/", "shows": "contamination-free coding benchmark across generation/self-repair/execution", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "A dataset is not 'capable'. Contamination-resistance (the 'live' continuously-updated design) is a quality strength but not scored on the 1-5 capability axis per the recipe; openness + adoption carry this category.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "LiveCodeBench code_generation_lite, release_v5 = 880 problems (May 2023-Jan 2025); a 'live', continuously-updated contamination-free coding benchmark (problems from LeetCode/AtCoder/Codeforces) covering code generation, self-repair, test-output prediction, execution. HF dataset verified live June 2026 (71,393 monthly downloads)." }, { "product": "EvalPlus", "org": "EvalPlus Team (UIUC)", "type": "dataset", "description": "HumanEval+ and MBPP+ extend the original benchmarks with 80x and 35x more test cases via automated mutation, exposing models that pass weak tests while emitting buggy code. Published at NeurIPS 2023; benchmarkers now prefer the Plus variants for discriminating among competitive frontier models, since the originals are saturated above 90%. OpenAI's o1 hits ~89% pass@1 on EvalPlus; ~1.7K GitHub stars and a maintained leaderboard make it the standard 'rigorous HumanEval' reference.", "openness": { "score": 5, "class": "open", "components": "data:downloadable(HF parquet, evalplus/humanevalplus + mbppplus);license:Apache-2.0(OSI/open, on both framework and datasets);datasheet:present(card documents structure: task_id/prompt/canonical_solution/test);redistributable:yes;framework-code:open(Apache-2.0)", "confidence": "high", "note": "Clean open data: Apache-2.0 on both datasets and the evaluation framework, downloadable with a documented schema, full open class.", "sources": [ { "url": "https://huggingface.co/datasets/evalplus/humanevalplus", "shows": "Apache-2.0 license, 164 problems, 25,817 monthly downloads, documented schema", "accessed": "2026-06-04" }, { "url": "https://github.com/evalplus/evalplus", "shows": "Apache-2.0 framework; HumanEval+/MBPP+ described", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "HumanEval+ alone: 25,817 monthly HF downloads (MBPP+ adds more). Top adoption signal for a benchmark = citation as a standard in frontier model release reports: EvalPlus is reported adopted by Meta (Llama 3.1/3.3), Alibaba (Qwen2.5-Coder/CodeQwen), DeepSeek, Allen AI (Tulu), Snowflake Arctic, StarCoder2. De facto standard for rigorous code-correctness eval. Level 4 on combined download volume + release-report standard status.", "sources": [ { "url": "https://huggingface.co/datasets/evalplus/humanevalplus", "shows": "25,817 monthly downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/evalplus/evalplus", "shows": "adopted by Meta/Qwen/DeepSeek/AllenAI/StarCoder2 for code eval; NeurIPS 2023", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not capability-scored. Its distinguishing quality (far denser test suites that catch overfitting vs base HumanEval/MBPP) is a quality strength, noted but not on the 1-5 axis.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "EvalPlus framework + datasets: HumanEval+ (164 problems, 80x more tests than original HumanEval) and MBPP+ (35x more tests than original MBPP), plus EvalPerf. Apache-2.0. HumanEval+ HF dataset verified live June 2026 (25,817 monthly downloads). Published NeurIPS 2023 (EvalPlus) / COLM 2024 (EvalPerf)." }, { "product": "MultiPL-E", "org": "Northeastern University / Brown / Wellesley", "type": "dataset", "description": "Multi-programming language translation of HumanEval and MBPP into 18+ languages (JavaScript, TypeScript, C++, Java, Rust, Go, etc.). Enables comparing code generation performance across languages rather than just Python. Published at TOCS 2023. The standard benchmark for evaluating multilingual code models, used in the BigCode and Open LLM Leaderboard evaluations.", "openness": { "score": 4, "class": "open", "components": "data:downloadable(HF parquet, 47 subsets);license:MIT(on HF dataset card);datasheet:present(card documents prompts/doctests/tests/stop-tokens);redistributable:yes(HF);caveat:GitHub repo LICENSE is a non-OSI 'BSD-3-Clause + no-ML-training' modified license, conflicting with the MIT on the HF data artifact", "confidence": "medium", "note": "The scored artifact (HF dataset) is MIT and openly downloadable with a datasheet, open class. But the source repo carries a modified BSD-3 with an explicit 'may not be used as training data for ML' clause (non-OSI), a conflict worth flagging; scored on the data artifact's MIT terms, capped at 4 for the licensing inconsistency.", "sources": [ { "url": "https://huggingface.co/datasets/nuprl/MultiPL-E", "shows": "MIT license, 47 subsets across 22-23 languages, downloadable parquet, documented schema", "accessed": "2026-06-04" }, { "url": "https://github.com/nuprl/MultiPL-E/blob/main/LICENSE", "shows": "modified BSD-3-Clause with no-ML-training-data restriction (non-OSI)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "The standard multilingual code-generation benchmark: integrated into the BigCode Code Generation LM Harness and HF's multilingual code-model evaluation, cited in multilingual code-model release reports (e.g. StarCoder2/CodeQwen). HF page did not surface a clean monthly-download count this run (showed likes only), and GitHub stars (~308) are last-resort and cannot exceed 3. Level 3 on harness-adoption / release-report traction; no verified usage_volume number, hence low confidence.", "sources": [ { "url": "https://huggingface.co/datasets/nuprl/MultiPL-E", "shows": "BigCode harness integration; multilingual code eval standard", "accessed": "2026-06-04" }, { "url": "https://github.com/nuprl/MultiPL-E", "shows": "BigCode harness integration; ~308 stars (last-resort signal)", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset/translation framework, not capability-scored.", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "MultiPL-E: translations of HumanEval (161 problems) and MBPP (397 problems) into 22-23 programming languages (Ada, Clojure, C++, C#, D, Dart, Elixir, Go, Haskell, Java, Julia, JS, Lua, OCaml, PHP, Perl, R, Ruby, Racket, Rust, Scala, Shell, Swift, TS). HF dataset (nuprl/MultiPL-E, 47 subsets) verified live June 2026; integrated into the BigCode evaluation harness. arXiv 2208.08227 (IEEE TSE)." }, { "product": "APPS", "org": "Hendrycks et al. (UC Berkeley)", "type": "dataset", "description": "Automated Programming Progress Standard, 10,000 coding problems at three difficulty levels (introductory, interview, competition) drawn from open-access coding challenge sites. Published at NeurIPS 2021. One of the first large-scale code benchmarks to test models across a realistic difficulty spectrum, bridging the gap between simple function-completion tasks and full competitive programming. Still used as a reference dataset for training and evaluation.", "openness": { "score": 5, "class": "open", "components": "data:downloadable(HF parquet, codeparrot/apps);license:MIT(OSI/open);datasheet:present(card documents 10k problems, 131,777 tests, difficulty levels, schema);redistributable:yes;paper:public(NeurIPS 2021)", "confidence": "high", "note": "Clean open data: MIT, downloadable, documented datasheet, public paper: full open class.", "sources": [ { "url": "https://huggingface.co/datasets/codeparrot/apps", "shows": "MIT license, 10,000 problems, 131,777 test cases, 13,834 monthly downloads", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "13,834 monthly HF downloads (primary). A well-established code-generation benchmark cited widely since 2021, but by 2026 partly superseded by harder/contamination-free suites (LiveCodeBench, SWE-bench) for frontier evaluation. Level 3 on download volume + sustained-but-not-headline citation; honest read is a foundational-but-aging standard.", "sources": [ { "url": "https://huggingface.co/datasets/codeparrot/apps", "shows": "13,834 monthly downloads", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not capability-scored.", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "APPS (Hendrycks et al., NeurIPS 2021, arXiv 2105.09938): 10,000 coding problems (5,000 train / 5,000 test) with 131,777 test cases, Introductory/Interview/Competition difficulty, Python solutions. HF dataset (codeparrot/apps) verified live June 2026 (13,834 monthly downloads, MIT)." }, { "product": "MATH", "org": "Hendrycks et al. (UC Berkeley)", "type": "dataset", "description": "A 12,500-problem benchmark of competition-style mathematics questions spanning algebra, counting, geometry, number theory, prealgebra, precalculus, and more. It is the standard hard math benchmark for measuring reasoning depth beyond GSM8K.", "openness": { "score": 2, "class": "gated", "components": "data:NOT downloadable from canonical HF repo (access disabled, DMCA takedown);license:MIT(declared);datasheet:present(card still documents schema);redistributability:BLOCKED on the primary registry; problems sourced from AMC/AIME competitions, takedown reflects unresolved rights", "confidence": "medium", "note": "Declared MIT, but the canonical HF dataset is access-disabled under a DMCA takedown, so it is not currently redistributable from the primary registry; scored gated/blocked rather than open. Mirrors and eval harnesses still embed it, but the primary source is walled; revisit if the takedown is resolved.", "sources": [ { "url": "https://huggingface.co/datasets/hendrycks/competition_math", "shows": "MIT tag, 12,500 problems, but 'Access Disabled' DMCA takedown notice (discussions/5)", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "MATH is one of the most-cited math-reasoning standards; MATH / MATH-500 appears as a headline metric across frontier model release reports (the top adoption signal for a benchmark). Despite the canonical HF repo being takedown-walled, the benchmark itself remains pervasively used via mirrors and eval harnesses. Level 4 on release-report standard status; no clean current download count obtainable from the primary repo (disabled), hence reported_traction not usage_volume.", "sources": [ { "url": "https://huggingface.co/datasets/hendrycks/competition_math", "shows": "12,500-problem MATH benchmark, NeurIPS 2021, the standard competition-math eval", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not capability-scored.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "MATH (Hendrycks et al., NeurIPS 2021, arXiv 2103.03874): 12,500 competition math problems (7,500 train / 5,000 test) with step-by-step LaTeX solutions, Level 1-5, 7 subjects. Original HF dataset (hendrycks/competition_math) verified June 2026 but ACCESS DISABLED via DMCA takedown; no longer redistributable from the canonical registry; license tag is MIT." }, { "product": "MMLU", "org": "Hendrycks et al. (UC Berkeley)", "type": "dataset", "description": "Massive Multitask Language Understanding: a 15,908-question benchmark spanning 57 subjects, designed to measure broad knowledge and reasoning across humanities, social sciences, STEM, and professional domains. It became one of the default headline evals for frontier models.", "openness": { "score": 5, "class": "open", "components": "data:downloadable(HF parquet, cais/mmlu);license:MIT(OSI/open);datasheet:present(card documents 57 subjects, splits dev/val/test, schema);redistributable:yes;paper:public(ICLR 2021)", "confidence": "high", "note": "Clean open data: MIT, downloadable, documented datasheet, public paper: full open class.", "sources": [ { "url": "https://huggingface.co/datasets/cais/mmlu", "shows": "MIT license, 57 subjects, 231,400 rows / 14,042 test, 541,723 monthly downloads, documented splits", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "541,723 monthly HF downloads (primary), highest in this batch. MMLU is the canonical general-knowledge LLM benchmark, reported in essentially every frontier model release (OpenAI, Anthropic, Google, Meta, DeepSeek, Qwen) since 2021, the strongest possible release-report standard signal. Even post-saturation it remains a universal reference. Level 5 on download volume + ubiquitous release-report citation.", "sources": [ { "url": "https://huggingface.co/datasets/cais/mmlu", "shows": "541,723 monthly downloads", "accessed": "2026-06-04" }, { "url": "https://www.lxt.ai/blog/llm-benchmarks/", "shows": "MMLU listed among the standard LLM benchmarks (corroborating, not headline)", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not capability-scored. Saturation/contamination (top models 88-94%, superseded by MMLU-Pro for differentiation) is a quality caveat noted in version_note, not an openness or capability deduction.", "sources": [] }, "maturity": 5.0, "mature": true, "version_note": "MMLU (Massive Multitask Language Understanding; Hendrycks et al., ICLR 2021, arXiv 2009.03300): 57 subjects, ~14,042 test questions (231,400 total rows incl. auxiliary_train), 4-option multiple choice. HF dataset (cais/mmlu) verified live June 2026 (541,723 monthly downloads, MIT). Note: now largely saturated (top models 88-94%) and largely superseded by MMLU-Pro for frontier differentiation." }, { "product": "Scale SEAL Leaderboard", "org": "Scale AI", "type": "dataset", "description": "Scale AI maintains proprietary evaluation datasets used for its SEAL (Systematic Evaluation of AI Language models) Leaderboard, including coding-specific eval sets. Problems are held private to prevent contamination and scored by expert annotators. Used by enterprise customers to evaluate models before deployment. The commercial alternative to public benchmarks, trading reproducibility for contamination resistance.", "openness": { "score": 1, "class": "closed", "components": "data:closed(proprietary held-out prompt sets, ~1,000 examples/domain, deliberately unpublished to prevent contamination/training-set leakage);datasheet:no(methodology described but examples not redistributable);license:none(not distributed);access:held-out(results visible on leaderboard, dataset not downloadable)", "confidence": "high", "note": "Intentionally private/held-out eval sets per the data-openness rubric (closed = proprietary held-out test suite); only the leaderboard results are public, the data itself is not.", "sources": [ { "url": "https://scale.com/blog/leaderboard", "shows": "Scale states SEAL datasets remain private/unpublished, combining proprietary held-out sets (~1,000 examples) with some open source sets", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "SEAL is a widely-referenced expert-driven private leaderboard cited across the LLM evaluation community as a contamination-resistant ranking; broad press and industry attention but no published download/usage count (the data is held-out by design). Reported-traction tier as an industry-standard reference rather than a measured usage_volume figure.", "sources": [ { "url": "https://scale.com/blog/leaderboard", "shows": "SEAL positioned as a trusted, tamper-proof frontier-model leaderboard used to rank major LLMs", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability is not a meaningful axis for an eval dataset; openness + adoption carry this category per the recipe.", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "Scale AI SEAL Leaderboards (Safety, Evaluations, and Alignment Lab); launched 2024, regularly updated with new domains (instruction-following, math, coding, multilinguality, etc.). The scored artifact is the private held-out eval sets behind the leaderboard. Verified live June 2026 via labs.scale.com/leaderboard (redirect from scale.com/leaderboard) and scale.com/blog/leaderboard." }, { "product": "OpenAI Internal Evals", "org": "OpenAI (Internal)", "type": "dataset", "description": "OpenAI maintains proprietary internal evaluation suites for coding capability assessment, used in model development and safety evaluations under their Preparedness Framework. These include harder coding challenges, agentic task completions, and cybersecurity-related coding tests that are never publicly released. Referenced in system cards and safety reports but datasets remain private to prevent contamination and maintain evaluation integrity.", "openness": { "score": 0, "class": "closed", "components": "data:closed(internal eval sets for frontier-risk Tracked Categories, never published);datasheet:no;license:none;access:internal-only(only summarized/redacted results released alongside major deployments)", "confidence": "high", "note": "Fully internal/private eval datasets; only redacted result summaries are published, the underlying example sets are not redistributable.", "sources": [ { "url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "shows": "Preparedness Framework v2 describes internal capability evaluations per Tracked Category with disclosures redacted/summarized to protect IP and safety", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "high", "note": "Internal-only eval sets used solely by OpenAI for its own deployment decisions; no external users, no downloads, no third-party citation as a runnable standard. No honest external adoption signal exists.", "sources": [ { "url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "shows": "evaluations are internal to OpenAI's deployment process, not a shared/usable external dataset", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability not a meaningful axis for an eval dataset.", "sources": [] }, "maturity": null, "mature": false, "version_note": "OpenAI internal dangerous-capability evaluations referenced in the Preparedness Framework (v2, updated 15 Apr 2025). The scored artifact is the internal eval sets used for Tracked Categories. Verified via the OpenAI Preparedness Framework v2 PDF (cdn.openai.com) and openai.com/index/updating-our-preparedness-framework. NOT the separate open source openai/evals harness (that is evaluation_code, not a dataset)." }, { "product": "Anthropic Internal Coding Evals", "org": "Anthropic (Internal)", "type": "dataset", "description": "Anthropic maintains proprietary coding evaluation datasets used to benchmark Claude models during development, including agentic coding tasks, multi-file editing scenarios, and tool-use evaluations. Referenced in model cards and responsible scaling policy documents. Like other frontier labs, these are kept private to prevent training data contamination and preserve evaluation signal.", "openness": { "score": 0, "class": "closed", "components": "data:closed(internal coding eval/take-home exam sets, never published);datasheet:no;license:none;access:internal-only(referenced in announcements/system cards, not distributed)", "confidence": "high", "note": "Internal proprietary coding eval sets; only narrative results are mentioned, the data is not redistributable.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-5", "shows": "references to internal coding benchmarks and an internal performance-engineering take-home exam, described as proprietary/internal, alongside (separately) public benchmarks", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "high", "note": "Internal-only eval sets used solely by Anthropic; no external users, downloads, or third-party citation as a runnable standard. No honest external adoption signal.", "sources": [ { "url": "https://www.anthropic.com/news/claude-opus-4-5", "shows": "evaluations described as internal benchmarks, not a shared external dataset", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability not a meaningful axis for an eval dataset.", "sources": [] }, "maturity": null, "mature": false, "version_note": "Anthropic internal coding evaluations / benchmarks referenced (not released) in Claude model launches and system cards (e.g. Claude Opus 4.5, Nov 2025; partners GitHub Copilot cite 'internal coding benchmarks'). The scored artifact is the internal coding eval sets. Verified via anthropic.com/news/claude-opus-4-5. These are distinct from the public benchmarks (SWE-bench Verified, Aider Polyglot) Anthropic also reports." }, { "product": "CRUXEval", "org": "Meta", "type": "dataset", "description": "800 Python (function, input, output) triples evaluating code reasoning, understanding, and execution via two paired tasks: CRUXEval-I (predict input given function + output) and CRUXEval-O (predict output given function + input). Bootstrapped by Meta FAIR using Code Llama 34B and filtered to short, low-compute problems, with a maintained public leaderboard at crux-eval.github.io. Published at ICML 2024, widely reported in code-model papers and model cards as the standard execution-reasoning eval distinct from generation benchmarks like HumanEval/MBPP; spawned a community multilingual extension (CRUXEval-X, 12.66K subjects across 19 languages). Repo archived October 2024 but benchmark remains in active comparative use.", "openness": { "score": 5, "class": "open", "components": "data:open(800 samples public on GitHub + HF);license:MIT(permissive, redistributable);datasheet:yes(README + arXiv paper documenting construction);access:public(downloadable, fully released, not held-out)", "confidence": "high", "note": "Fully open public benchmark dataset under MIT with documented construction methodology.", "sources": [ { "url": "https://github.com/facebookresearch/cruxeval", "shows": "MIT-licensed 800-sample Python code-reasoning benchmark, publicly downloadable with CRUXEval-I/-O tasks", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Established code-reasoning benchmark cited in code-model evaluations and research since 2024; from Meta FAIR with a dedicated leaderboard. Solid research/eval-standard traction but a specialist code-execution benchmark, not mass-scale; placed at 3 on reported research/citation traction rather than a measured download figure.", "sources": [ { "url": "https://github.com/facebookresearch/cruxeval", "shows": "maintained Meta FAIR benchmark repo with leaderboard, used as a code reasoning/execution evaluation standard", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability not a meaningful axis for an eval dataset.", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) by Meta/FAIR (Gu, Roziere, Leather, Solar-Lezama, Synnaeve, Wang; arXiv 2401.03065, 2024). 800 Python functions with input/output pairs; two tasks: CRUXEval-I (input prediction) + CRUXEval-O (output prediction). Verified live June 2026 on github.com/facebookresearch/cruxeval." }, { "product": "AI2 ARC", "org": "Allen Institute for AI", "type": "dataset", "description": "A benchmark of 7,787 genuine grade-school science questions assembled to study advanced question answering. The dataset is split into an Easy set and a Challenge set, and it remains one of the canonical multiple-choice reasoning benchmarks for language models.", "openness": { "score": 5, "class": "open", "components": "data:open(7,787 questions, downloadable on HF);license:CC-BY-SA-4.0(open data license, redistributable);datasheet:yes(HF dataset card + arXiv paper);access:public(train/val/test splits all released, not held-out)", "confidence": "high", "note": "Fully open under CC-BY-SA-4.0 with dataset card; the canonical open-data benchmark profile.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/ai2_arc", "shows": "CC-BY-SA-4.0, 7,787 questions, public splits, dataset card, ~479K downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~479K HF downloads in the last month alone (so >1M cumulative) and one of the most-cited standard reasoning benchmarks routinely reported in frontier model release cards. Strong usage_volume signal as an entrenched evaluation standard.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/ai2_arc", "shows": "~479,090 downloads in the last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability not a meaningful axis for an eval dataset.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "AI2 Reasoning Challenge (ARC), Allen Institute for AI (Clark et al. 2018, arXiv:1803.05457). 7,787 grade-school science multiple-choice questions: ARC-Challenge (2,590) + ARC-Easy (5,197). Foundational LLM commonsense/reasoning benchmark, still a standard in 2026 model release reports. Verified live June 2026 on huggingface.co/datasets/allenai/ai2_arc." }, { "product": "HellaSwag", "org": "Allen Institute for AI", "type": "dataset", "description": "A commonsense NLI and sentence-completion benchmark introduced in 2019. It contains roughly 70,000 multiple-choice examples and is widely used to test whether models can choose the plausible continuation of a sentence in everyday contexts.", "openness": { "score": 5, "class": "open", "components": "data:open(59,950 examples, downloadable on HF);license:MIT(permissive, redistributable);datasheet:yes(HF dataset card + ACL 2019 paper);access:public(train/val/test all released, not held-out)", "confidence": "high", "note": "Fully open under MIT with dataset card; canonical open-data benchmark profile.", "sources": [ { "url": "https://huggingface.co/datasets/Rowan/hellaswag", "shows": "MIT license, 59,950 examples, public splits, dataset card, ~270K downloads last month", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~270K HF downloads last month (>1M cumulative) and a near-ubiquitous commonsense benchmark cited across LLM release reports and the Open LLM Leaderboard. Strong usage_volume signal as an entrenched standard.", "sources": [ { "url": "https://huggingface.co/datasets/Rowan/hellaswag", "shows": "~270,262 downloads in the last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Capability not a meaningful axis for an eval dataset.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "HellaSwag (Zellers et al., ACL 2019), commonsense NLI sentence-completion benchmark. 59,950 examples (39,905 train / 10,042 val / 10,003 test); 4-way multiple-choice over ActivityNet-derived contexts. Canonical commonsense benchmark reported in nearly every LLM release. Verified live June 2026 on huggingface.co/datasets/Rowan/hellaswag. Registry attributes to AI2/Allen Institute; primary authors are UW/AI2 (Rowan Zellers)." }, { "product": "GPQA", "org": "", "type": "dataset", "description": "GPQA (Graduate-Level Google-Proof Q&A), 448 expert-written multiple-choice questions in biology/physics/chemistry; the 198-question 'GPQA Diamond' high-quality subset is the headline benchmark cited in most 2025-2026 frontier model reports. Paper arXiv:2311.12022. HF dataset Idavidrein/gpqa verified live June 2026.", "openness": { "score": 4, "class": "gated", "components": "license:CC-BY-4.0(open,redistributable+attribution);access:gated(HF login + click-through agreement NOT to publish examples in plaintext, to limit training-corpus contamination);datasheet:present(paper+card);splits:public(no hidden held-out test)", "confidence": "high", "note": "Open CC-BY-4.0 license and fully redistributable, but access is login-walled with an anti-contamination click-through agreement, so the headline class is gated rather than open.", "sources": [ { "url": "https://huggingface.co/datasets/Idavidrein/gpqa", "shows": "CC-BY-4.0 license, gated access with anti-leakage agreement, 448 questions, dataset card present", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/abs/2311.12022", "shows": "GPQA paper documenting 448 expert questions + Diamond subset", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~129,881 HF downloads in the trailing month (primary, from the dataset page); GPQA Diamond is a de-facto-standard reasoning benchmark cited across frontier model release reports (appears as a capability basis throughout the v3 flagship set), which is the top adoption signal for a benchmark.", "sources": [ { "url": "https://huggingface.co/datasets/Idavidrein/gpqa", "shows": "129,881 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Per recipe, a benchmark dataset is not 'capable'; openness and adoption carry this category. No capability axis assessed.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "GPQA (Graduate-Level Google-Proof Q&A), 448 expert-written multiple-choice questions in biology/physics/chemistry; the 198-question 'GPQA Diamond' high-quality subset is the headline benchmark cited in most 2025-2026 frontier model reports. Paper arXiv:2311.12022. HF dataset Idavidrein/gpqa verified live June 2026." }, { "product": "IFEval", "org": "Google", "type": "dataset", "description": "Instruction-Following Eval (IFEval) is a benchmark of 541 verifiable instructions used to measure how well models follow explicit constraints such as word-count limits, keyword inclusion, and format requirements. It is widely used for instruction-following and controllability evaluation.", "openness": { "score": 5, "class": "open", "components": "license:Apache-2.0(OSI/open,redistributable);access:public(not gated);datasheet:present(card+structure+citation);splits:public(single train split, no held-out)", "confidence": "high", "note": "Fully open: Apache-2.0, ungated, redistributable, with a complete dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/google/IFEval", "shows": "Apache-2.0 license, not gated, 541 examples, dataset card present", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~110,541 HF downloads in the trailing month (primary); a core benchmark on the HF Open LLM Leaderboard and routinely cited as the standard instruction-following eval, the top adoption signal for a benchmark.", "sources": [ { "url": "https://huggingface.co/datasets/google/IFEval", "shows": "110,541 downloads last month; described as core Open LLM Leaderboard benchmark", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'; openness and adoption carry this category per recipe.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "IFEval (Instruction-Following Eval), 541 prompts with ~25 verifiable instruction types for programmatic (rule-based) scoring of LLM instruction-following. Paper arXiv:2311.07911. A core component of the HF Open LLM Leaderboard. HF dataset google/IFEval verified live June 2026." }, { "product": "MMLU-Pro", "org": "TIGER-Lab", "type": "dataset", "description": "A harder successor to MMLU with 12,000 complex multiple-choice questions across many disciplines. It is designed to benchmark large language models more rigorously than the original MMLU and has become a common upgrade path for general knowledge and reasoning evals.", "openness": { "score": 5, "class": "open", "components": "license:MIT(OSI/open,redistributable);access:public(not gated);datasheet:present(comprehensive card+leaderboard);splits:public(test+validation, no hidden held-out)", "confidence": "high", "note": "Fully open: MIT, ungated, redistributable, comprehensive card.", "sources": [ { "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "shows": "MIT license, not gated, 12,032 questions, dataset card + leaderboard present", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/abs/2406.01574", "shows": "MMLU-Pro paper documenting 14 disciplines, 10-option robust benchmark", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~154,919 HF downloads in the trailing month (primary, the highest of this batch); MMLU-Pro is a de-facto-standard knowledge/reasoning benchmark cited across frontier model reports (appears as a capability basis throughout the v3 flagship set).", "sources": [ { "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "shows": "154,919 downloads last month; active leaderboard with 27+ models", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'; openness and adoption carry this category per recipe.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "MMLU-Pro, ~12,032 questions across 14 disciplines with 10 answer options (vs MMLU's 4), expert-reviewed and contamination-hardened; test split (12,000) + small validation (70). Paper arXiv:2406.01574. HF dataset TIGER-Lab/MMLU-Pro verified live June 2026." }, { "product": "MMMU", "org": "", "type": "dataset", "description": "MMMU (Massive Multi-discipline Multimodal Understanding), ~11,550 college-level multimodal (image+text) questions across 30 subjects/183 subfields, 30 image types. Splits: dev 150 / validation 900 / test 10,500. Test-set answers were previously held out behind EvalAI but were RELEASED on 2026-02-12, so all splits are now publicly answerable. CVPR 2024, paper arXiv:2311.16502. HF dataset MMMU/MMMU verified live June 2026.", "openness": { "score": 5, "class": "open", "components": "license:Apache-2.0(OSI/open,redistributable);access:public(not gated);datasheet:present(comprehensive card);splits:dev+validation+test all public (test answers released 2026-02-12; previously EvalAI-gated)", "confidence": "high", "note": "Fully open: Apache-2.0, ungated, redistributable, comprehensive card. Test-set answers were released 2026-02-12 (previously held out via EvalAI), so there is no longer a held-out element; scored 5 as a fully-public benchmark.", "sources": [ { "url": "https://huggingface.co/datasets/MMMU/MMMU", "shows": "Apache-2.0, not gated, 11,550 questions (dev 150/val 900/test 10,500); test answers released 2026-02-12 (no longer EvalAI-held-out); 86,107 downloads last month", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/abs/2311.16502", "shows": "MMMU paper documenting 30 subjects, 183 subfields, multimodal benchmark", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~86,107 HF downloads in the trailing month (primary); MMMU / MMMU-Pro is the de-facto-standard multimodal-understanding benchmark cited in frontier multimodal model reports (e.g. GPT-5 reports MMMU; Gemini reports MMMU-Pro in the v3 flagship set).", "sources": [ { "url": "https://huggingface.co/datasets/MMMU/MMMU", "shows": "86,107 downloads last month; EvalAI evaluation infrastructure", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'; openness and adoption carry this category per recipe.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "MMMU (Massive Multi-discipline Multimodal Understanding), ~11,550 college-level multimodal (image+text) questions across 30 subjects/183 subfields, 30 image types. Splits: dev 150 / validation 900 / test 10,500. Test-set answers were previously held out behind EvalAI but were RELEASED on 2026-02-12, so all splits are now publicly answerable. CVPR 2024, paper arXiv:2311.16502. HF dataset MMMU/MMMU verified live June 2026." }, { "product": "TruthfulQA", "org": "", "type": "dataset", "description": "TruthfulQA, 817 questions across 38 categories crafted to elicit imitative falsehoods/misconceptions; generation + multiple_choice configs. Paper arXiv:2109.07958 (2021). Now relatively dated and saturated by frontier models but still a widely-cited truthfulness benchmark. HF dataset truthfulqa/truthful_qa verified live June 2026.", "openness": { "score": 5, "class": "open", "components": "license:Apache-2.0(OSI/open,redistributable);access:public(not gated);datasheet:present(card+creation methodology+citation);splits:public(817 Qs, two configs, no hidden held-out)", "confidence": "high", "note": "Fully open: Apache-2.0, ungated, redistributable, complete card with creation methodology.", "sources": [ { "url": "https://huggingface.co/datasets/truthfulqa/truthful_qa", "shows": "Apache-2.0 license, not gated, 817 questions, dataset card present", "accessed": "2026-06-04" }, { "url": "https://arxiv.org/abs/2109.07958", "shows": "TruthfulQA paper documenting 817 questions, 38 categories", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "~97,282 HF downloads in the trailing month (primary); long-standing widely-cited truthfulness benchmark. Medium confidence on the level because the benchmark is older (2021) and increasingly saturated, so download volume may overstate its current role in frontier evaluation vs GPQA/MMLU-Pro.", "sources": [ { "url": "https://huggingface.co/datasets/truthfulqa/truthful_qa", "shows": "97,282 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "Dataset, not 'capable'; openness and adoption carry this category per recipe.", "sources": [] }, "maturity": 4.0, "mature": false, "version_note": "TruthfulQA, 817 questions across 38 categories crafted to elicit imitative falsehoods/misconceptions; generation + multiple_choice configs. Paper arXiv:2109.07958 (2021). Now relatively dated and saturated by frontier models but still a widely-cited truthfulness benchmark. HF dataset truthfulqa/truthful_qa verified live June 2026." }, { "product": "MT-Bench", "org": "SGLang (LMSYS / UC Berkeley)", "type": "dataset", "description": "An 80-question, multi-turn, open-ended benchmark for LLM instruction-following and conversational ability, scored with a strong LLM (e.g. GPT-4) as judge. Created by LMSYS, it ships inside FastChat with both questions and GPT-4 reference answers public. Introduced in the NeurIPS 2023 'LLM-as-a-Judge' paper.", "openness": { "score": 5, "class": "open", "components": "questions:public;answers:public(GPT-4 references in FastChat);license:Apache-2.0", "confidence": "high", "note": "Both questions and reference answers are public and openly licensed.", "sources": [ { "url": "https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge/data/mt_bench", "shows": "public questions + GPT-4 references", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "Ships in FastChat (~39.5k stars, shared across the platform) - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/lm-sys/FastChat", "shows": "Apache-2.0, ~39.5k stars, MT-Bench in llm_judge", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "80 multi-turn questions across 8 categories; LLM-as-judge scoring", "confidence": "medium", "note": "Seminal multi-turn chat benchmark that pioneered LLM-as-judge; now small and partly saturated vs harder peers.", "sources": [ { "url": "https://arxiv.org/abs/2306.05685", "shows": "MT-Bench / LLM-as-a-Judge paper", "accessed": "2026-06-17" } ] }, "maturity": 3.4, "mature": false, "version_note": "Questions + GPT-4 reference answers public in FastChat (Apache-2.0); companion human-judgments dataset is CC-BY-4.0. Small (80 Q) and partly saturated now." }, { "product": "LiveBench", "org": "LiveBench", "type": "dataset", "description": "An open, contamination-limited LLM benchmark with objective ground-truth answers and automatic scoring (no LLM judges), spanning six categories (math, coding, reasoning, language, instruction following, data analysis). New questions are released monthly and older ones retired to limit contamination. Built by Abacus.AI, NYU, Nvidia, UMD, USC; ICLR 2025 Spotlight.", "openness": { "score": 5, "class": "open", "components": "questions:public;answers:public(ground_truth released);license:Apache-2.0/MIT", "confidence": "high", "note": "All questions, answers, and code are released openly; rotation is for freshness, not access control.", "sources": [ { "url": "https://huggingface.co/datasets/livebench/reasoning", "shows": "public ground_truth column, release/removal dates", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~1.2k GitHub stars plus multi-category HF dataset downloads (genuine multi-source adoption).", "sources": [ { "url": "https://github.com/livebench/livebench", "shows": "~1.2k stars, six categories, monthly releases", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "6 domains; objective ground-truth auto-scoring (no LLM judge); monthly question rotation for contamination resistance", "confidence": "high", "note": "Among the most influential contamination-resistant benchmarks; broader than code-only LiveCodeBench and judge-free.", "sources": [ { "url": "https://arxiv.org/abs/2406.19314", "shows": "ICLR 2025 Spotlight, releases all questions/code/answers", "accessed": "2026-06-17" } ] }, "maturity": 4.4, "mature": false, "version_note": "Questions and ground-truth answers public (Apache-2.0/MIT). ~1.2k stars; multi-category HF dataset downloads. Monthly rotation is freshness management, not gating." }, { "product": "GAIA (General AI Assistants)", "org": "Meta", "type": "dataset", "description": "A benchmark for General AI Assistants from Meta AI / Hugging Face: 466 real-world questions across 3 difficulty levels requiring reasoning, multi-modality, web browsing, and tool use. Simple for humans (92%) but hard for frontier systems (GPT-4+plugins scored 15% at release). A 300-question test set with private answers powers a submission-based leaderboard.", "openness": { "score": 2, "class": "gated", "components": "questions:public(val);answers:held-out(300-Q test,private);access:HF-gated(accept-terms,anti-resharing)", "confidence": "high", "note": "Test answers are private and the dataset itself is access-gated.", "sources": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "shows": "gated access, private test answers", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~42k monthly HF downloads; widely cited and integrated into agent eval harnesses.", "sources": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "shows": "~42,218 downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "466 questions, 3 levels; multi-step tool use, live browsing, multi-modal files with one checkable answer", "confidence": "high", "note": "The canonical end-to-end agentic-assistant benchmark; distinct from knowledge and code benchmarks, contamination-resistant.", "sources": [ { "url": "https://arxiv.org/abs/2311.12983", "shows": "466 Q, 92% human vs 15% GPT-4, held-out answers", "accessed": "2026-06-17" } ] }, "maturity": 4.4, "mature": false, "version_note": "Double-gated: test-set answers held out (server-side scoring) AND the HF dataset is access-gated (accept terms, anti-resharing clause). ~42k monthly downloads." }, { "product": "Humanity's Last Exam (HLE)", "org": "Center for AI Safety", "type": "dataset", "description": "A multi-modal frontier-knowledge benchmark from the Center for AI Safety and Scale AI, built with ~1,000 subject-matter experts: 2,500 expert-written closed-ended questions (MC + short answer, text+image) across 100+ subjects, each with an unambiguous answer not quickly findable online. Positioned as the hardest frontier academic benchmark.", "openness": { "score": 2, "class": "gated", "components": "questions+answers:public(MIT,2500-Q);access:HF-gated(accept-conditions,no-redistribution,canary);holdout:private", "confidence": "medium", "note": "Public set is MIT-licensed with answers, but access is gated and a private held-out set exists - judgment call (could be open under a license-weighted reading).", "sources": [ { "url": "https://huggingface.co/datasets/cais/hle", "shows": "MIT, gated access, no-redistribution notice", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~35k monthly HF downloads; ~1.6k GitHub stars on the eval code.", "sources": [ { "url": "https://huggingface.co/datasets/cais/hle", "shows": "~34,984 downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "2,500 expert questions across 100+ subjects; multimodal; private held-out set; designed as successor to saturated MMLU/MATH", "confidence": "high", "note": "Far harder and broader than MMLU/MATH; GPQA is a narrower difficulty peer.", "sources": [ { "url": "https://lastexam.ai", "shows": "~1,000 experts, 2,500 Q, 100+ subjects, private holdout", "accessed": "2026-06-17" } ] }, "maturity": 4.4, "mature": false, "version_note": "The 2,500 public questions include answers under MIT, but the HF dataset is access-gated (accept conditions, no-redistribution + canary string) and a separate private held-out set detects overfitting -> scored gated (a license-weighted reading could argue open). ~35k monthly downloads." }, { "product": "Compar:IA Datasets", "org": "Ministere de la Culture (France)", "type": "dataset", "description": "The Compar:IA datasets are human-preference and conversation data released from the French Government's Compar:IA LLM arena, published on Hugging Face by the Ministere de la Culture (SNUM). The collection includes comparia-votes (150,000+ conversation-level pairwise preference annotations), comparia-conversations, and comparia-reactions. The data is ~89% French and is intended for finetuning, alignment, and benchmarking of multilingual LLMs.", "openness": { "score": 3, "class": "gated", "components": "license:Etalab-2.0(permissive French-gov open license);gated:HF contact-info agreement required;dataset card present", "confidence": "high", "note": "Open license and documented, but gated access requiring agreement caps it at gated.", "sources": [ { "url": "https://huggingface.co/datasets/ministere-culture/comparia-votes", "shows": "License Etalab 2.0; gated access requiring contact-info agreement; dataset card present", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "niche", "signal_type": "usage_volume", "confidence": "high", "note": "comparia-votes ~155 and comparia-conversations ~202 downloads last month on Hugging Face.", "sources": [ { "url": "https://huggingface.co/datasets/ministere-culture/comparia-votes", "shows": "155 downloads last month", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Open license and documented, but gated access requiring agreement caps it at gated." } ] }, "training_synthetic_datasets": { "label": "Training & synthetic datasets", "arc": "Model components", "layer": "model_components", "stage": { "num": 2, "name": "Emerging Alternatives" }, "gaps": [ "maturity", "adoption" ], "products": [ { "product": "FineWeb", "org": "Hugging Face", "type": "dataset", "description": "FineWeb is a large-scale English pretraining dataset of over 18.5 trillion tokens of cleaned and deduplicated text derived from 96 CommonCrawl snapshots (2013-2024), released by Hugging Face's FineWeb team. It is distributed as Parquet across ~114 date-based subsets. It was built to provide an open, high-quality web corpus that outperforms other public web datasets for LLM training, and is a widely cited reference in the open-data pretraining community.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY;gated:false;dataset_card:present;format:parquet", "confidence": "high", "note": "Permissive ODC-BY license with ungated download and a full dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb", "shows": "Dataset card present, ODC-BY license, public ungated download", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100k-1M", "signal_type": "usage_volume", "confidence": "high", "note": "327,873 monthly downloads on Hugging Face.", "sources": [ { "url": "https://huggingface.co/api/datasets/HuggingFaceFW/fineweb", "shows": "downloads field = 327,873 monthly", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive ODC-BY license with ungated download and a full dataset card." }, { "product": "FineWeb-Edu", "org": "Hugging Face", "type": "dataset", "description": "FineWeb-Edu is an educational subset of FineWeb (~1.3 trillion tokens) created by Hugging Face's FineWeb team. It was produced by filtering FineWeb with an educational-quality classifier trained on Llama-3-70B annotations, retaining text scored as highly educational. It is intended to improve LLM performance on knowledge- and reasoning-heavy benchmarks relative to unfiltered web data.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY;gated:false;dataset_card:present;format:parquet", "confidence": "high", "note": "Permissive ODC-BY license with ungated download and full dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", "shows": "Dataset card present, ODC-BY license, public ungated download", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100k-1M", "signal_type": "usage_volume", "confidence": "high", "note": "411,911 monthly downloads on Hugging Face.", "sources": [ { "url": "https://huggingface.co/api/datasets/HuggingFaceFW/fineweb-edu", "shows": "downloads field = 411,911 monthly", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive ODC-BY license with ungated download and full dataset card." }, { "product": "FineWeb-2", "org": "Hugging Face", "type": "dataset", "description": "FineWeb-2 is the multilingual successor to FineWeb, covering over 1,000 languages and writing systems with more than 1 trillion tokens of filtered and deduplicated CommonCrawl text, released by Hugging Face's FineWeb team. Each language is processed through a per-language pipeline with deduplication and quality filtering. It extends the open-data pretraining work beyond English to support multilingual LLM training.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY;gated:false;dataset_card:present;languages:1000+", "confidence": "high", "note": "Permissive ODC-BY license, ungated, with a dataset card spanning 1000+ languages.", "sources": [ { "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-2", "shows": "Dataset page exists on public hub, ODC-BY, 1000+ languages", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "94,143 monthly downloads on Hugging Face.", "sources": [ { "url": "https://huggingface.co/api/datasets/HuggingFaceFW/fineweb-2", "shows": "downloads field = 94,143 monthly", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive ODC-BY license, ungated, with a dataset card spanning 1000+ languages." }, { "product": "DCLM-baseline", "org": "DataComp-LM (ML Foundations)", "type": "dataset", "description": "DCLM-baseline-1.0 is a curated CommonCrawl-derived pretraining dataset of about 4 trillion tokens across 3 billion documents (~7.2TB), produced by the DataComp-LM (ML Foundations) project. It was built using heuristic cleaning, Bloom-filter deduplication, and fastText model-based quality filtering, and is the reference dataset for the DCLM benchmark. It is positioned as a research dataset that outperforms prior open web datasets when training 7B models.", "openness": { "score": 5, "class": "open", "components": "license:CC-BY-4.0;gated:false;dataset_card:present;github:mlfoundations/dclm", "confidence": "high", "note": "Permissive CC-BY-4.0 license, ungated download, with open code repo and dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0", "shows": "Dataset card, CC-BY-4.0 license, ungated public download, GitHub link", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100k-1M", "signal_type": "usage_volume", "confidence": "high", "note": "542,120 monthly downloads on Hugging Face.", "sources": [ { "url": "https://huggingface.co/api/datasets/mlfoundations/dclm-baseline-1.0", "shows": "downloads field = 542,120 monthly", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive CC-BY-4.0 license, ungated download, with open code repo and dataset card." }, { "product": "RefinedWeb", "org": "Technology Innovation Institute", "type": "dataset", "description": "Falcon RefinedWeb is a large English web dataset (~968M rows, ~2.8TB) built by the Technology Innovation Institute (TII) via stringent filtering and exact-plus-fuzzy deduplication of CommonCrawl. The public release is an extract of the larger corpus used to train the Falcon-7B/40B models. It demonstrated that rigorously filtered web data alone can match or beat curated corpora for LLM pretraining.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY-1.0;gated:false;dataset_card:present;note:public_extract", "confidence": "high", "note": "Permissive ODC-BY 1.0 license, ungated download, with a dataset card (a public extract of the full corpus).", "sources": [ { "url": "https://huggingface.co/datasets/tiiuae/falcon-refinedweb", "shows": "Dataset card, ODC-By 1.0 license, ungated public download", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "11,850 monthly downloads on Hugging Face.", "sources": [ { "url": "https://huggingface.co/api/datasets/tiiuae/falcon-refinedweb", "shows": "downloads field = 11,850 monthly", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive ODC-BY 1.0 license, ungated download, with a dataset card (a public extract of the full corpus)." }, { "product": "Dolma", "org": "Allen Institute for AI", "type": "dataset", "description": "Dolma is an open pretraining corpus of roughly 3 trillion tokens drawn from Common Crawl web text, academic publications, code, books, and encyclopedic material. It was created by the Allen Institute for AI (AI2) and used to train the OLMo family of models. The release includes the corpus plus an open source toolkit for replicating and curating the data.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY;ungated:yes;dataset_card:yes;code:apache-2.0-toolkit", "confidence": "high", "note": "Permissively licensed (ODC-BY), ungated download with a full dataset card and replication toolkit.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/dolma", "shows": "ODC-BY license, ungated access, dataset card, ~3T tokens", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10k", "signal_type": "usage_volume", "confidence": "high", "note": "HF reports 4,074 monthly downloads.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/dolma", "shows": "4,074 monthly downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 1.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissively licensed (ODC-BY), ungated download with a full dataset card and replication toolkit." }, { "product": "RedPajama-Data-v2", "org": "Together AI", "type": "dataset", "description": "RedPajama-Data-v2 is a large open web dataset built from Common Crawl, containing over 100B text documents with around 30B documents annotated with quality signals and ~20B deduplicated documents (tens of trillions of tokens). It was created by Together Computer to support open language-model pretraining. The processing scripts are Apache-2.0, while the underlying text follows Common Crawl's Terms of Use.", "openness": { "score": 5, "class": "open", "components": "license:CommonCrawl-ToU+Apache-2.0(code);ungated:yes;dataset_card:yes", "confidence": "high", "note": "Ungated with a full dataset card; redistributable per Common Crawl ToU with Apache-2.0 processing code.", "sources": [ { "url": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2", "shows": "Common Crawl ToU + Apache-2.0 code license, ungated access, dataset card", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10k", "signal_type": "usage_volume", "confidence": "high", "note": "HF reports 7,258 monthly downloads.", "sources": [ { "url": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2", "shows": "7,258 monthly downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 1.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Ungated with a full dataset card; redistributable per Common Crawl ToU with Apache-2.0 processing code." }, { "product": "The Stack v2", "org": "BigCode Project", "type": "dataset", "description": "The Stack v2 is a large source-code dataset of 3.28B unique files (67.53TB uncompressed) from 104.2M GitHub repositories spanning 658 programming languages, sourced via Software Heritage. It was created by the BigCode project and used to train the StarCoder2 models. Files carry permissive licenses, but access requires accepting terms of use tied to Software Heritage principles.", "openness": { "score": 3, "class": "gated", "components": "license:permissive(SPDX);gated:terms-acceptance+contact-info;dataset_card:yes", "confidence": "high", "note": "Requires acceptance of terms and sharing contact info before download, so access is gated.", "sources": [ { "url": "https://huggingface.co/datasets/bigcode/the-stack-v2", "shows": "gated access requiring terms acceptance and contact-info sharing, permissive licenses, dataset card", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "HF reports 23,823 monthly downloads.", "sources": [ { "url": "https://huggingface.co/datasets/bigcode/the-stack-v2", "shows": "23,823 monthly downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Requires acceptance of terms and sharing contact info before download, so access is gated." }, { "product": "C4", "org": "Allen Institute for AI", "type": "dataset", "description": "C4 (Colossal Clean Crawled Corpus) is a cleaned version of Common Crawl created for the T5 paper, hosted on Hugging Face by AI2. It offers multiple variants, including the cleaned English split (~305GB) and the multilingual mC4 covering 108 languages (~9.7TB). It is one of the most widely used English pretraining corpora.", "openness": { "score": 5, "class": "open", "components": "license:ODC-BY;ungated:yes;dataset_card:yes", "confidence": "high", "note": "Permissively licensed (ODC-BY) and ungated with a full dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/c4", "shows": "ODC-BY license, ungated access, dataset card, size variants", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100k-1M", "signal_type": "usage_volume", "confidence": "high", "note": "HF reports 831,302 monthly downloads.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/c4", "shows": "831,302 monthly downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissively licensed (ODC-BY) and ungated with a full dataset card." }, { "product": "Common Corpus", "org": "PleIAs", "type": "dataset", "description": "Common Corpus is a multilingual pretraining dataset of roughly 2.27 trillion tokens containing only uncopyrighted or openly licensed content across 367+ languages. It was assembled by PleIAs with partners including the AI Alliance, organized into six curated collections (public-domain books, government documents, academic content, code, semantic data, and web text) with document-level provenance and toxicity filtering. It is one of the largest openly licensed text corpora for LLM training.", "openness": { "score": 5, "class": "open", "components": "license:open(uncopyrighted/freely-licensed only);ungated:yes;dataset_card:yes", "confidence": "high", "note": "Ungated with a full dataset card and built exclusively from uncopyrighted or openly licensed sources.", "sources": [ { "url": "https://huggingface.co/datasets/PleIAs/common_corpus", "shows": "open-license-only content, ungated access, dataset card, 2.27T tokens", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "HF reports 57,926 monthly downloads.", "sources": [ { "url": "https://huggingface.co/datasets/PleIAs/common_corpus", "shows": "57,926 monthly downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Ungated with a full dataset card and built exclusively from uncopyrighted or openly licensed sources." }, { "product": "CulturaX", "org": "University of Oregon NLP (UONLP)", "type": "dataset", "description": "CulturaX is a multilingual web corpus for LLM pretraining containing 6.3 trillion tokens across 167 languages, combined and cleaned from mC4 and OSCAR. It was created by the University of Oregon NLP group (UONLP) and applies a multi-stage pipeline of language identification, URL filtering, metric-based cleaning, document refinement, and MinHash fuzzy deduplication. Distributed as ~16TB of parquet.", "openness": { "score": 3, "class": "gated", "components": "license:follows mC4 + OSCAR-2301 terms;gated:auto(contact-info agreement);card:present", "confidence": "high", "note": "Dataset card is rich but downloads require agreeing to share contact information (auto gate).", "sources": [ { "url": "https://huggingface.co/datasets/uonlp/CulturaX", "shows": "Card with 6.3T tokens/167 languages, mC4+OSCAR license inheritance, gated access requiring contact-info agreement", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "Hugging Face reports 18,449 downloads in the last month.", "sources": [ { "url": "https://huggingface.co/api/datasets/uonlp/CulturaX", "shows": "downloads = 18449", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Dataset card is rich but downloads require agreeing to share contact information (auto gate)." }, { "product": "MADLAD-400", "org": "Allen Institute for AI", "type": "dataset", "description": "MADLAD-400 is a document-level multilingual web corpus derived from Common Crawl, covering 419 languages, released by the Allen Institute for AI (AI2). It ships in a noisy (minimally filtered) variant and a clean (quality-filtered) variant, totaling roughly 38TB, and was designed to improve coverage of low-resource languages. It underpins the MADLAD-400 machine-translation models.", "openness": { "score": 5, "class": "open", "components": "license:odc-by(API tag; card cites CC-BY-4.0);gated:false;card:present", "confidence": "high", "note": "Ungated with a permissive open-data license and a complete dataset card.", "sources": [ { "url": "https://huggingface.co/api/datasets/allenai/MADLAD-400", "shows": "gated = false, license = odc-by", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "Hugging Face reports 41,696 downloads in the last month.", "sources": [ { "url": "https://huggingface.co/api/datasets/allenai/MADLAD-400", "shows": "downloads = 41696", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Ungated with a permissive open-data license and a complete dataset card." }, { "product": "Aya Collection", "org": "Cohere", "type": "dataset", "description": "The Aya Collection is a large multilingual instruction-tuning dataset spanning 114+ languages, built from templated and translated subsets for tasks like classification, summarization, and translation. It was created by Cohere Labs (formerly Cohere For AI) as part of the Aya open-science initiative. The collection holds hundreds of millions of examples across 64 subsets.", "openness": { "score": 5, "class": "open", "components": "license:apache-2.0;gated:false;card:present", "confidence": "high", "note": "Ungated, Apache-2.0 licensed, with a complete dataset card.", "sources": [ { "url": "https://huggingface.co/datasets/CohereLabs/aya_collection", "shows": "Card describing 114+ languages, templated/translated subsets, Apache-2.0, ungated", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "Hugging Face reports 36,400 downloads in the last month.", "sources": [ { "url": "https://huggingface.co/api/datasets/CohereLabs/aya_collection", "shows": "downloads = 36400", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Ungated, Apache-2.0 licensed, with a complete dataset card." }, { "product": "OSCAR-2301", "org": "OSCAR Project", "type": "dataset", "description": "OSCAR-2301 is a multilingual web corpus extracted and filtered from the late-2022 Common Crawl snapshot, covering 151 languages and totaling ~3.57TB of raw unannotated text for LLM pretraining. It is published by the OSCAR project (oscar-corpus) and includes KenLM-based content quality metrics and deduplication hashes. The corpus content is from Common Crawl with no copyright held by the authors; metadata is CC0-1.0.", "openness": { "score": 3, "class": "gated", "components": "license:cc0-1.0(metadata);gated:manual(login + terms acceptance);card notes access temporarily suspended", "confidence": "high", "note": "Manually gated and the card states access is temporarily suspended pending clarification.", "sources": [ { "url": "https://huggingface.co/datasets/oscar-corpus/OSCAR-2301", "shows": "Card describing 151 languages/~3.57TB, CC0 metadata, gated access with a notice suspending access", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 1, "reach": "<10k", "signal_type": "usage_volume", "confidence": "high", "note": "Hugging Face reports 1,345 downloads in the last month (depressed by the access suspension).", "sources": [ { "url": "https://huggingface.co/api/datasets/oscar-corpus/OSCAR-2301", "shows": "downloads = 1345", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 1.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Manually gated and the card states access is temporarily suspended pending clarification." }, { "product": "Cosmopedia", "org": "Hugging Face", "type": "dataset", "description": "Cosmopedia is a large-scale synthetic dataset of over 31 million textbooks, blog posts, stories, and educational articles (~25 billion tokens) generated with Mixtral-8x7B-Instruct using seed samples from web data, Stanford courses, OpenStax, Khan Academy, and WikiHow. It was created by Hugging Face's Smol Models Research team and described as the largest open synthetic dataset of its kind at release. It is a reference corpus for synthetic-data pretraining recipes.", "openness": { "score": 5, "class": "open", "components": "license:apache-2.0;ungated;dataset card present;parquet downloadable", "confidence": "high", "note": "Permissive Apache-2.0 license, ungated, full dataset card and downloadable parquet files.", "sources": [ { "url": "https://huggingface.co/datasets/HuggingFaceTB/cosmopedia", "shows": "Apache-2.0 license, 31M rows, full dataset card", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "About 18.3k downloads in the last month per HF.", "sources": [ { "url": "https://huggingface.co/api/datasets/HuggingFaceTB/cosmopedia", "shows": "downloads=18336", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissive Apache-2.0 license, ungated, full dataset card and downloadable parquet files." }, { "product": "OpenThoughts-114k", "org": "Open Thoughts", "type": "dataset", "description": "OpenThoughts-114k is a synthetic reasoning dataset of roughly 114,000 examples spanning math, science, code, and puzzles, built by curating problem sets and generating reasoning traces with DeepSeek-R1 followed by correctness verification. It was created by the Open Thoughts team and documented in arXiv 2506.04178. It was used to train the OpenThinker-7B and OpenThinker-32B models.", "openness": { "score": 5, "class": "open", "components": "license:apache-2.0;ungated;dataset card present;GitHub repo + paper", "confidence": "high", "note": "Apache-2.0 licensed, ungated, with dataset card, code repo, and paper.", "sources": [ { "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts-114k", "shows": "Apache-2.0 license, dataset card, GitHub and arXiv links", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "About 83k downloads in the last month per HF.", "sources": [ { "url": "https://huggingface.co/api/datasets/open-thoughts/OpenThoughts-114k", "shows": "downloads=83018", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed, ungated, with dataset card, code repo, and paper." }, { "product": "WildChat-1M", "org": "Allen Institute for AI", "type": "dataset", "description": "WildChat-1M is a corpus of around 838k real user-chatbot conversations collected by the Allen Institute for AI (Ai2) by offering free access to GPT-3.5/GPT-4 in exchange for consenting to data collection. It spans 74 languages and includes metadata such as timestamps, hashed IPs, geographic region, model used, and moderation scores. It is a widely cited source of naturalistic instruction and chat data.", "openness": { "score": 5, "class": "open", "components": "license:odc-by;gated:false per HF API;dataset card present;parquet downloadable", "confidence": "medium", "note": "ODC-BY licensed and reported ungated by the HF API, though Ai2 datasets sometimes show an access notice.", "sources": [ { "url": "https://huggingface.co/api/datasets/allenai/WildChat-1M", "shows": "gated=false, license=odc-by", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "About 28.8k downloads in the last month per HF.", "sources": [ { "url": "https://huggingface.co/api/datasets/allenai/WildChat-1M", "shows": "downloads=28779", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. ODC-BY licensed and reported ungated by the HF API, though Ai2 datasets sometimes show an access notice." }, { "product": "Tulu 3 SFT Mixture", "org": "Allen Institute for AI", "type": "dataset", "description": "The Tulu 3 SFT Mixture is a supervised fine-tuning dataset of roughly 939k samples combining 18 data sources, created by the Allen Institute for AI (Ai2) for its Tulu 3 post-training recipe. It covers conversation, instruction-following, math, code, safety, and multilingual data across 70+ languages. The mixture is ODC-BY-1.0 licensed while incorporating subsets under varying licenses. It underpins the open Tulu 3 model family.", "openness": { "score": 5, "class": "open", "components": "license:odc-by(mixed subset licenses incl CC-BY-NC);ungated;dataset card present", "confidence": "high", "note": "ODC-BY licensed and ungated with a full dataset card, though some subsets carry non-commercial licenses.", "sources": [ { "url": "https://huggingface.co/datasets/allenai/tulu-3-sft-mixture", "shows": "ODC-BY-1.0 license, ~939k rows, dataset card", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "About 19k downloads in the last month per HF.", "sources": [ { "url": "https://huggingface.co/api/datasets/allenai/tulu-3-sft-mixture", "shows": "downloads=18959", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. ODC-BY licensed and ungated with a full dataset card, though some subsets carry non-commercial licenses." }, { "product": "OpenHermes 2.5", "org": "Teknium", "type": "dataset", "description": "OpenHermes 2.5 is a large synthetic instruction-following dataset of ~1M conversational examples compiled and generated primarily from GPT-4 outputs across math, code, roleplay, and general-knowledge tasks. Created by Teknium as an expansion of OpenHermes 1, it was used to train the popular OpenHermes 2.5 fine-tunes and is widely used as a base SFT mix for open instruction-tuned models.", "openness": { "score": 5, "class": "open", "components": "license:not-clearly-stated-on-card(commonly cited as permissive);card:present;ungated:yes;format:JSON/Parquet;rows:~1M", "confidence": "medium", "note": "Publicly downloadable and ungated with a full card, though an explicit license string is not stated on the card.", "sources": [ { "url": "https://huggingface.co/datasets/teknium/OpenHermes-2.5", "shows": "Public ungated dataset card, ~1M rows, synthetic/GPT-4 tags", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": null, "signal_type": "usage_volume", "confidence": "medium", "note": "About 16k downloads in the last month per the HF API, plus 861 likes (among the highest in the category), placing it alongside other level-2 synthetic SFT datasets like Tulu 3 and Cosmopedia.", "sources": [ { "url": "https://huggingface.co/api/datasets/teknium/OpenHermes-2.5", "shows": "HF API reports downloads=15954 (30d) and likes=861", "accessed": "2026-06-23" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Publicly downloadable and ungated with a full card, though an explicit license string is not stated on the card." }, { "product": "PersonaHub", "org": "Tencent AI Lab", "type": "dataset", "description": "PersonaHub is a persona-driven synthetic data resource from Tencent AI Lab, backed by the paper 'Scaling Synthetic Data Creation with 1,000,000,000 Personas' (arXiv 2406.20094). The repo releases a large set of distilled personas (200K preview plus ~370M elite personas) plus seed synthetic samples such as 50K math problems, 50K logical-reasoning problems, and 50K instructions. It is a method and dataset for generating diverse, scalable synthetic instruction and reasoning data.", "openness": { "score": 5, "class": "open", "components": "license:cc-by-nc-sa-4.0(non-commercial);card:present;ungated:yes;scope:~370M personas+seed samples", "confidence": "high", "note": "Publicly downloadable and ungated under CC-BY-NC-SA-4.0, which permits redistribution for non-commercial research.", "sources": [ { "url": "https://huggingface.co/datasets/proj-persona/PersonaHub", "shows": "CC-BY-NC-SA-4.0 license, ungated card, persona counts and paper reference", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10k", "signal_type": "usage_volume", "confidence": "high", "note": "The dataset page reports 6,572 downloads in the last month.", "sources": [ { "url": "https://huggingface.co/datasets/proj-persona/PersonaHub", "shows": "Downloads last month: 6,572", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 1.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Publicly downloadable and ungated under CC-BY-NC-SA-4.0, which permits redistribution for non-commercial research." }, { "product": "UltraFeedback", "org": "OpenBMB", "type": "dataset", "description": "UltraFeedback is a large-scale GPT-4-annotated preference dataset comprising ~64K prompts, ~256K responses, and ~380K fine-grained feedback annotations across instruction-following, truthfulness, honesty, and helpfulness. Prompts are drawn from six public datasets with responses from 17 different models. Built by OpenBMB, it is a foundational dataset for training reward and critic models and underpins many open RLHF/DPO pipelines.", "openness": { "score": 5, "class": "open", "components": "license:mit;card:present;ungated:yes;rows:63,967;size:940MB", "confidence": "high", "note": "Publicly downloadable and ungated under the permissive MIT license.", "sources": [ { "url": "https://huggingface.co/datasets/openbmb/UltraFeedback", "shows": "MIT license, ungated card, 63,967 rows / 940 MB", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10k", "signal_type": "usage_volume", "confidence": "high", "note": "The dataset page reports 4,657 downloads last month.", "sources": [ { "url": "https://huggingface.co/datasets/openbmb/UltraFeedback", "shows": "Downloads last month: 4,657", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 1.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Publicly downloadable and ungated under the permissive MIT license." }, { "product": "HH-RLHF (Anthropic Helpful/Harmless)", "org": "Anthropic", "type": "dataset", "description": "HH-RLHF is Anthropic's human preference dataset for RLHF, containing ~169K rows of chosen-vs-rejected response pairs on helpfulness and harmlessness, plus red-teaming dialogues. Released alongside Anthropic's papers on training helpful and harmless assistants, it is intended for preference modeling rather than supervised dialogue training. It is one of the most influential early open RLHF preference datasets.", "openness": { "score": 5, "class": "open", "components": "license:mit;card:present;ungated:yes;rows:169,352;size:94.7MB", "confidence": "high", "note": "Publicly downloadable and ungated under the permissive MIT license.", "sources": [ { "url": "https://huggingface.co/datasets/Anthropic/hh-rlhf", "shows": "MIT license, ungated card, 169,352 rows, 94.7 MB", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10k-100k", "signal_type": "usage_volume", "confidence": "high", "note": "The dataset page reports 30,331 downloads in the last month.", "sources": [ { "url": "https://huggingface.co/datasets/Anthropic/hh-rlhf", "shows": "Downloads last month: 30,331", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Publicly downloadable and ungated under the permissive MIT license." }, { "product": "SYNTH", "org": "PleIAs", "type": "dataset", "description": "SYNTH is a large-scale synthetic dataset from PleIAs (released with the AI Alliance) designed for training small reasoning models end-to-end. It contains tens of millions of synthetic samples amplifying content from tens of thousands of Wikipedia articles, Wikibooks, and internal documents, spanning exercise types like arithmetic, creative writing, RAG, and information extraction. Samples include reasoning traces and seed attribution, and roughly 20% covers non-English European languages.", "openness": { "score": 5, "class": "open", "components": "license:CC-BY-4.0(permissive);ungated;dataset card present", "confidence": "high", "note": "Permissively licensed, ungated dataset with a documented card.", "sources": [ { "url": "https://huggingface.co/datasets/PleIAs/SYNTH", "shows": "License CC-BY-4.0, not gated, dataset card present", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "niche", "signal_type": "usage_volume", "confidence": "high", "note": "~9.5k downloads in the last month on Hugging Face (near the 10k threshold; recently released).", "sources": [ { "url": "https://huggingface.co/datasets/PleIAs/SYNTH", "shows": "9,488 downloads last month", "accessed": "2026-06-22" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "a dataset is not 'capable'", "sources": [] }, "maturity": 2.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Permissively licensed, ungated dataset with a documented card." } ] }, "orchestration_agents": { "label": "Orchestration & agents", "arc": "Product / UX", "layer": "product_ux", "stage": { "num": 5, "name": "Mature Open Ecosystem" }, "gaps": [], "products": [ { "product": "Aider", "org": "Aider AI", "type": "software", "description": "Terminal-based AI pair programming tool that edits code directly in your local git repo. Differentiates through deep git integration (automatic commits with sensible messages), support for 20+ LLM providers, and a sophisticated diff-based editing approach that minimizes token usage. 45K GitHub stars; widely regarded as the benchmark for CLI-based coding agents and the tool that pioneered the 'agentic coding in terminal' pattern.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);no feature-gated core; model-agnostic (Claude/DeepSeek/OpenAI/local)", "confidence": "high", "note": "Verified Apache-2.0 OSI, full source public, no managed/enterprise tier gating the core. Repo owner Aider-AI/aider, 45.8k stars confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/Aider-AI/aider", "shows": "Apache-2.0 license, public source, 45.8k stars (verified)", "accessed": "2026-06-04" }, { "url": "https://pypi.org/project/aider-chat/", "shows": "aider-chat v0.86.2 published Feb 12 2026", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "GitHub 'used by' ~605 dependent projects and broad terminal-CLI install base; 45.8k stars corroborate but are not the basis. No published download/user count in the millions, so placed at 3 on dependent/usage-volume signal.", "sources": [ { "url": "https://github.com/Aider-AI/aider", "shows": "~605 dependent repos ('used by')", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified ~31.4% (architect mode, Claude Sonnet 4.5)", "confidence": "medium", "note": "Mid-tier on SWE-bench Verified; well below OpenHands (68.4%) and Cline (59.8%). NOTE: headline figure sourced from aggregator leaderboard (awesomeagents.ai), not a primary SWE-bench publication; kept at medium confidence.", "sources": [ { "url": "https://awesomeagents.ai/leaderboards/swe-bench-coding-agent-leaderboard/", "shows": "Aider (architect mode) SWE-bench Verified 31.4% (aggregator)", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Aider v0.86.2 (PyPI aider-chat, Feb 12 2026); GitHub repo Aider-AI/aider active, 45.8k stars. Terminal AI pair-programming agent that plans edits across a repo and applies git commits. Confirmed live June 2026." }, { "product": "Cline", "org": "Cline", "type": "software", "description": "Autonomous coding agent available as a VS Code extension, CLI tool, and SDK. Can create and edit files, run terminal commands, use the browser, and orchestrate multi-step development tasks with human-in-the-loop approval. Differentiates through its IDE-native experience and extensible SDK for embedding agent capabilities into other products. 62K GitHub stars with 1,078 commits in 90 days, one of the fastest-growing coding agent projects.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI, (c)2026 Cline Bot Inc.);source:public;no feature-gated core; model-agnostic (30+ providers + MCP)", "note": "Verified Apache-2.0 OSI, full source public; free extension with no proprietary core gating. License/version/stars confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/cline/cline", "shows": "Apache 2.0 license, public source, CLI v3.0.17 Jun 4 2026, 62.7k stars (verified)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "4,191,569 installs on the VS Code Marketplace (primary). One of the most-installed agentic coding extensions; firmly in the 1-10M reach band on install volume (not stars).", "sources": [ { "url": "https://marketplace.visualstudio.com/items?itemName=saoudrizwan.claude-dev", "shows": "4,191,569 VS Code Marketplace installs", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 59.8% (Autonomous Mode, Claude Sonnet 4.6)", "confidence": "medium", "note": "Second only to OpenHands (68.4%) among open coding agents on the April 2026 leaderboard; above SWE-agent and Aider. NOTE: figure from aggregator leaderboard (awesomeagents.ai), not primary; kept at medium confidence.", "sources": [ { "url": "https://awesomeagents.ai/leaderboards/swe-bench-coding-agent-leaderboard/", "shows": "Cline (Autonomous Mode) SWE-bench Verified 59.8% (aggregator)", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Cline (Cline Bot Inc.), CLI v3.0.17 (Jun 4 2026); GitHub cline/cline 62.7k stars. Autonomous coding agent in IDE/terminal, plans tasks, edits files, runs commands with human approval. Confirmed live June 2026." }, { "product": "SWE-agent", "org": "Princeton NLP", "type": "software", "description": "Research-driven autonomous coding agent from Princeton NLP that takes a GitHub issue and attempts to automatically fix it using LLMs. Also supports offensive cybersecurity and competitive coding challenges. Published at NeurIPS 2024 and served as an early benchmark-setter for the SWE-bench leaderboard. 19.3K stars with 92 contributors; development pace has slowed in 2026 (16 commits in 90 days) but the project remains influential as a reference architecture.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:MIT(OSI);source:public;no managed tier; model-agnostic", "note": "Verified MIT OSI, full source public; academic project with no proprietary tier. License/stars/release date confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/SWE-agent/SWE-agent/blob/main/LICENSE", "shows": "MIT License text", "accessed": "2026-06-04" }, { "url": "https://github.com/SWE-agent/SWE-agent", "shows": "19.4k stars, v1.1.0 May 22 2025, Princeton/Stanford (verified)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Primarily a research/benchmark agent; 19.4k stars in the research community. No published download/user volume; capped per stars-fallback rule. Adoption is research-circle, not production end-user scale.", "sources": [ { "url": "https://github.com/SWE-agent/SWE-agent", "shows": "19.4k stars, 2.1k forks (research adoption)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:SWE-bench Verified", "value": "SWE-bench Verified 43.2% (SWE-agent v1, Claude Sonnet 4.5)", "confidence": "medium", "note": "Originator of the SWE-bench agent scaffold; mid-tier on the 2026 leaderboard (43.2%), above Aider but below Cline/OpenHands. NOTE: figure from aggregator (awesomeagents.ai), not primary.", "sources": [ { "url": "https://awesomeagents.ai/leaderboards/swe-bench-coding-agent-leaderboard/", "shows": "SWE-agent v1 SWE-bench Verified 43.2% (aggregator)", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "SWE-agent (Princeton/Stanford), latest release v1.1.0 (May 22 2025); GitHub SWE-agent/SWE-agent 19.4k stars. Research agent that lets an LLM autonomously fix GitHub issues. Last tagged release >12mo old at scoring time (foundational research tool, repo still referenced)." }, { "product": "Goose", "org": "Block", "type": "software", "description": "Open-source extensible AI agent from Block (fka Square) that goes beyond code suggestions to install packages, execute commands, edit files, and run tests with any LLM. Differentiates through first-class MCP (Model Context Protocol) support and a Rust-based core for performance. 45.7K GitHub stars with 452 contributors and 918 commits in 90 days, one of the most actively developed open source agents, backed by a major tech company.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI);source:public;governance:Agentic AI Foundation/Linux Foundation (neutral foundation); model-agnostic; MCP-native; no feature-gated core", "note": "Verified Apache-2.0 OSI, now under Linux Foundation (AAIF) governance; full source public, no proprietary tier. Repo confirmed relocated to aaif-goose/goose (block/goose redirects).", "sources": [ { "url": "https://github.com/aaif-goose/goose", "shows": "Apache-2.0, v1.37.0 Jun 3 2026, repo moved from block/goose to AAIF/Linux Foundation, 46.4k stars (verified)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "46.4k stars and active multi-surface distribution (desktop/CLI/API), backed by Block and now Linux Foundation. No published download/MAU figure found, so capped at level 3 on stars-fallback; directional.", "sources": [ { "url": "https://github.com/aaif-goose/goose", "shows": "46.4k stars, active development", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomy: yes (autonomous task execution); tool-use: 70+ MCP extensions; multi-agent: limited; model-agnostic: 15+ providers; eval/reliability tooling: limited. No headline SWE-bench/GAIA leaderboard placement found.", "confidence": "medium", "note": "Broad general-purpose agent with strong MCP tool breadth and model-agnosticism, but no published SWE-bench/GAIA number; scored on feature matrix. Below benchmark-frontier coding agents.", "sources": [ { "url": "https://github.com/aaif-goose/goose", "shows": "70+ MCP extensions, 15+ LLM providers, autonomous agent features", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Goose, v1.37.0 (Jun 3 2026). VERIFIED: repo has moved from block/goose to aaif-goose/goose under the Agentic AI Foundation (AAIF) at the Linux Foundation; 46.4k stars. General-purpose on-machine AI agent (Rust) with 70+ MCP extensions, 15+ LLM providers. Confirmed live June 2026." }, { "product": "Codex CLI", "org": "OpenAI", "type": "software", "description": "Lightweight open source coding agent from OpenAI that runs in your terminal. Built in Rust for speed, it reads your codebase, proposes a plan, and executes multi-file changes with sandboxed command execution. Differentiates as OpenAI's official open source agent offering (Apache-2.0) with tight integration to OpenAI models. 84.6K GitHub stars with an extraordinary 2,893 commits in 90 days and 454 contributors, the most rapidly developed OSS agent as of May 2026.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI, the CLI agent);source:public(GitHub);note:agent is OSS but optimized for/defaults to OpenAI models (provider-leaning, not license-restricted)", "note": "Verified Apache-2.0 OSI with public Rust source (96.1% Rust); OpenAI's own docs call it open source. Backing models are proprietary, but the orchestration agent scored here is fully open. License/version/stars confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/openai/codex", "shows": "Apache-2.0 license, public Rust source, v0.137.0 Jun 4 2026, 88.6k stars (verified)", "accessed": "2026-06-04" }, { "url": "https://developers.openai.com/codex/cli", "shows": "OpenAI describes Codex CLI as open source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "high", "note": "OpenAI confirmed Codex surpassed 4M weekly developers (Apr 21 2026 announcement; 5M+ reported more recently). Firmly in the 1-10M+ reach band on disclosed developer count. Primary OpenAI announcement now cited as the basis (was previously sourced only to a distribution page).", "sources": [ { "url": "https://openai.com/index/scaling-codex-to-enterprises-worldwide/", "shows": "OpenAI: Codex surpassed 4M weekly developers (Apr 2026)", "accessed": "2026-06-04" }, { "url": "https://github.com/openai/codex", "shows": "88.6k stars, broad active install base (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Autonomy: full terminal coding agent (plan/edit/run/commit); tool-use: shell+MCP; multi-agent: yes (parallel sessions); model-agnostic: largely OpenAI-locked; reliability: sandboxed exec. No agent-attributed SWE-bench number on the April 2026 leaderboard.", "confidence": "medium", "note": "Capable, heavily-used coding agent backed by frontier OpenAI models, but no standalone agent SWE-bench placement published. Scored on feature matrix; high but not the open-agent benchmark frontier-definer (OpenHands 68.4%).", "sources": [ { "url": "https://developers.openai.com/codex/cli", "shows": "autonomous terminal agent: plan, edit, run commands, MCP, sandboxed exec", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "OpenAI Codex CLI v0.137.0 (Jun 4 2026); GitHub openai/codex 88.6k stars, built in Rust. Open-source terminal coding agent (the agent itself is OSS; it calls OpenAI models). Confirmed live June 2026." }, { "product": "AutoGPT", "org": "Significant Gravitas", "type": "software", "description": "One of the earliest and most well-known autonomous AI agent platforms, AutoGPT lets users define goals and the agent recursively breaks them into sub-tasks, executes actions, and learns from results. Has evolved from a viral demo into a more mature platform with a visual builder and marketplace. 184K GitHub stars (highest in the category) with 429 contributors, though much of the star count reflects the 2023 hype wave. Still actively developed with 668 commits in 90 days.", "openness": { "score": 3, "class": "source_available", "confidence": "high", "components": "license:DUAL, autogpt_platform/ under PolyForm Shield (non-OSI, source-available, anti-compete); everything else (Classic, Forge, agbenchmark) MIT(OSI);source:public", "note": "Verified dual license: the current flagship, the AutoGPT Platform, is PolyForm Shield (source-available, non-OSI), while legacy components stay MIT. Mixed-tier: scored source_available because the active product is non-OSI. Confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/Significant-Gravitas/AutoGPT", "shows": "dual license: PolyForm Shield for autogpt_platform, MIT for the rest (verified); 185k stars; autogpt-platform-beta-v0.6.62 May 28 2026", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "185k stars (one of the most-starred repos on GitHub) reflects huge historical interest, but stars cannot exceed level 3 and recent production-usage signal is unclear. No published platform MAU/download volume found; capped at 3, directional.", "sources": [ { "url": "https://github.com/Significant-Gravitas/AutoGPT", "shows": "185k stars, autogpt-platform-beta-v0.6.62 May 28 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Autonomy: yes (continuous agents); tool-use: block/connector library; multi-agent: workflow blocks; model-agnostic: yes; eval/reliability: agbenchmark + monitoring. No headline SWE-bench/GAIA placement.", "confidence": "medium", "note": "Pioneering but the autonomous loop has been surpassed; the low-code platform is its current value. No competitive agent-benchmark number published; scored low-mid on feature matrix.", "sources": [ { "url": "https://github.com/Significant-Gravitas/AutoGPT", "shows": "Agent Builder, workflow blocks, marketplace, monitoring features", "accessed": "2026-06-04" } ] }, "maturity": 2.3, "mature": false, "version_note": "AutoGPT (Significant-Gravitas/AutoGPT), latest autogpt-platform-beta-v0.6.62 (May 28 2026); 185k GitHub stars. The original autonomous-agent project; now centered on the AutoGPT Platform (low-code agent builder). Confirmed live June 2026." }, { "product": "GPT Researcher", "org": "Assaf Elovic", "type": "software", "description": "Autonomous research agent that conducts deep web research on any topic by planning research queries, browsing multiple sources, synthesizing findings, and generating comprehensive reports. Differentiates as a purpose-built research agent (not coding-focused) that orchestrates search, scraping, and synthesis into a coherent workflow. 27.2K GitHub stars with 210 contributors; an early mover in the 'deep research' agent pattern that inspired similar features in ChatGPT and Gemini.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core; model-agnostic; Docker + frontend included", "note": "Verified Apache-2.0 OSI, full source public, no proprietary tier. License/version/stars confirmed via primary GitHub source.", "sources": [ { "url": "https://github.com/assafelovic/gpt-researcher", "shows": "Apache-2.0, public source, v3.5.0 May 28 2026, 27.5k stars (verified)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "~105,483 PyPI downloads last month (gpt-researcher); 'used by' 261 dependent repos; 27.5k stars corroborate. Places it in the 100K-1M reach band on monthly download volume.", "sources": [ { "url": "https://pypistats.org/packages/gpt-researcher", "shows": "105,483 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/assafelovic/gpt-researcher", "shows": "261 dependent repos, 27.5k stars", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomy: deep-research loop (tree exploration); tool-use: web + local retrieval; multi-agent: yes (research sub-agents); model-agnostic: yes; reliability: cited multi-source reports. No GAIA/agent-benchmark placement published.", "confidence": "medium", "note": "Solid, focused deep-research agent; no headline GAIA/benchmark number, scored on feature matrix. Narrower than general coding/multi-tool agents; mid-tier.", "sources": [ { "url": "https://github.com/assafelovic/gpt-researcher", "shows": "deep research, tree exploration, multi-source cited reports, local+web", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "GPT Researcher (assafelovic/gpt-researcher), v3.5.0 (May 28 2026); 27.5k stars, Apache-2.0. Autonomous deep-research agent that plans sub-queries, searches web/local sources, and writes cited reports. Confirmed live June 2026." }, { "product": "Suna", "org": "Kortix AI", "type": "software", "description": "Open-source generalist AI agent positioned as an 'autonomous company operating system'; can browse the web, write and execute code, manage files, interact with APIs, and complete complex multi-step business tasks. Differentiates through an ambitious scope beyond coding (document creation, data analysis, web research) with a polished web UI. 19.8K GitHub stars with explosive development velocity (2,522 commits in 90 days), suggesting heavy investment in rapid iteration.", "openness": { "score": 3, "class": "source_available", "confidence": "high", "components": "license:Elastic License 2.0(non-OSI, source-available, restricts hosting as a managed service);source:public;managed cloud + self-host tiers", "note": "Verified Elastic License 2.0 in the LICENSE file, source-available, NOT OSI; restricts providing the software as a hosted/managed service. Marketed as 'open source' but carries managed-service restrictions; flagged open_washed.", "sources": [ { "url": "https://github.com/kortix-ai/suna/blob/main/LICENSE", "shows": "Elastic License 2.0 text (verified, non-OSI)", "accessed": "2026-06-04" }, { "url": "https://github.com/kortix-ai/suna", "shows": "v0.9.20 Jun 4 2026, 19.8k stars, cloud + self-host", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "19.8k stars and a managed cloud offering, but no published download/user-count figure; capped on stars fallback. Placed at 2 given the relatively young, smaller community vs the leading agents; directional.", "sources": [ { "url": "https://github.com/kortix-ai/suna", "shows": "19.8k stars, 44 releases", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomy: yes (specialist agents in isolated sandboxes, change-request workflow); tool-use: 3,000+ connectors; multi-agent: yes (parallel branches); model-agnostic: yes; reliability: review-before-merge gating. No public agent-benchmark placement.", "confidence": "medium", "note": "Feature-rich generalist agent platform with broad integrations and a novel review/merge workflow, but no published benchmark; scored on feature matrix.", "sources": [ { "url": "https://github.com/kortix-ai/suna", "shows": "isolated sandboxes, 3,000+ connectors, parallel agents, change requests", "accessed": "2026-06-04" } ] }, "maturity": 2.7, "mature": false, "version_note": "Suna / Kortix (kortix-ai/suna), v0.9.20 (Jun 4 2026); 19.8k stars. Open generalist AI agent / 'AI command center' with isolated-sandbox specialist agents and 3,000+ connectors. Confirmed live June 2026." }, { "product": "Claude Code", "org": "Anthropic", "type": "software", "description": "Agentic coding tool from Anthropic that lives in the terminal, understands your entire codebase, and autonomously executes multi-step development tasks including code generation, refactoring, git workflows, and testing. Differentiates through deep integration with Claude models, a rich skill/hook system for customization, and native MCP tool support. 125.7K GitHub stars; source-available but not open source (no license). Reportedly the fastest-growing developer tool in Anthropic's portfolio.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "license:Proprietary(closed-source CLI/app, requires Claude subscription or Console account);model:Claude(proprietary, API-only);note:Agent SDK published but the product is closed SaaS", "note": "Closed/proprietary product gated behind an Anthropic subscription; the coding agent itself is not open source.", "sources": [ { "url": "https://code.claude.com/docs/en/overview", "shows": "requires Claude subscription or Anthropic Console account; proprietary surfaces", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Reuters (Feb 2026): Claude Code exceeded ~$2.5B run-rate revenue, with business subscriptions quadrupled since the start of 2026 and enterprise now >half of Claude Code revenue. No clean user-count, so reported_traction; revenue scale implies well into the 1-10M+ developer band.", "sources": [ { "url": "https://code.claude.com/docs/en/overview", "shows": "broad multi-surface paid product (terminal/IDE/desktop/web/CI)", "accessed": "2026-06-04" }, { "url": "https://www.saastr.com/anthropic-just-hit-14-billion-in-arr-up-from-1-billion-just-14-months-ago/", "shows": "Anthropic $14B ARR Feb 2026; Claude Code a major component (corroborating Reuters)", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": "Backed by frontier Claude Opus models leading SWE-bench Verified (Claude Opus 4.8 ~88.6%, 4.7 ~87.6%); Claude Code is the reference agent harness for these scores.", "confidence": "high", "note": "Frontier-defining coding agent, pairs the leading SWE-bench Verified models with a mature multi-surface agent harness (sub-agents, hooks, MCP, scheduling). One of the 1-2 capability frontier-definers in this category. Headline SWE-bench Verified 88.6% triangulated to Anthropic primary source.", "sources": [ { "url": "https://llm-stats.com/benchmarks/swe-bench-verified", "shows": "Claude Opus 4.8 88.6% / 4.7 87.6% lead SWE-bench Verified", "accessed": "2026-06-04" }, { "url": "https://www.anthropic.com/news/claude-opus-4-8", "shows": "Anthropic primary announcement of Claude Opus 4.8 (headline coding model behind Claude Code); SWE-bench Verified 88.6%", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Claude Code (Anthropic), agentic coding tool across terminal/IDE/desktop/web; GA May 2025, actively shipping June 2026. Proprietary/SaaS. Requires a Claude subscription or Anthropic Console account. Confirmed live June 2026." }, { "product": "Cursor", "org": "Anysphere", "type": "software", "description": "AI-first code editor (VS Code fork) with an integrated agent mode that can autonomously make multi-file changes, run terminal commands, and iterate on code based on errors. Differentiates through seamless IDE integration where the agent works alongside manual editing, plus frontier model access and a background agent feature for asynchronous task completion. Reportedly 100K+ paying users at ~$20/month, making it the most commercially successful AI coding tool outside GitHub Copilot. Raised $900M+ at a $9B+ valuation.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "license:Proprietary(closed SaaS IDE, no public source);model:multi-provider (OpenAI/Anthropic/Gemini/xAI) but the product is proprietary", "note": "Fully proprietary commercial product; no open source license or public source.", "sources": [ { "url": "https://www.cursor.com/", "shows": "proprietary commercial IDE, no OSS licensing, enterprise/Fortune-500 positioning", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "active_users", "confidence": "high", "note": "1M+ daily active users (reported 2025, sustained Dec 2025) and ~$2B ARR by Feb 2026; ~64% of Fortune 500 active. Confirmed via press coverage of company-reported figures. 1M+ DAU puts it in the 1-10M band (not yet a confirmed >10M mass-market count).", "sources": [ { "url": "https://www.cursor.com/", "shows": "'trusted by over half of the Fortune 500', enterprise positioning", "accessed": "2026-06-04" }, { "url": "https://research.contrary.com/company/cursor", "shows": "Cursor 1M+ DAU, ~$2B ARR Feb 2026 (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:SWE-bench Verified", "value": "Cursor Background Agent SWE-bench Verified 65.7% (April 2026 coding-agent leaderboard)", "confidence": "medium", "note": "High-tier autonomous coding agent (Background Agent 65.7%), just below OpenHands (68.4%); mature Composer/agent UX. NOTE: figure from aggregator leaderboard (awesomeagents.ai), not primary; kept at medium confidence.", "sources": [ { "url": "https://awesomeagents.ai/leaderboards/swe-bench-coding-agent-leaderboard/", "shows": "Cursor Background Agent SWE-bench Verified 65.7% (aggregator)", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Cursor (Anysphere Inc.), AI coding IDE with agent mode (Composer 2.5) + Background Agent + CLI; proprietary SaaS. ~$2B ARR (Feb 2026), 1M+ DAU. Confirmed live June 2026. Scored here for its autonomous agent mode (chat-editor surface would be ui_api)." }, { "product": "Devin", "org": "Cognition", "type": "software", "description": "The first product marketed as a fully autonomous AI software engineer. Operates in a cloud-based sandboxed environment with its own shell, browser, and editor, completing entire development tasks from specification to pull request. Differentiates as the highest-profile 'autonomous developer' product with full environment access rather than just file editing. Raised $175M at a $2B valuation; commercially available via API and web interface, though real-world performance reviews are mixed compared to initial demo claims.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(SaaS-only);self-host:no", "confidence": "high", "note": "Proprietary subscription SaaS; no source or self-hosting, matching the recipe's 'closed' example (Devin).", "sources": [ { "url": "https://docs.devin.ai", "shows": "proprietary SaaS coding agent, app.devin.ai signup, Individual/Teams plans", "accessed": "2026-06-04" }, { "url": "https://devin.ai/pricing", "shows": "paid tiers Free/$20 Pro/$200 Max/Teams/Enterprise, proprietary SaaS", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Widely adopted commercial coding agent with paid Individual/Teams/Enterprise tiers, but Cognition discloses no hard user count. Placed at 3 on reported commercial traction; no verified usage_volume figure.", "sources": [ { "url": "https://devin.ai/pricing", "shows": "multiple commercial tiers incl. Teams/Enterprise indicating broad paid adoption", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "benchmark:SWE-bench Verified", "value": "Cognition's own technical report: 13.86% (2024, 570-problem subset); Devin 2.0 ~45.8% SWE-bench Verified (unassisted single-agent, vendor-reported and corroborated)", "confidence": "medium", "note": "Devin 2.0 at 45.8% SWE-bench Verified is independently corroborated (single-agent, no human-in-loop), mid-tier, well below the OpenHands SWE-bench-Verified SOTA anchor. Scored 3.", "sources": [ { "url": "https://cognition.ai/blog/swe-bench-technical-report", "shows": "Cognition-reported 13.86% SWE-bench, unassisted end-to-end methodology", "accessed": "2026-06-04" }, { "url": "https://www.tembo.io/blog/devin-alternatives-2025", "shows": "Devin 2.0 scores 45.8% on SWE-Bench Verified, standard unassisted single-agent eval (corroborating)", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Devin by Cognition AI, autonomous SaaS software-engineering agent (web IDE + CLI). Verified live June 2026 at docs.devin.ai and devin.ai/pricing; Individual/Teams/Enterprise plans. Devin 2.0 generation." }, { "product": "Antigravity", "org": "Google", "type": "software", "description": "Google's agent-first developer platform, launched as a standalone IDE in November 2025 (public preview competing with Cursor), then expanded at Google I/O in May 2026 into Antigravity 2.0 with a Go-based CLI, SDK, and multi-agent orchestration. The CLI replaces Gemini CLI (which sunsets for free tiers in June 2026). Powered by Gemini 3.5 Flash with deep Google Cloud / Firebase / Android integration.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(Google product, free public preview);self-host:no", "confidence": "high", "note": "Proprietary Google agentic dev platform; free in preview but not open source and locked to Google's stack.", "sources": [ { "url": "https://antigravity.google", "shows": "Google Antigravity product page (agentic development platform)", "accessed": "2026-06-04" }, { "url": "https://developers.googleblog.com/all-the-news-from-the-google-io-2026-developer-keynote/", "shows": "Google's own I/O 2026 keynote announcement of Antigravity 2.0 desktop app/CLI/SDK as a proprietary agentic dev platform", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Free public preview launched at I/O 2026 (~weeks old at scoring); Google discloses no user count for Antigravity specifically. No honest adoption signal yet.", "sources": [ { "url": "https://blog.google/innovation-and-ai/technology/developers-tools/google-io-2026-developer-highlights/", "shows": "Antigravity available in public preview, no usage figures disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Multi-agent orchestration (plan/code/test-in-browser/deploy), Agent Manager, Gemini 3.5 Flash backbone, deep Google-surface integration, desktop app + CLI + SDK; no first-party SWE-bench number published for the platform itself", "confidence": "medium", "note": "Strong agentic feature breadth (autonomous multi-agent plan-code-test-deploy) on a frontier Gemini backbone; no published SWE-bench for the platform so scored on feature matrix. Below the OpenHands benchmark anchor absent a verified score.", "sources": [ { "url": "https://developers.googleblog.com/build-with-google-antigravity-our-new-agentic-development-platform/", "shows": "multi-agent plan/code/test/deploy autonomy and Gemini integration features", "accessed": "2026-06-04" }, { "url": "https://blog.google/innovation-and-ai/technology/developers-tools/google-io-2026-developer-highlights/", "shows": "Antigravity 2.0 + Gemini 3.5 Flash agentic capabilities", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Google Antigravity 2.0, agent-first AI development platform (desktop app + CLI + SDK) powered by Gemini 3.5 Flash; launched at Google I/O 2026 (May 2026), public preview free for individuals. Verified live June 2026." }, { "product": "Windsurf", "org": "Codeium", "type": "software", "description": "AI-native IDE (VS Code fork) with 'Cascade', an agentic flow engine that maintains context across multi-step coding tasks, automatically suggests and executes terminal commands, file edits, and debugging steps. Differentiates through 'Flows' that blend co-pilot suggestions with autonomous agent actions, plus strong context awareness across large codebases. Reportedly millions of users across free and paid tiers; Codeium raised $150M at a $1.25B valuation before being acquired by Windsurf (the brand name superseded Codeium).", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(SaaS, Cognition);self-host:no", "confidence": "high", "note": "Proprietary agentic IDE; now part of Cognition's Devin Desktop. Closed source, subscription SaaS.", "sources": [ { "url": "https://devin.ai/desktop", "shows": "Windsurf rebranded as Devin Desktop, proprietary agentic IDE (Cognition)", "accessed": "2026-06-04" }, { "url": "https://cognition.ai/blog/windsurf", "shows": "Cognition's own acquisition announcement of Windsurf (IP, product, trademark, brand)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Devin Desktop (Windsurf) page reports '1M+ Users' and '4000+ Enterprise Customers'. Vendor-reported headline count places it in the 1-10M band. Cognition-reported Windsurf ARR ~$82M corroborates commercial scale.", "sources": [ { "url": "https://devin.ai/desktop", "shows": "vendor-stated '1M+ Users' and '4000+ Enterprise Customers'", "accessed": "2026-06-04" }, { "url": "https://cognition.ai/blog/windsurf", "shows": "Windsurf reached $82M ARR, enterprise ARR doubling QoQ (corroborating scale)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Command Center (plan/delegate/review), multi-agent + multi-model via Agent Client Protocol (ACP, run Codex/Claude side-by-side), Devin Local (Rust), Spaces shared context, in-house SWE-1.6 model; no public SWE-bench number for the IDE", "confidence": "medium", "note": "Mature agentic-IDE feature set (multi-agent orchestration, model-agnostic via ACP) but no first-party SWE-bench score published; scored on feature matrix, a notch below the benchmark anchor.", "sources": [ { "url": "https://devin.ai/desktop", "shows": "Command Center, multi-agent ACP support (Codex/Claude), Spaces, SWE-1.6 features", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Windsurf (formerly Codeium) acquired by Cognition (def. agreement July 2025) and rebranded 'Devin Desktop' on June 2 2026; windsurf.com redirects to devin.ai/desktop. Agent manager wrapped in a full IDE; Cascade legacy, successor Devin Local (Rust). Verified June 2026." }, { "product": "Amazon Q Developer", "org": "Amazon Web Services", "type": "software", "description": "AWS's AI-powered development assistant with an autonomous agent mode that can implement features, fix bugs, write tests, and perform code transformations across entire codebases. Differentiates through deep AWS service integration (can deploy, debug Lambda functions, query CloudWatch) and enterprise security features. Available in IDE extensions, CLI, and AWS Console; included in AWS enterprise agreements, giving it a large potential installed base.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(AWS managed service);self-host:no", "confidence": "high", "note": "Proprietary AWS managed service; free and Pro tiers but no source or self-hosting.", "sources": [ { "url": "https://aws.amazon.com/q/developer/", "shows": "AWS proprietary agentic coding assistant, Free/Pro tiers, enterprise security controls", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Distributed across AWS's large developer base via IDEs/CLI/Console with free + Pro tiers and enterprise case studies, but AWS discloses no hard Q Developer user count. Placed at 3 on reported traction.", "sources": [ { "url": "https://aws.amazon.com/q/developer/", "shows": "enterprise case studies and Free/Pro distribution, no disclosed user count", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Agentic autonomy (implement/document/test/review/refactor), AWS-stated top SWE-Bench Leaderboard/Lite scores; specific 2026 SWE-bench Verified number not independently on a public leaderboard", "confidence": "medium", "note": "AWS claims top SWE-Bench Leaderboard placement but no clean current SWE-bench Verified figure is independently verifiable, so the benchmark claim is downgraded to vendor-stated and capability scored on feature matrix at 4, strong agentic breadth, below a verified-benchmark ceiling.", "sources": [ { "url": "https://aws.amazon.com/q/developer/", "shows": "agentic task autonomy + AWS-stated SWE-Bench leaderboard claim (vendor-stated, not independently confirmed)", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Amazon Q Developer, AWS's agentic GenAI software-development assistant across IDEs/CLI/Console/chat. Free tier (50 agentic chats/mo) + Pro tier. Verified live June 2026 at aws.amazon.com/q/developer." }, { "product": "GitHub Copilot", "org": "GitHub (Microsoft)", "type": "software", "description": "GitHub Copilot Agent Mode, assigns autonomous background coding tasks (plan/explore/execute), supports first-party Copilot agent plus Claude (Anthropic) and Codex (OpenAI). Verified live June 2026 at github.com/features/copilot.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(GitHub/Microsoft SaaS);self-host:no", "confidence": "high", "note": "Proprietary GitHub/Microsoft product; agent mode is a closed SaaS feature, model-agnostic on the backend but not open source.", "sources": [ { "url": "https://github.com/features/copilot", "shows": "GitHub Copilot proprietary product, agent mode autonomous task execution, third-party model support", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "GitHub Copilot surpassed 20M+ all-time users (Microsoft July 2025 earnings): 'world's most widely adopted AI developer tool.' Agent mode is the autonomous surface of that base. >10M band. Note: 20M is all-time, not active; agent-mode-specific count not disclosed, but base scale firmly supports level 5.", "sources": [ { "url": "https://github.com/features/copilot", "shows": "'world's most widely adopted AI developer tool', millions of users and tens of thousands of business customers", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2025/07/30/github-copilot-crosses-20-million-all-time-users/", "shows": "20M+ all-time users confirmed on Microsoft earnings call (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Autonomous background agents (plan/explore/execute), multi-model orchestration (Copilot + Claude + Codex), deep GitHub/PR/CI integration; no first-party SWE-bench number on the agent-mode page", "confidence": "medium", "note": "Broad agentic feature set with frontier-model backends and best-in-class repo integration; no published SWE-bench for Copilot agent mode itself, so feature_matrix at 4, strong but below a verified-benchmark ceiling.", "sources": [ { "url": "https://github.com/features/copilot", "shows": "agent-mode autonomy + multi-model (Claude/Codex) orchestration features", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "GitHub Copilot Agent Mode, assigns autonomous background coding tasks (plan/explore/execute), supports first-party Copilot agent plus Claude (Anthropic) and Codex (OpenAI). Verified live June 2026 at github.com/features/copilot." }, { "product": "Replit Agent", "org": "Replit", "type": "software", "description": "Cloud-based autonomous development agent embedded in Replit's online IDE that can build entire applications from natural language descriptions, scaffolding projects, installing packages, writing code, configuring databases, and deploying to production. Differentiates through zero-setup cloud execution (no local environment needed) and end-to-end deployment capability, making it the most accessible agent for non-developers. Replit has 30M+ registered users; raised $200M+ total funding.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(Replit cloud SaaS);self-host:no", "confidence": "high", "note": "Proprietary cloud agent embedded in Replit's hosted IDE; no source or self-hosting.", "sources": [ { "url": "https://replit.com/agent", "shows": "Replit Agent 4 proprietary AI app-building product within Replit's cloud platform", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Replit reported 50M+ platform users with agent-driven consumption as primary growth driver (ARR ~$250M). User count is platform-wide, not agent-specific, so placed conservatively at 4 on reported traction rather than treating the 50M as an agent active-user figure.", "sources": [ { "url": "https://replit.com/agent", "shows": "Replit Agent as flagship product driving platform usage", "accessed": "2026-06-04" }, { "url": "https://sacra.com/c/replit/", "shows": "Replit 50M+ users, ~$250M ARR driven by AI agents (corroborating traction)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomous parallel multi-task app building, design+build concurrency, task-based team workflows, end-to-end deploy on Replit infra; no published SWE-bench/GAIA number", "confidence": "low", "note": "Capable autonomous app-builder but oriented to full-app generation/deploy rather than benchmark-graded SWE tasks; no first-party benchmark, scored 3 on feature matrix.", "sources": [ { "url": "https://replit.com/agent", "shows": "parallel multi-task autonomy, design+functionality build, deploy features", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Replit Agent (Agent 4 generation), autonomous AI app-building agent in the Replit cloud IDE, parallel multi-task build, design + functionality in parallel. Verified live June 2026 at replit.com/agent; proprietary, consumption-priced on top of Replit subscriptions." }, { "product": "v0", "org": "Vercel", "type": "software", "description": "AI-powered generative UI and full-stack development agent from Vercel that builds web applications from natural language prompts. Can autonomously generate React/Next.js components, assemble full pages, manage state, connect to APIs, and iterate based on visual feedback. Differentiates through frontend-first design quality and tight Next.js/Vercel deployment integration. Evolved from a UI generation tool into a full autonomous development agent in 2025.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(Vercel SaaS);self-host:no", "confidence": "high", "note": "Proprietary Vercel product; closed source, subscription SaaS tied to Vercel deploy infra.", "sources": [ { "url": "https://v0.app/", "shows": "proprietary Vercel agentic app builder, 'agentic by default', Opus 4.8 backbone", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Vercel reports multi-million (4M+) cumulative v0 users since GA. Falls in the 1-10M band; reported_traction since the figure is vendor cumulative-user rather than a hard active-user metric.", "sources": [ { "url": "https://v0.app/", "shows": "v0 product with large template marketplace / active usage", "accessed": "2026-06-04" }, { "url": "https://vercel.com/blog/introducing-the-new-v0", "shows": "Vercel-reported multi-million v0 user base (corroborating 4M+)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomous plan/code/DB management, NL-to-app generation, visual design mode, GitHub integration, one-click Vercel deploy; underlying model not publicly named; no published SWE-bench/GAIA", "confidence": "low", "note": "Strong UI/full-stack generation autonomy scoped to web-app building; no benchmark and the backbone model is not publicly disclosed (record's 'Opus 4.8' unverified), scored 3 on feature matrix.", "sources": [ { "url": "https://v0.app/", "shows": "agentic plan/code/DB autonomy, design mode, deploy features", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "v0 by Vercel (v0.app), agentic full-stack web app builder ('agentic by default', autonomous plan/create-tasks/connect-DB as it builds); web + iOS, subscription (v0 Pro). Verified live June 2026 at v0.app. (Note: prior record's 'powered by Claude Opus 4.8' could NOT be confirmed on the product page. No model named publicly; claim softened.)" }, { "product": "Lovable", "org": "Lovable", "type": "software", "description": "Lovable (formerly GPT Engineer), proprietary agentic AI app/website builder ('chat to build', autonomous prototype-to-deploy). Verified live June 2026 at lovable.dev; includes a Lovable Agent and MCP server.", "openness": { "score": 1, "class": "closed", "components": "source:closed;license:proprietary(SaaS);self-host:no", "confidence": "high", "note": "Proprietary SaaS app builder; closed source. (GPT Engineer, its open source predecessor, is a separate legacy repo; the scored product is the proprietary Lovable platform.)", "sources": [ { "url": "https://lovable.dev", "shows": "proprietary SaaS AI app builder, chat-to-build, one-click deploy", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Lovable reports millions of projects built and 100K+ new projects/day; ~8M users reported late 2025; $400M ARR (Feb/Mar 2026, 146 employees). Usage volume in the 1-10M band.", "sources": [ { "url": "https://lovable.dev", "shows": "millions of projects built, thousands created daily", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/03/11/lovable-says-it-added-100m-in-revenue-last-month-alone-with-just-146-employees/", "shows": "$400M ARR, ~8M users (corroborating scale)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomous chat-to-app generation, screenshot/doc input, real-time prototype build, one-click deploy, MCP server, integrations; no published SWE-bench/GAIA", "confidence": "low", "note": "Capable autonomous app builder targeted at non-developers; full-app generation rather than benchmark SWE work, no first-party benchmark; scored 3 on feature matrix.", "sources": [ { "url": "https://lovable.dev", "shows": "autonomous build-from-prompt, MCP server, integrations features", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Lovable (formerly GPT Engineer), proprietary agentic AI app/website builder ('chat to build', autonomous prototype-to-deploy). Verified live June 2026 at lovable.dev; includes a Lovable Agent and MCP server." }, { "product": "Vellum", "org": "Vellum", "type": "software", "description": "Consumer-facing 'Personal Intelligence' agent, a multi-platform (web, macOS, iOS, CLI) assistant with episodic, semantic, and procedural memory that takes actions on the user's behalf. Pivoted in 2026 from an LLMops/prompt-engineering platform (prompts, workflows, evals) to a consumer agent product after a $20M Series A in July 2025; legacy LLMops surface still available but no longer the headline.", "openness": { "score": 4, "class": "open_source", "components": "license:MIT(OSI, Copyright 2025 Vellum AI);source:public+full-agent(github.com/vellum-ai/vellum-assistant: gateway+clients+skill-framework+memory);self-host:yes(by design);managed-cloud:optional-add-on(memory/security)", "confidence": "high", "note": "CORRECTED from record's 'closed/score 1'. Primary check disproves it: the agent's complete source is public and MIT-licensed (LICENSE file: 'Copyright (c) 2025 Vellum AI', full app incl. backend gateway, Mac/Slack/Telegram clients, sandboxed skill framework, memory system), self-hostable by design, not a client/SDK stub. Vendor's own marketing comparison table paradoxically labels Vellum 'Proprietary' (vs MIT for Hermes/OpenClaw) because it sells a managed cloud tier; but the core agent is genuinely open source. Class=open_source; score 4 rather than 5 because the differentiated managed memory/security layer is a hosted add-on and the vendor markets the primary offering as proprietary cloud.", "sources": [ { "url": "https://www.vellum.ai", "shows": "personal AI assistant 'AI layer for your life'; comparison table labels Vellum 'Proprietary' vs MIT Hermes/OpenClaw, but links to its own GitHub; cloud or self-hosted", "accessed": "2026-06-04" }, { "url": "https://www.vellum.ai/blog/best-hermes-agent-alternatives", "shows": "Vellum positioning as a personal assistant vs Hermes/OpenClaw (confirms entity)", "accessed": "2026-06-04" }, { "url": "https://github.com/vellum-ai/vellum-assistant", "shows": "PRIMARY: full open source agent source (gateway+clients+skill framework+memory), MIT, ~577 stars, 120+ releases (latest v0.8.7 Jun 2026)", "accessed": "2026-06-04" }, { "url": "https://github.com/vellum-ai/vellum-assistant/blob/main/LICENSE", "shows": "PRIMARY: MIT License text, 'Copyright (c) 2025 Vellum AI'", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<1K", "signal_type": "stars_fallback", "confidence": "low", "note": "No download/user/usage metrics disclosed (it's a self-hosted app, not a registry package). Only signal available is GitHub stars (~577 on vellum-ai/vellum-assistant), used here strictly as a last-resort proxy per the adoption!=stars rule; small/new entrant. Previously null; set to a conservative level 1 now that a real (if weak) signal exists.", "sources": [ { "url": "https://github.com/vellum-ai/vellum-assistant", "shows": "~577 stars (last-resort star proxy; no other adoption signal disclosed)", "accessed": "2026-06-04" }, { "url": "https://www.vellum.ai", "shows": "landing page with no disclosed user/usage numbers", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Autonomous always-on personal agent: 8-type memory, real-world action via managed OAuth connections, multi-channel (Mac/Web/iOS/Slack/Telegram/phone), out-of-process credential isolation, ambient/proactive engine; no benchmark", "confidence": "low", "note": "Solid personal-agent feature breadth (8-type memory + multi-channel autonomous action + credential isolation + proactive engine), now confirmed against the open MIT source. No benchmark and no frontier differentiator; scored 3 on feature matrix.", "sources": [ { "url": "https://www.vellum.ai", "shows": "8-type memory, multi-channel autonomous action, credential isolation, ambient engine features", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "Vellum (vellum.ai), a PERSONAL AI ASSISTANT/agent ('the AI layer for your life', 8-type structured memory, multi-channel autonomous action across Mac/Web/iOS/Slack/Telegram, proactive hourly check-ins, out-of-process credential isolation). The agent's full source ships MIT-licensed at github.com/vellum-ai/vellum-assistant (backend gateway, multi-platform clients, skill framework); a managed cloud (memory/security) is offered on top. Verified live June 2026; positions itself vs Hermes Agent / OpenClaw." }, { "product": "smolagents", "org": "Hugging Face", "type": "software", "description": "Minimal code-first agent framework from Hugging Face (~1,000 lines of core code) whose headline CodeAgent writes actions as executable Python rather than JSON tool calls, which research shows uses ~30% fewer steps than traditional tool-calling agents on hard benchmarks. Differentiates through model-agnostic design (Hugging Face Inference, OpenAI, Anthropic, local transformers/llama.cpp) and first-class MCP support, making it the path of least resistance for HF users building agents. 27.5K GitHub stars, 203 contributors, Apache-2.0; featured in Hugging Face's official agents course and referenced in Anthropic and Cohere agent-building guides.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(github.com/huggingface/smolagents);no-feature-gated-core;model-agnostic", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), entire library public, no proprietary/managed core, top openness, above open-core framework anchors (CrewAI/LangChain).", "sources": [ { "url": "https://github.com/huggingface/smolagents", "shows": "Apache-2.0 license, full public source, ~1k-LOC code-agent framework", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "694,642 PyPI downloads in the last month (pypistats, confirmed). Monthly download volume in the 100K-1M band, meaningful but below heavyweight framework anchors (LangChain, CrewAI).", "sources": [ { "url": "https://pypistats.org/packages/smolagents", "shows": "694,642 downloads in the last month (verified)", "accessed": "2026-06-04" }, { "url": "https://github.com/huggingface/smolagents", "shows": "GitHub stars (corroborating, not the basis)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Code-as-action agents, model-agnostic (local/OpenAI/Anthropic + others), sandboxed execution (E2B/Modal/Docker), MCP + LangChain tool import, multi-agent support; no first-party SWE-bench/GAIA headline for the framework", "confidence": "medium", "note": "Deliberately lightweight: strong on model-agnosticism, tool-use, and sandboxing but thinner orchestration/eval surface than heavier frameworks; no benchmark, scored 3 on feature matrix.", "sources": [ { "url": "https://github.com/huggingface/smolagents", "shows": "code-action agents, sandboxed exec, MCP/LangChain tools, multi-model features", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "smolagents by Hugging Face, minimal (~1k-LOC core) code-writing agent framework ('agents that think in code'). Apache-2.0. Verified live June 2026 on GitHub." }, { "product": "Hermes Agent", "org": "Nous Research", "type": "software", "description": "Open-source autonomous agent that grows with the user through persistent memory, reusable skills, and a built-in learning loop. It runs on a VPS or serverless backend, can be controlled through Telegram and other messaging surfaces, and is designed to keep improving over long-lived sessions rather than acting as a one-shot chat wrapper. Scale signal: the Hermes Agent repo is MIT-licensed, has 163k+ GitHub stars, and shows very heavy recent development activity.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(github.com/NousResearch/hermes-agent);no-feature-gated-core;model-agnostic(multi-provider);self-host-by-design", "confidence": "high", "note": "Fully MIT-licensed, complete public source, self-hosted by design, no proprietary core; top openness, above open-core framework anchors.", "sources": [ { "url": "https://github.com/NousResearch/hermes-agent", "shows": "MIT LICENSE, full public source, self-improving multi-channel agent (Python)", "accessed": "2026-06-04" }, { "url": "https://hermes-agent.nousresearch.com", "shows": "open source MIT autonomous agent product page", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Real usage signal beyond stars (adoption!=stars rule): Hermes Agent reported #1 on OpenRouter's GLOBAL token rankings, confirmed by Nous Research's own announcement (~271B tokens, May 6 2026) and an OpenRouter app page; secondary reports give ~224B daily tokens by May 10 (surpassing OpenClaw's ~186B). Both figures are reported; token-volume mapped to 1-10M-equivalent reach (inference, not a disclosed installed-user count). GitHub stars (~181K, <4 months) corroborate but are not the basis. Level 4 / medium confidence: deciding metric now backed by a primary source, but reach is inferred from token volume.", "sources": [ { "url": "https://github.com/NousResearch/hermes-agent", "shows": "active releases (latest v0.15.2, May 29 2026), ~181K stars in <4 months (corroborating, not basis)", "accessed": "2026-06-04" }, { "url": "https://x.com/NousResearch/status/2052904761087729897", "shows": "PRIMARY: Nous Research announcing Hermes Agent #1 on OpenRouter global token rankings", "accessed": "2026-06-04" }, { "url": "https://openrouter.ai/apps/hermes-agent", "shows": "PRIMARY: OpenRouter app page for Hermes Agent (token-usage ranking platform)", "accessed": "2026-06-04" }, { "url": "https://chatforest.com/builders-log/hermes-agent-nous-research-openrouter-number-one-self-improving-builder-guide/", "shows": "aggregator detail: ~271B tokens, surpassing OpenClaw's 186B (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Self-improving learning loop (auto skill creation + refinement), persistent cross-session memory, 40+ tools, subagent delegation, NL cron scheduling, containerized sandboxing, web/browser control, multi-channel, model-agnostic; no first-party SWE-bench/GAIA headline", "confidence": "medium", "note": "Distinctive autonomy (self-improvement loop + persistent memory + subagent delegation) and very broad tool/channel coverage place it high on the feature matrix; no published benchmark so scored 4, just below a verified-benchmark ceiling.", "sources": [ { "url": "https://github.com/NousResearch/hermes-agent", "shows": "self-improving skills, persistent memory, 40+ tools, subagents, cron, sandboxing features", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Hermes Agent by Nous Research, open source self-improving autonomous agent (auto-generates skills, persistent memory, 40+ tools, subagent delegation, NL cron, multi-channel). MIT. Released ~Feb 2026; verified live June 2026 (latest release v0.15.2, 29 May 2026; ~181K stars)." }, { "product": "Dify", "org": "LangGenius", "type": "software", "description": "Active 2026; ~144K GitHub stars", "openness": { "score": 3, "class": "source_available", "components": "license:Dify-Open-Source-License(Apache-2.0-based,non-OSI,added-conditions:multi-tenant/branding-restrictions);source:public;cloud+enterprise-tiers", "confidence": "high", "note": "license:Dify-Open-Source-License(Apache-2.0-based,non-OSI,added-conditions:multi-tenant/branding-restrictions);source:public;cloud+enterprise-tiers", "sources": [ { "url": "https://github.com/langgenius/dify", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://dify.ai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "high", "note": "~144K GitHub stars plus Dify Cloud managed tier; no honest active-user/download volume figure verified, so capped at 3 on stars fallback.", "sources": [ { "url": "https://github.com/langgenius/dify", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Production agentic-workflow builder: visual workflow, RAG pipeline, agent capabilities, model management across hundreds of LLMs, and built-in observability (Langfuse, Opik, Arize Phoenix). Rated on feature breadth.", "sources": [ { "url": "https://github.com/langgenius/dify", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://dify.ai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Active 2026; ~144K GitHub stars" }, { "product": "CrewAI", "org": "CrewAI", "type": "software", "description": "Active 2026; ~52.4k GitHub stars (late May 2026)", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(framework);source:public;CrewAI-Enterprise/AMP-managed-platform-tier", "confidence": "high", "note": "license:MIT(framework);source:public;CrewAI-Enterprise/AMP-managed-platform-tier", "sources": [ { "url": "https://github.com/crewAIInc/crewAI/blob/main/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/crewaiinc/crewai", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://crewai.com/open-source", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "27M+ cumulative PyPI downloads with ~5M in the last month, 2B+ agent executions in past 12 months. Monthly download volume places it in the 1-10M range.", "sources": [ { "url": "https://www.getpanto.ai/blog/crewai-platform-statistics", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://automationatlas.io/tools/crewai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Role-based multi-agent orchestration framework (crews/flows). No first-party SWE-bench/GAIA score verified for the framework itself; rated on feature matrix as an opinionated agent-orchestration library.", "sources": [ { "url": "https://github.com/crewaiinc/crewai", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.crewai.com/en/concepts/agents", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "Active 2026; ~52.4k GitHub stars (late May 2026)" }, { "product": "OpenHands", "org": "All Hands AI", "type": "software", "description": "Open-source autonomous coding agent (formerly OpenDevin) that can write code, run commands, browse the web, and interact with APIs to complete software engineering tasks end-to-end. Differentiates through a sandboxed execution environment and strong SWE-bench performance, consistently ranking among the top open source agents. 74.5K GitHub stars with 460+ contributors and extremely active development (710+ commits in 90 days).", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core);enterprise-dir:separate-license;source:public;OpenHands-Cloud-managed-tier", "confidence": "high", "note": "license:MIT(core);enterprise-dir:separate-license;source:public;OpenHands-Cloud-managed-tier", "sources": [ { "url": "https://github.com/All-Hands-AI/OpenHands", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.openhands.dev/sdk", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "~75.8k GitHub stars, $18.8M Series A, contributors from major firms, OpenHands Cloud with multi-user/Slack/Jira/Linear integrations. No precise active-user/download count verified; rated 3 on reported traction.", "sources": [ { "url": "https://github.com/All-Hands-AI/OpenHands", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://theaiagentindex.com/agents/openhands", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "benchmark:SWE-bench Verified", "value": null, "confidence": "high", "note": "Reported 77.6% on SWE-bench Verified, at or above proprietary coding agents; model-agnostic (100+ LLM providers, BYOK).", "sources": [ { "url": "https://github.com/All-Hands-AI/OpenHands", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://theaiagentindex.com/agents/openhands", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.marktechpost.com/2026/05/15/best-ai-agents-for-software-development-ranked-a-benchmark-driven-look-at-the-current-field/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "Active 2026 (All-Hands AI); ~75.8k GitHub stars; SWE-bench Verified ~77.6" }, { "product": "LangChain", "org": "LangChain", "type": "software", "description": "Active 2026; MIT core (langchain-core, langgraph, integrations); LangSmith is commercial", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(langchain-core,langgraph,integrations);commercial:LangSmith-platform/cloud;source:public", "confidence": "high", "note": "license:MIT(langchain-core,langgraph,integrations);commercial:LangSmith-platform/cloud;source:public", "sources": [ { "url": "https://github.com/langchain-ai/langchain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.langchain.com/langchain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://rvernica.github.io/2026/03/langchain-license", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "LangChain + LangGraph open source frameworks surpassed 1B cumulative downloads with 90M+ combined monthly downloads. Monthly volume exceeds 10M, placing it at the top band.", "sources": [ { "url": "https://www.langchain.com/langchain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://grokipedia.com/page/LangChain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Broadest LLM/agent orchestration ecosystem: chains, LangGraph stateful agent graphs, hundreds of model/tool integrations. No single benchmark for the framework; rated on feature breadth.", "sources": [ { "url": "https://github.com/langchain-ai/langchain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.langchain.com/langchain", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "Active 2026; MIT core (langchain-core, langgraph, integrations); LangSmith is commercial" }, { "product": "LangGraph", "org": "LangChain", "type": "software", "description": "Low-level orchestration framework from LangChain for building stateful, multi-actor agent applications. Unlike higher-level frameworks it exposes an explicit graph/state-machine model (nodes, edges, persistence, human-in-the-loop checkpoints), the production choice when you need fine-grained control over loops, branching, and durable resumable execution.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core);commercial:LangGraph-Platform/LangSmith;source:public", "confidence": "high", "note": "MIT-licensed self-hostable core with a commercial managed deployment platform.", "sources": [ { "url": "https://github.com/langchain-ai/langgraph", "shows": "MIT, ~35k stars, low-level agent orchestration", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~58M PyPI downloads/month (directional - largely transitive via LangChain).", "sources": [ { "url": "https://pypistats.org/packages/langgraph", "shows": "~58.6M downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "graph control flow; checkpointing/persistence; durable resumable execution; HITL interrupts; streaming; subgraphs", "confidence": "high", "note": "Deepest durable-execution control among the orchestration peers.", "sources": [ { "url": "https://github.com/langchain-ai/langgraph", "shows": "feature set: graph control flow, persistence, HITL", "accessed": "2026-06-17" } ] }, "maturity": 5.0, "mature": true, "version_note": "MIT core; commercial LangGraph Platform / LangSmith is the managed layer (open-core). ~35k stars, ~58M PyPI downloads/month (heavily transitive via LangChain)." }, { "product": "AutoGen", "org": "Microsoft", "type": "software", "description": "Microsoft's framework for agentic AI centred on multi-agent conversation patterns where LLM agents collaborate, critique, and run tools/code. Its event-driven v0.4 redesign and code-execution focus distinguish it from graph- or role-based frameworks.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(code,via LICENSE-CODE);docs:CC-BY-4.0;source:public;no-feature-gated-core", "confidence": "high", "note": "Code is MIT and fully open; the GitHub-surfaced CC-BY-4.0 is only the docs license.", "sources": [ { "url": "https://github.com/microsoft/autogen/blob/main/LICENSE-CODE", "shows": "MIT code license", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~1.37M downloads/month for autogen-agentchat; trajectory flat as it enters maintenance mode.", "sources": [ { "url": "https://pypistats.org/packages/autogen-agentchat", "shows": "~1.37M downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "multi-agent conversation orchestration; preset team patterns; tool use; code execution; async/event-driven; AutoGen Studio", "confidence": "medium", "note": "Strong multi-agent conversation model; less durable-execution depth than LangGraph and now in maintenance.", "sources": [ { "url": "https://github.com/microsoft/autogen", "shows": "feature set: multi-agent conversation, code execution", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "Code is MIT (LICENSE-CODE; the repo's root LICENSE is CC-BY-4.0 for docs, which mislabels the GitHub license field). In maintenance mode - Microsoft now directs new work to the Microsoft Agent Framework (Semantic Kernel + AutoGen merge). ~59k stars." }, { "product": "PydanticAI", "org": "Pydantic", "type": "software", "description": "Python agent framework from the Pydantic team bringing type-safety to production LLM agents: structured/validated outputs, typed dependency injection, and model-agnostic providers, with first-class Pydantic Logfire observability.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(library);commercial:Pydantic-Logfire;source:public", "confidence": "high", "note": "MIT type-first agent library with a commercial Logfire observability platform.", "sources": [ { "url": "https://github.com/pydantic/pydantic-ai", "shows": "MIT, ~17.8k stars, Logfire integration", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "medium", "note": "~31M PyPI downloads/month (heavily transitive/CI-driven; directional).", "sources": [ { "url": "https://pypistats.org/packages/pydantic-ai", "shows": "~31M downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "typed/structured outputs; typed DI; model-agnostic; tool calling; streaming; multi-agent; MCP; OTel/Logfire", "confidence": "medium", "note": "Lean, type-safe DX; narrower built-in RAG than LlamaIndex and fewer prebuilt role abstractions than CrewAI.", "sources": [ { "url": "https://pypi.org/project/pydantic-ai/", "shows": "MIT, typed agent framework", "accessed": "2026-06-17" } ] }, "maturity": 4.3, "mature": false, "version_note": "MIT library; commercially-operated Pydantic Logfire platform makes the overall offering open-core. ~18k stars." }, { "product": "Haystack", "org": "deepset", "type": "software", "description": "deepset's open source orchestration framework for production LLM apps in Python, built around composable modular pipelines and agent workflows with explicit control over retrieval, routing, memory, and generation. Originally RAG/search-focused; Haystack 2.x is a general orchestrator.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(core);commercial:Haystack-Enterprise;source:public", "confidence": "high", "note": "Apache-2.0 self-hostable core with a commercial enterprise tier.", "sources": [ { "url": "https://github.com/deepset-ai/haystack", "shows": "Apache-2.0, ~25.6k stars, Haystack Enterprise tiers", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~883k PyPI downloads/month for the 2.x haystack-ai package.", "sources": [ { "url": "https://pypistats.org/packages/haystack-ai", "shows": "~883k downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "modular pipeline/DAG composition; retrievers + document stores (RAG); routing; memory; agent workflows; tool calling; evaluation; broad integrations", "confidence": "high", "note": "Most mature combined RAG + orchestration feature set among the peers.", "sources": [ { "url": "https://github.com/deepset-ai/haystack", "shows": "feature set: pipelines, RAG, agents, eval", "accessed": "2026-06-17" } ] }, "maturity": 4.7, "mature": true, "version_note": "Apache-2.0 core; commercial Haystack Enterprise (managed/governance) makes the overall offering open-core. ~26k stars, ~883k PyPI downloads/month (haystack-ai)." }, { "product": "LlamaIndex", "org": "LlamaIndex", "type": "software", "description": "Open-source data framework (Python and TypeScript) for building LLM apps over private data: data connectors, indexing/retrieval, and an agent/Workflows orchestration layer. Its depth is in RAG and data ingestion - 'document agents', many-format parsing, and structured extraction.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core);commercial:LlamaCloud/LlamaParse;source:public", "confidence": "high", "note": "MIT self-hostable core with a commercial managed cloud.", "sources": [ { "url": "https://github.com/run-llama/llama_index", "shows": "MIT, ~50.2k stars, LlamaCloud commercial tier", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~10.18M PyPI downloads/month.", "sources": [ { "url": "https://pypistats.org/packages/llama-index", "shows": "~10.18M downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "data connectors (LlamaHub); multiple index/retrieval strategies; query engines; agentic Workflows; structured extraction; multimodal/OCR; TS port", "confidence": "high", "note": "Best-in-class RAG/data-ingestion depth plus an agent layer.", "sources": [ { "url": "https://github.com/run-llama/llama_index", "shows": "feature set: connectors, indices, Workflows, extraction", "accessed": "2026-06-17" } ] }, "maturity": 5.0, "mature": true, "version_note": "MIT core; commercial LlamaCloud / LlamaParse (parse/extract/index APIs) makes the overall offering open-core. ~50k stars, ~10M PyPI downloads/month." }, { "product": "OpenManus", "org": "FoundationAgents", "type": "software", "description": "Open-source reimplementation of the Manus autonomous agent, built by MetaGPT/DeepWisdom team members as a no-invite-code general agent that plans and executes tasks via tools, browser automation, and computer use. Framed as a run-from-source product rather than a reusable library.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;self-host:primary;no-feature-gated-core", "confidence": "high", "note": "Fully MIT, run from source, no commercial layer.", "sources": [ { "url": "https://github.com/FoundationAgents/OpenManus", "shows": "MIT, ~56.6k stars, MetaGPT-team maintainers", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~57k stars but negligible package usage (~186 PyPI downloads/month) - stars_fallback (capped at 3).", "sources": [ { "url": "https://pypistats.org/packages/openmanus", "shows": "~186 downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "single autonomous agent loop; tool use; browser automation; computer-use actions; planning; MCP; experimental RL branch", "confidence": "medium", "note": "Capable demo agent but narrower and less battle-tested than the framework peers.", "sources": [ { "url": "https://github.com/FoundationAgents/OpenManus", "shows": "feature set: autonomous agent, browser, MCP", "accessed": "2026-06-17" } ] }, "maturity": 3.0, "mature": false, "version_note": "MIT, no paid/feature-gated core. ~57k stars but a vestigial PyPI package (~186 downloads/month) - run via git clone, not pip." }, { "product": "DSPy", "org": "Stanford NLP", "type": "software", "description": "Stanford framework for 'programming - not prompting - language models': compose typed declarative modules (signatures, Predict/ChainOfThought/ReAct) into pipelines, then hand them to optimizers (MIPROv2, BootstrapFewShot, GEPA) that auto-tune prompts and/or weights against a metric.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core;no-commercial-saas", "confidence": "high", "note": "Fully MIT with no commercial tier.", "sources": [ { "url": "https://github.com/stanfordnlp/dspy", "shows": "MIT, ~35.1k stars", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~6.36M PyPI downloads/month for the dspy package.", "sources": [ { "url": "https://pypistats.org/packages/dspy", "shows": "~6.36M downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "declarative modules + typed signatures; automatic prompt optimization/compilation; weight fine-tuning; ReAct/CoT/RAG patterns; assertions; eval", "confidence": "high", "note": "Best-in-class on prompt-optimization/compilation; thinner multi-agent coordination than CrewAI/AutoGen.", "sources": [ { "url": "https://github.com/stanfordnlp/dspy", "shows": "feature set: modules, optimizers, compilation", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT, no commercial cloud from the project. ~35k stars, ~6.4M PyPI downloads/month." }, { "product": "GraphRAG", "org": "Microsoft", "type": "software", "description": "Microsoft Research's modular graph-based RAG system: LLMs extract a knowledge graph (entities + relationships) from unstructured text, cluster it into hierarchical communities, generate community summaries, and answer via Global/Local/DRIFT search - built for corpus-wide 'sensemaking' questions where naive vector RAG falls short.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/microsoft/graphrag", "shows": "MIT, ~33.8k stars", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~34k stars; no published download telemetry - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/microsoft/graphrag", "shows": "~33.8k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "KG extraction; hierarchical community detection; community summarization; Global/Local/DRIFT query modes; auto prompt-tuning", "confidence": "medium", "note": "Turnkey for graph-RAG sensemaking but a deliberately narrow single technique.", "sources": [ { "url": "https://microsoft.github.io/graphrag/", "shows": "architecture + Global/Local/DRIFT modes", "accessed": "2026-06-17" } ] }, "maturity": 3.7, "mature": false, "version_note": "MIT, no feature-gated core. ~34k stars. Indexing is compute/cost-intensive." }, { "product": "LightRAG", "org": "HKU Data Science Lab", "type": "software", "description": "Lightweight graph-based RAG framework from HKU Data Science Lab (EMNLP 2025) combining knowledge graphs with vector embeddings via dual-layer retrieval. Drops GraphRAG's expensive community-report step and supports incremental updates (add data without rebuilding the global index).", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/HKUDS/LightRAG", "shows": "MIT, ~36.7k stars, EMNLP 2025", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~251k PyPI downloads/month (lightrag-hku) corroborating ~37k stars.", "sources": [ { "url": "https://pepy.tech/projects/lightrag-hku", "shows": "~251k downloads/month", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "dual-level (local+global) graph retrieval; KG construction; incremental updates; naive/local/global/hybrid query; pluggable storage; multimodal parsing; API + web UI", "confidence": "medium", "note": "Deeper and more efficient than GraphRAG for graph-RAG; narrower than general orchestrators.", "sources": [ { "url": "https://github.com/HKUDS/LightRAG", "shows": "feature set: dual-level retrieval, incremental updates", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT. ~37k stars, ~251k monthly PyPI downloads (lightrag-hku)." }, { "product": "nano-graphrag", "org": "gusye1234", "type": "software", "description": "A simple, hackable ~1,100-line reimplementation of Microsoft's GraphRAG, built because the official version is hard to read or modify. Portable, fully async/typed, with swappable graph/vector/LLM backends - a lightweight reference and customization base rather than a production system.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/gusye1234/nano-graphrag", "shows": "MIT, ~3.9k stars", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 2, "signal_type": "stars_fallback", "confidence": "medium", "note": "~3.9k stars and ~1.7k monthly PyPI; no release since Oct 2024 (likely stale) - stars_fallback.", "sources": [ { "url": "https://pypi.org/project/nano-graphrag/", "shows": "v0.0.8.2, last release Oct 2024", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "global+local queries; KG community detection; pluggable stores; multi-provider LLMs; incremental insert; async+typed", "confidence": "medium", "note": "Trades breadth/polish for a small readable core.", "sources": [ { "url": "https://github.com/gusye1234/nano-graphrag", "shows": "~1,100-line GraphRAG reimplementation", "accessed": "2026-06-17" } ] }, "maturity": 2.7, "mature": false, "version_note": "MIT. ~3.9k stars. Likely stale - no release since Oct 2024." }, { "product": "Pathway", "org": "Pathway", "type": "software", "description": "Python ETL/stream-processing framework on a Rust dataflow engine, unifying batch and streaming under one API with production RAG/LLM templates and live connectors. Unlike glue-layer frameworks it is an incremental dataflow engine - it keeps vector indexes continuously synced with changing data (real-time RAG) instead of batch re-indexing.", "openness": { "score": 2, "class": "source_available", "components": "license:BSL-1.1(non-OSI:single-machine + no-compete, Change-Date 2027-07-20 -> Apache-2.0);commercial:Scale/Enterprise;source:public", "confidence": "high", "note": "Source-available under BSL 1.1, not open source; production use is license-restricted with paid enterprise tiers.", "sources": [ { "url": "https://github.com/pathwaycom/pathway", "shows": "BSL 1.1, ~62.9k stars", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "signal_type": "usage_volume", "confidence": "medium", "note": "~30k monthly PyPI downloads alongside ~63k stars (star count inflated for an infra repo).", "sources": [ { "url": "https://pypi.org/project/pathway/", "shows": "BSL, active releases, download stats", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "unified batch+streaming incremental dataflow (Rust); real-time RAG with continuously-synced indexing; 300+ connectors; hybrid search; single-node -> distributed/K8s", "confidence": "medium", "note": "Stronger on streaming/incremental infra than LLM-orchestration breadth.", "sources": [ { "url": "https://github.com/pathwaycom/pathway", "shows": "feature set: incremental dataflow, real-time RAG", "accessed": "2026-06-17" } ] }, "maturity": 3.7, "mature": false, "version_note": "Business Source License 1.1 (BSL, NOT OSI): single-machine production limit + no-compete clause, converts to Apache-2.0 on 2027-07-20. Commercial Scale/Enterprise tiers. ~63k stars." }, { "product": "OpenCode", "org": "SST / Anomaly", "type": "software", "description": "Open-source terminal (TUI) AI coding agent from the SST team (now Anomaly), aggressively provider-agnostic with 75+ LLM providers (Claude, GPT, Gemini, Bedrock, local models). A polished native TUI with multi-session agents, shareable sessions, built-in LSP, and a client/server architecture.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/sst/opencode", "shows": "MIT, ~176k stars, open coding agent", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "reported_traction", "confidence": "medium", "note": "~176k GitHub stars and active npm distribution (opencode-ai); clean download count not retrievable.", "sources": [ { "url": "https://registry.npmjs.org/opencode-ai/latest", "shows": "npm opencode-ai, MIT, active", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "native TUI agent; 75+ providers incl. local; multi-session concurrent agents; shareable sessions; built-in LSP; client/server", "confidence": "high", "note": "Matches/exceeds the coding-agent peers on breadth, with standout provider-neutrality and TUI/LSP polish.", "sources": [ { "url": "https://opencode.ai/", "shows": "feature set: TUI, 75+ providers, LSP, sessions", "accessed": "2026-06-17" } ] }, "maturity": 4.7, "mature": true, "version_note": "MIT. ~176k stars; repo recently renamed sst/opencode -> anomalyco/opencode (old URL redirects). Distributed via npm (opencode-ai)." }, { "product": "Semantic Kernel", "org": "Microsoft", "type": "software", "description": "Microsoft's lightweight open source SDK for building AI agents and integrating LLMs into C#, Python, or Java code. It exposes your code as model-callable 'plugins' (often via OpenAPI), with connectors to AI services and vector stores, telemetry, and responsible-AI hooks. Its differentiator vs the Python-first frameworks is genuine, first-class multi-language support (.NET and Java, not Python-only).", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(C#/Python/Java);model-agnostic;no-feature-gated-core", "confidence": "high", "note": "MIT multi-language SDK; first-class Azure connectors but model-agnostic and self-hostable.", "sources": [ { "url": "https://github.com/microsoft/semantic-kernel", "shows": "MIT, ~28.2k stars, multi-language agent SDK", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~28k stars; PyPI volume not surfaced. Entering maintenance as Microsoft Agent Framework becomes the successor - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/microsoft/semantic-kernel", "shows": "~28.2k stars; MAF-successor banner", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "multi-LLM connectors; plugin/OpenAPI system; auto function-calling orchestration; agent abstractions; memory/vector connectors; telemetry; RAI filters; first-class .NET + Java", "confidence": "medium", "note": "Enterprise multi-language orchestration with uniquely strong .NET/Java coverage; now superseded by Microsoft Agent Framework.", "sources": [ { "url": "https://learn.microsoft.com/en-us/agent-framework/overview/", "shows": "MAF is the successor to SK + AutoGen", "accessed": "2026-06-18" } ] }, "maturity": 3.7, "mature": false, "version_note": "MIT. ~28k stars. Microsoft has converged Semantic Kernel + AutoGen into the Microsoft Agent Framework (the successor); SK is now in maintenance mode (bug/security fixes only), supported ~1 year past MAF GA." }, { "product": "BeeAI Framework", "org": "IBM", "type": "software", "description": "Open-source toolkit for production-ready multi-agent systems with full feature parity across Python and TypeScript - its signature differentiator vs the Python-first peers. Emphasizes production concerns (caching, memory, OpenTelemetry observability), constraint-based agent governance, native MCP + A2A interop, and 10+ LLM backends.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Python+TS);no-feature-gated-core", "confidence": "high", "note": "Apache-2.0 across Python + TypeScript with no commercial gating.", "sources": [ { "url": "https://github.com/i-am-bee/beeai-framework", "shows": "Apache-2.0, ~3.3k stars, dual-language agent framework", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~3.3k stars; package download volume not retrievable - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/i-am-bee/beeai-framework", "shows": "~3.3k stars", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Python+TS parity; agents/multi-agent workflows; RAG; tools; memory; YAML + decorator orchestration (parallelism/retries/replanning); constraint governance; MCP + A2A; OpenTelemetry; 10+ LLM backends", "confidence": "medium", "note": "Distinctive dual-language parity and production focus; smaller adoption than the leading frameworks.", "sources": [ { "url": "https://lfaidata.foundation/projects/beeai/", "shows": "LF-hosted; dual-language production agent framework", "accessed": "2026-06-18" } ] }, "maturity": 3.7, "mature": false, "version_note": "Apache-2.0. IBM-originated, now under the Linux Foundation (LF AI & Data), alongside Docling and Data Prep Kit. ~3.3k stars." }, { "product": "Gemini CLI", "org": "Google", "type": "software", "description": "Google's open source terminal AI coding agent, bringing Gemini models into the shell for coding, debugging, and task automation. Ships built-in tools (Google Search grounding, file ops, shell, web fetch), is extensible via MCP and GEMINI.md system-prompt files, and offers a generous free tier on a personal Google account. Unlike the closed peers (Cursor, Claude Code), it is fully Apache-2.0.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully Apache-2.0; commercial relationship is on model quotas, not the tool.", "sources": [ { "url": "https://github.com/google-gemini/gemini-cli", "shows": "Apache-2.0, ~105k stars, open AI agent CLI", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~2.86M npm downloads/month for @google/gemini-cli, plus ~105k stars.", "sources": [ { "url": "https://api.npmjs.org/downloads/point/last-month/@google/gemini-cli", "shows": "~2.86M downloads/month", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "agentic terminal loop (read/write, shell, planning); MCP extensibility; built-in Google Search grounding; large context; OAuth/API-key/Vertex auth; VS Code + Actions integration", "confidence": "high", "note": "Top-tier coding agent; Search grounding is a standout vs peers.", "sources": [ { "url": "https://blog.google/innovation-and-ai/technology/developers-tools/introducing-gemini-cli-open-source-ai-agent/", "shows": "official launch; free tier, MCP, GEMINI.md", "accessed": "2026-06-18" } ] }, "maturity": 5.0, "mature": true, "version_note": "Apache-2.0. ~105k stars, ~2.86M npm downloads/month. Free tier via personal Google account; paid via AI Studio/Vertex." }, { "product": "Zed", "org": "Zed Industries", "type": "software", "description": "High-performance, multiplayer code editor written in Rust from the creators of Atom and Tree-sitter, built for human-AI collaboration. A GUI editor (peer to Cursor/Windsurf) with agentic editing, parallel agents across projects, an open source edit-prediction model (Zeta2), inline assistant, MCP servers, and pluggable external agents (Claude, Codex, OpenCode) via the Agent Communication Protocol - BYO-key or Zed-hosted models.", "openness": { "score": 4, "class": "open_core", "components": "license:GPL-3.0-or-later(editor)+AGPL-3.0(collab-server)+Apache-2.0(GPUI);source:public;commercial:Zed-Pro/Business(hosted AI)", "confidence": "high", "note": "Fully open editor core under GPL/AGPL; paid hosted-AI and governance tiers make the overall offering open-core.", "sources": [ { "url": "https://github.com/zed-industries/zed", "shows": "Rust, ~85k stars, GPL/AGPL/Apache licensing", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "reported_traction", "confidence": "medium", "note": "~85k stars; a widely-used released editor with paid Pro/Business tiers, but no public install count.", "sources": [ { "url": "https://zed.dev/pricing", "shows": "Personal $0 / Pro $10 / Business $30 tiers; released product", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "GUI editor; agentic editing with task delegation + review; parallel multi-agent; OSS edit prediction (Zeta2); inline assistant; multi-model BYO-key + hosted; MCP; external agents via ACP; multiplayer; native Rust perf", "confidence": "high", "note": "Full AI-editor feature set on par with Cursor/Windsurf, model-agnostic.", "sources": [ { "url": "https://zed.dev", "shows": "agentic editing, Zeta2, ACP/MCP, multi-model", "accessed": "2026-06-18" } ] }, "maturity": 4.7, "mature": true, "version_note": "Editor is GPL-3.0-or-later; collab server AGPL-3.0; GPUI framework Apache-2.0. Paid Pro ($10/mo, hosted AI) and Business ($30/seat/mo, governance) tiers layer on the OSS editor -> open-core. ~85k stars." }, { "product": "Spec Kit", "org": "GitHub (Microsoft)", "type": "software", "description": "GitHub's open source toolkit for Spec-Driven Development that works with coding agents rather than being one. A structured Specify -> Plan -> Tasks -> Implement workflow produces Markdown artifacts that feed each phase, giving agents rich structured context instead of ad-hoc prompts. The specify CLI scaffolds these files and orchestrates 30+ agents (Copilot, Claude Code, Cursor, Gemini CLI, Codex CLI, Windsurf, Zed, OpenCode).", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core;no-commercial-tier", "confidence": "high", "note": "Fully MIT, free.", "sources": [ { "url": "https://github.com/github/spec-kit", "shows": "MIT, ~113k stars, Spec-Driven Development toolkit", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~113k stars but git-distributed with no download telemetry - stars_fallback (capped at 3; true usage higher).", "sources": [ { "url": "https://api.github.com/repos/github/spec-kit", "shows": "~113k stars, SPDX MIT", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "four-phase Spec-Driven workflow producing chained Markdown artifacts; specify CLI scaffolding; agent-agnostic (30+ agents); constitution/principles guardrail", "confidence": "medium", "note": "Methodology/orchestration layer over coding agents rather than a code-writing engine itself.", "sources": [ { "url": "https://github.blog/ai-and-ml/generative-ai/spec-driven-development-with-ai-get-started-with-a-new-open-source-toolkit/", "shows": "SDD definition, four-phase process", "accessed": "2026-06-18" } ] }, "maturity": 3.7, "mature": false, "version_note": "MIT (c) GitHub. ~113k stars. Git-based distribution (uv tool install from the repo); no PyPI telemetry. No paid tier." }, { "product": "Langflow", "org": "Langflow (DataStax)", "type": "software", "description": "Open-source, low-code visual builder for AI agents and RAG/workflow applications. You wire components on a drag-and-drop canvas, then expose flows as APIs or run them as agents. Its differentiator vs code-first peers (LangChain, LlamaIndex) is the visual editor with hundreds of pre-built components, while staying model-agnostic and Python-extensible.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core;commercial:DataStax-hosted(separate)", "confidence": "high", "note": "MIT core, fully open; managed cloud does not gate the OSS core.", "sources": [ { "url": "https://github.com/langflow-ai/langflow", "shows": "MIT, ~149.8k stars, visual agent builder", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~150k stars and ~71k PyPI downloads/month (modest pip volume; mostly Docker/desktop).", "sources": [ { "url": "https://pypi.org/project/langflow/", "shows": "package langflow, MIT", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "visual drag-and-drop flow builder; 100s of pre-built components; agent + multi-agent; RAG pipelines; MCP; API/playground deployment; Python-extensible custom components", "confidence": "medium", "note": "Mature low-code visual builder (peer to Dify); model/vendor-agnostic.", "sources": [ { "url": "https://www.langflow.org/", "shows": "visual low-code agent/RAG builder", "accessed": "2026-06-18" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT. ~150k stars, ~71k PyPI downloads/month. Began as Logspace; acquired by DataStax (2024), now an IBM company (IBM->DataStax announced Feb 2025); IBM committed to the OSS community. Commercial managed cloud exists but does not gate the core." }, { "product": "RAGFlow", "org": "InfiniFlow", "type": "software", "description": "Open-source RAG engine built around deep document understanding - template-based chunking and layout-aware parsing of complex documents (PDFs, tables, figures, scans) to produce grounded, citation-backed answers. Unlike general orchestration frameworks, it is a batteries-included RAG application/server with strong document parsing and visible source citations, recently extended with agent capabilities.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;self-host:primary(Docker);commercial:cloud/enterprise(separate)", "confidence": "high", "note": "Apache-2.0 self-hostable engine; cloud is the commercial tier.", "sources": [ { "url": "https://github.com/infiniflow/ragflow", "shows": "Apache-2.0, ~83k stars, RAG engine", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~83k stars; Docker-distributed with no download metric retrieved - stars_fallback (capped at 3).", "sources": [ { "url": "https://ragflow.io", "shows": "official site, InfiniFlow, self-host + cloud", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "deep document understanding / layout-aware parsing; template-based chunking; grounded citations with source traceback; hybrid (vector + full-text) retrieval; knowledge-base UI; LLM-agnostic; agent workflows", "confidence": "medium", "note": "Opinionated, document-centric RAG application; deeper parsing than general frameworks, narrower scope.", "sources": [ { "url": "https://github.com/infiniflow/ragflow", "shows": "feature set: deep doc understanding, citations, hybrid retrieval", "accessed": "2026-06-18" } ] }, "maturity": 3.7, "mature": false, "version_note": "Apache-2.0. ~83k stars. Deployed via Docker Compose (no first-party PyPI/npm). Commercial managed cloud / enterprise edition exists; the self-hosted engine is fully open." }, { "product": "n8n", "org": "n8n", "type": "software", "description": "Fair-code workflow automation platform with native AI/agent capabilities. It pairs a visual node-based canvas with drop-in JavaScript/Python code, 400+ integrations, and LangChain-based nodes for building LLM agents and AI workflows. It is a general automation/iPaaS platform first, with AI as a native layer, and is self-hostable.", "openness": { "score": 2, "class": "source_available", "components": "license:Sustainable-Use-License(fair-code,non-OSI: internal-use-only,no-resale/SaaS)+n8n-Enterprise-License;source:public;commercial:n8n-Cloud/Enterprise", "confidence": "high", "note": "Source-available fair-code, not OSI open source: the license restricts commercial/hosted use.", "sources": [ { "url": "https://docs.n8n.io/sustainable-use-license/", "shows": "Sustainable Use License: fair-code, non-OSI, use restrictions", "accessed": "2026-06-18" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~193k stars (one of the most-starred repos) and ~316k npm downloads/month.", "sources": [ { "url": "https://www.npmjs.com/package/n8n", "shows": "npm n8n, ~316k downloads/month", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "visual node-based workflow builder; 400+ integrations; native AI/LLM + agent nodes (LangChain); custom JS/Python code; webhooks/triggers/scheduling; self-host or cloud; RBAC (gated)", "confidence": "medium", "note": "Broad automation/iPaaS scope with AI as one native layer; wider but shallower on pure agent depth than the dedicated frameworks.", "sources": [ { "url": "https://github.com/n8n-io/n8n", "shows": "feature set: 400+ integrations, AI/agent nodes, code nodes", "accessed": "2026-06-18" } ] }, "maturity": 4.3, "mature": false, "version_note": "Sustainable Use License (fair-code, NOT OSI: internal-business-use only, no resale/hosting-as-a-service) + n8n Enterprise License; GitHub reports NOASSERTION. ~193k stars, ~316k npm downloads/month. n8n Cloud + Enterprise are the commercial tiers." }, { "product": "Datasette Agent", "org": "Datasette", "type": "software", "description": "LLM-powered agent assistant for Datasette, Simon Willison's tool for exploring and publishing structured data in SQLite. A chat interface at /-/agent answers natural-language questions against databases: the agent writes and runs SQL (save_query, execute_write_sql tools), explores data in the background, and renders results. Uses the datasette-llm plugin to call models and exposes an extensible tool surface (charts, image generation) for third-party tools.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);PyPI;no feature-gated core", "confidence": "high", "note": "Apache-2.0 verified against LICENSE body; full source public on the datasette org; no gated tier.", "sources": [ { "url": "https://github.com/datasette/datasette-agent/blob/main/LICENSE", "shows": "Apache License 2.0 full text", "accessed": "2026-06-19" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "stars_fallback", "confidence": "high", "note": "Pre-release alpha (PyPI 0.3a0, Jun 2026) roughly one month old; ~96 GitHub stars. Minimal adoption to date, included as an emerging on-thesis structured-data agent.", "sources": [ { "url": "https://github.com/datasette/datasette-agent", "shows": "~96 stars, 7 tags; PyPI 0.3a0 (15 Jun 2026)", "accessed": "2026-06-19" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "Natural-language-to-SQL agent over SQLite via Datasette; tools for save_query / execute_write_sql, background exploration, and pluggable tools (charts, image gen). Domain-specific; not a general coding agent; no standardized benchmark.", "confidence": "medium", "note": "Narrow, early structured-data agent on an alpha release; capable within its domain but limited in scope and maturity, so scored 2.", "sources": [ { "url": "https://agent.datasette.io/", "shows": "docs: /-/agent NL-to-SQL chat, save_query/execute_write_sql tools, pluggable tool surface", "accessed": "2026-06-19" } ] }, "maturity": 1.7, "mature": false, "version_note": "Apache-2.0 (verified LICENSE body). EARLY/ALPHA: latest PyPI 0.3a0 (15 Jun 2026), announced ~21 May 2026, runs on Datasette 1.0a33. ~96 stars, 7 tags. Verified 2026-06-19." }, { "product": "OpenFn", "org": "Open Function Group (OpenFn)", "type": "software", "description": "OpenFn (Open Function Group) is an open source workflow automation and data-integration platform used by governments and NGOs to orchestrate service workflows across legacy systems, web APIs, databases, and increasingly LLMs/AI tools. Its flagship product, Lightning, is built in Elixir/Phoenix and ships with 70+ adaptors. It is a verified Digital Public Good and Digital Square Global Good, maintained by a public benefit corporation since 2014.", "openness": { "score": 5, "class": "open_source", "components": "license:LGPL-3.0/GPL-3.0(OSI copyleft);source:public;no feature-gated core per maintainers", "confidence": "high", "note": "Fully open source DPG with copyleft licensing and no feature-gated core.", "sources": [ { "url": "https://github.com/OpenFn/lightning", "shows": "OpenFn/lightning, dual-licensed LGPL-3.0/GPL-3.0, fully open source, verified DPG", "accessed": "2026-06-22" }, { "url": "https://www.openfn.org/about", "shows": "Described as the leading DPG for workflow automation, FOSS since 2014", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "stars_fallback", "confidence": "medium", "note": "286 GitHub stars; reported use by 70+ NGOs/governments across 40+ countries.", "sources": [ { "url": "https://github.com/OpenFn/lightning", "shows": "286 GitHub stars", "accessed": "2026-06-22" }, { "url": "https://www.openfn.org/about", "shows": "used by 70+ NGOs and government ministries across 40+ countries", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "End-to-end workflow automation: visual builder, 70+ integration adaptors, AI/LLM orchestration, run monitoring, alerting, per-project access management.", "confidence": "high", "note": "Broad integration platform spanning authoring, many connectors, and operational monitoring.", "sources": [ { "url": "https://docs.openfn.org/documentation", "shows": "70+ adaptors and AI-orchestration / data-integration capabilities", "accessed": "2026-06-22" } ] }, "maturity": 3.7, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Fully open source DPG with copyleft licensing and no feature-gated core." }, { "product": "Hexabot", "org": "Hexastack", "type": "software", "description": "Hexabot is an AI chatbot / agent builder by the company Hexastack for creating multi-channel, multilingual conversational agents, deployed across North and East Africa. It offers a visual flow editor, a plugin/extension system, and integrations with LLMs (ChatGPT, Gemini, Mistral, Ollama, Claude) plus channels like WhatsApp, Telegram, Slack, and Facebook. v3 combines workflows, actions, agents, and conversational channels in one runtime.", "openness": { "score": 2, "class": "source_available", "components": "license:FCL-1.0-ALv2 (Fair Core License, NOT OSI; converts to Apache-2.0 two years after each release; competing/hosted commercial use restricted until conversion). Earlier v1/v2 were AGPL-3.0.", "confidence": "high", "note": "Fair Core License is a delayed-open / fair-code model, so the current core is source-available rather than open source.", "sources": [ { "url": "https://github.com/Hexastack/Hexabot", "shows": "Licensed under FCL-1.0-ALv2; GitHub reports license as 'Other'", "accessed": "2026-06-22" }, { "url": "https://hexabot.ai/", "shows": "Fair Core License: converts to Apache-2.0 after 2 years; competing commercial use restricted", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "stars_fallback", "confidence": "medium", "note": "974 GitHub stars and 218 forks; no package download figures available.", "sources": [ { "url": "https://github.com/Hexastack/Hexabot", "shows": "974 stars, 218 forks", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Visual chatbot flow editor, multilingual support, plugin marketplace, multi-channel connectors (WhatsApp/Telegram/Slack/Facebook), LLM+NLU integration, and (v3) agentic workflows with MCP support.", "confidence": "high", "note": "Broad conversational-AI platform spanning authoring, channels, LLM integration, and agentic workflows.", "sources": [ { "url": "https://github.com/Hexastack/Hexabot", "shows": "agentic workflows, actions, MCP support, multi-channel, admin UI and REST API", "accessed": "2026-06-22" } ] }, "maturity": 3.7, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Fair Core License is a delayed-open / fair-code model, so the current core is source-available rather than open source." } ] }, "ui_api": { "label": "UI & API", "arc": "Product / UX", "layer": "product_ux", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "Azure AI Model Inference", "org": "Microsoft Azure", "type": "software", "description": "Managed inference service providing a unified API across OpenAI models, open source models (Llama, Mistral, Phi), and Microsoft's own models. Handles model loading, scaling, and hardware allocation behind a single REST API. Supports serverless (pay-per-token) and provisioned (dedicated capacity) deployments. The enterprise default for organizations on Azure, integrated with Azure AD, networking, and compliance controls.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(Azure managed service);source:closed;interface:OpenAI-compatible REST endpoint over Foundry model deployments;hardware:Azure-managed", "confidence": "high", "note": "Fully proprietary managed Azure service; no source released. Scored closed.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints", "shows": "single managed Azure inference endpoint over model deployments; proprietary Azure resource, OpenAI-compatible API", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "Default inference surface for Microsoft Foundry across Azure's enterprise base; broad distribution but no published per-service usage number. Level 4 reflects Azure-enterprise scale (reported), not a measured count.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints", "shows": "unified endpoint for all Foundry model deployments, enterprise Azure integration", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "unified OpenAI-compatible endpoint; multi-provider model deployments (OpenAI, DeepSeek, MAI-DS-R1, etc.); keyless Entra ID auth; content filtering, rate limiting, batch inference", "confidence": "medium", "note": "Capability is managed-endpoint breadth (multi-model routing, auth, governance), not raw engine throughput. This is closer to a managed API gateway over model deployments than a weight-loading inference engine; see recategorize flag.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints", "shows": "single endpoint switches between model deployments with content filtering, rate limiting, keyless auth", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Microsoft Foundry Models unified inference endpoint (Azure AI Model Inference). Proprietary managed cloud service exposing many model deployments through one OpenAI-compatible endpoint; beta Azure AI Inference SDK deprecated, retiring Aug 26 2026 in favor of OpenAI/v1 API. Confirmed live (docs updated Apr 2026)." }, { "product": "OGX", "org": "Meta", "type": "software", "description": "Meta's open inference + deployment runtime for LLMs; originally launched as Llama Stack (`meta-llama/llama-stack`) in 2024 and rebranded in 2026 as OGX / Open GenAI Stack (`ogx-ai/ogx`) with model-agnostic positioning. Unified APIs for inference, agents, fine-tuning, and eval; 25+ ecosystem partners including AWS, NVIDIA, Databricks, Groq, and Azure. MIT licensed, 8.4K+ stars with continuous commit history across the rename. Anchors the 1B+ Llama downloads ecosystem.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core;implements OpenAI + Anthropic API surfaces; 23 pluggable inference providers", "confidence": "high", "note": "Fully open (MIT, full source). Openness scored straightforwardly; the issue is category fit, not openness.", "sources": [ { "url": "https://github.com/ogx-ai/ogx", "shows": "MIT license; v1.0.2 (May 13 2026); ~8.4k stars; agentic API server composing inference providers + vector stores + tools", "accessed": "2026-06-04" }, { "url": "https://ogx-ai.github.io/blog/from-llama-stack-to-ogx", "shows": "renamed from Llama Stack; model-agnostic, multi-SDK, routes to inference backends rather than serving weights itself", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Only ~8.4k GitHub stars and no published download/usage volume found this run; capped low on stars fallback (cannot exceed level 3). Recently renamed from Llama Stack, so historical signal is muddled.", "sources": [ { "url": "https://github.com/ogx-ai/ogx", "shows": "~8.4k stars; no download/usage figure published", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "OpenAI + Anthropic + Responses API surfaces; 23 inference providers; composes vector stores, safety backends, tool runtimes, file/batch storage into one agentic API server", "confidence": "medium", "note": "Capability is gateway/agentic-server breadth (multi-API-surface, provider composition), not inference throughput. As an inference engine it scores low because it does no forward pass itself; scored on its actual feature set in its true category. See recategorize.", "sources": [ { "url": "https://github.com/ogx-ai/ogx", "shows": "composes 23 inference providers + vector/safety/tool/file backends behind OpenAI/Anthropic-compatible API", "accessed": "2026-06-04" } ] }, "maturity": 2.3, "mature": false, "version_note": "OGX (Open GenAI Stack), formerly Meta's Llama Stack; renamed/rescoped to a model-agnostic, multi-SDK agentic API server. v1.0.2 (May 13, 2026), MIT-licensed, ~8.4k GitHub stars. NOT itself an inference engine: it composes/routes to inference backends (vLLM, Ollama, Bedrock) plus vector stores, safety, tools, file storage." }, { "product": "Open WebUI", "org": "Open WebUI", "type": "software", "description": "Self-hosted, extensible web UI for chatting with any OpenAI-compatible endpoint or local Ollama model, with built-in RAG, multimodal support, and a tools/plugins marketplace. Picked over alternatives for its breadth of features (knowledge bases, web search, document chat) and very active development; the most popular open chat frontend with ~138k GitHub stars and 280M+ container pulls by May 2026.", "openness": { "score": 3, "class": "source_available", "components": "license:Open-WebUI-License(BSD-3-Clause-based + non-OSI branding clause: branding may not be removed above 50 end-users/30d without enterprise license or written permission);source:public;enterprise-tier:commercial license to remove branding", "confidence": "high", "note": "Source-available, not OSI: a branding-retention clause caps free branding removal at <=50 users/30d, which fails the OSS Definition; widely marketed as open source.", "sources": [ { "url": "https://github.com/open-webui/open-webui/blob/main/LICENSE", "shows": "BSD-3-Clause-derived license with Section 4 branding-retention clause (<=50 users/30d), non-OSI", "accessed": "2026-06-04" }, { "url": "https://github.com/open-webui/open-webui", "shows": "v0.9.6 latest release, self-hosted AI interface", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.72M PyPI downloads last month (pypistats, primary); vendor reports 290M cumulative downloads and 420K community members. De facto leading self-hosted LLM chat UI; firmly in 1-10M band on monthly install volume (stars 140k corroborate only).", "sources": [ { "url": "https://pypistats.org/packages/open-webui", "shows": "1,717,371 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth (Ollama + any OpenAI-compatible API, multi-model conversations); RAG (9 vector DB options, 15+ web-search providers); multimodal (STT/TTS, image gen via DALL-E/Gemini/ComfyUI/A1111); multi-user (RBAC, groups, LDAP/SCIM/SSO); plugins (Pipelines framework)", "confidence": "high", "note": "Broadest feature surface among open self-hosted chat UIs: hits all five recipe dimensions (model breadth, RAG, multimodal, plugins, multi-user) with enterprise auth. Top-tier for the category.", "sources": [ { "url": "https://github.com/open-webui/open-webui", "shows": "feature list: RAG with 9 vector DBs, multimodal voice/image, RBAC/LDAP/SCIM/SSO, Pipelines plugins", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "v0.9.6 (June 1, 2026), confirmed live on GitHub June 2026. Self-hosted chat UI (Ollama + OpenAI-compatible APIs)." }, { "product": "LibreChat", "org": "LibreChat", "type": "software", "description": "Open-source ChatGPT clone with native support for OpenAI, Anthropic, Google, Mistral, and local model backends, plus multi-user auth, agents, code interpreter, and prompt sharing. Differentiator is enterprise-grade auth/RBAC and the closest faithful clone of ChatGPT's UX, making it popular for org-wide self-hosted deployments; ~37k GitHub stars by mid-2026.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core(RAG API is a separate companion OSS repo)", "confidence": "high", "note": "Fully OSI (MIT), full source public, no paid feature-gated core. Enterprise backing via ClickHouse acquisition has not introduced a closed core.", "sources": [ { "url": "https://github.com/danny-avila/LibreChat", "shows": "MIT license; full-featured self-hosted chat UI", "accessed": "2026-06-04" }, { "url": "https://clickhouse.com/blog/clickhouse-acquires-librechat", "shows": "ClickHouse acquires LibreChat (Nov 4, 2025); LibreChat remains open source/MIT, no closed core introduced, ongoing OSS investment", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "23M+ cumulative Docker pulls by early 2026 (includes CI/upgrades, so a soft upper bound); 38.1k GitHub stars, 6,800+ forks, 9,000+ Discord members; ClickHouse acquisition signals enterprise deployment. Placed at 4 on Docker-pull volume; medium confidence because pulls overcount unique deployments.", "sources": [ { "url": "https://github.com/danny-avila/LibreChat", "shows": "38.1k stars; MIT self-hosted chat platform with broad provider support", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "model breadth (Anthropic/OpenAI/Azure/Google/Bedrock/Ollama/Groq/Mistral/DeepSeek + 15 via custom endpoints); RAG (companion RAG API); multimodal (vision, file chat); multi-user (OAuth2/LDAP/email auth, moderation, token-spend tools); code interpreter (8 langs), MCP tools, agents+subagents, code artifacts (React/HTML/Mermaid), web search", "confidence": "high", "note": "Frontier feature surface for the category: code interpreter, MCP, agents/subagents and artifacts on top of the standard chat-UI matrix. At/above Open WebUI on breadth.", "sources": [ { "url": "https://github.com/danny-avila/LibreChat", "shows": "feature list: multi-provider, code interpreter, MCP, agents/subagents, artifacts, OAuth2/LDAP", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "Active 2026 (changelog/releases current); MIT-licensed self-hosted multi-provider chat UI. Acquired by ClickHouse in 2025. Confirmed live on GitHub June 2026." }, { "product": "Lobe Chat", "org": "LobeHub", "type": "software", "description": "Modern-design open source chat framework supporting OpenAI, Claude, Gemini, Ollama, Azure, DeepSeek and others, with knowledge bases, vision/TTS, plugin marketplace, and one-click deployment. Chosen for its polished UI and breadth of multi-modal/plugin support compared to barebones alternatives; ~77k GitHub stars makes it the second-most-starred OSS chat UI.", "openness": { "score": 3, "class": "source_available", "components": "license:LobeHub-Community-License(Apache-2.0-based, non-OSI: commercial license required to develop+distribute a derivative work; CLA lets LobeHub relicense contributions for cloud editions);source:public;cloud-tier:LobeHub Cloud", "confidence": "high", "note": "Source-available, not OSI: the derivative-work commercial-license requirement and contributor relicensing terms exceed Apache-2.0; still presented publicly as open source.", "sources": [ { "url": "https://github.com/lobehub/lobe-chat/blob/main/LICENSE", "shows": "LobeHub Community License (Apache-2.0 based) requiring a commercial license to distribute derivative works; non-OSI", "accessed": "2026-06-04" }, { "url": "https://github.com/lobehub/lobehub", "shows": "v2.2.2 latest; 78.2k stars; self-host + LobeHub Cloud tiers", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "78.2k GitHub stars, 15.4k forks; Docker image (lobehub/lobehub) and LobeHub Cloud exist but no verified pull/MAU figure was obtainable from a primary source this run. Capped at 3 on stars fallback per rubric.", "sources": [ { "url": "https://github.com/lobehub/lobehub", "shows": "78.2k stars, 15.4k forks; Docker + Cloud deployment", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth (multi-model/multi-provider 'Unified Intelligence'); plugins (10,000+ skills, MCP-compatible); multi-user/agent-groups for team collaboration; knowledge base/RAG; multimodal; agent memory + scheduling", "confidence": "medium", "note": "Strong feature surface across model breadth, MCP plugins, knowledge base and multi-user. Below Open WebUI/LibreChat on enterprise auth/RAG depth verifiability; agent-orchestration framing edges toward orchestration_agents but core remains human-driven chat.", "sources": [ { "url": "https://github.com/lobehub/lobehub", "shows": "feature list: multi-model, 10,000+ MCP skills, agent groups, knowledge base, memory", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "v2.2.2 (June 4, 2026); repo rebranded lobehub/lobehub. License changed from MIT to a non-OSI 'LobeHub Community License' (Apache-2.0 based + derivative-distribution commercial restriction). Product now markets agent-team orchestration but remains a human-driven chat UI at core." }, { "product": "AnythingLLM", "org": "Mintplex Labs", "type": "software", "description": "All-in-one desktop and Docker app for private, on-device chat with documents (RAG), agents, and 30+ LLM providers including local Ollama. Differentiator is zero-config desktop setup with bundled embedder and vector DB, plus team/Docker mode with RBAC and embeddable widgets; ~60k GitHub stars.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core;optional hosted instance tier (managed convenience, not a gated core)", "confidence": "high", "note": "Fully OSI (MIT), full source public; a hosted instance is offered but the self-hosted Desktop/Docker app is the complete product.", "sources": [ { "url": "https://github.com/Mintplex-Labs/anything-llm", "shows": "MIT license; all-in-one local-first RAG/agent app, v1.13.0", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Docker Hub mintplexlabs/anythingllm shows 1M+ total pulls and ~46k pulls/week (primary); 53-61k GitHub stars; Desktop distributed standalone. Placed at 4 on cumulative pull volume; medium confidence as exact all-time count isn't published and Desktop installs aren't separately counted.", "sources": [ { "url": "https://hub.docker.com/r/mintplexlabs/anythingllm", "shows": "1M+ total Docker pulls; ~45,859 pulls this week", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth (40+ LLM providers, multiple vector DBs, dynamic model routing); RAG (document ingestion core use case); multimodal; multi-user (instance with permissioning); agents (no-code builder, web browsing); recent meeting-recording/summarization features", "confidence": "high", "note": "Strong full-stack chat+RAG+agent surface hitting all category dimensions; provider/vector-DB breadth and no-code agents put it on par with LibreChat/Open WebUI. Reserved 5 for the OSI leaders on raw breadth.", "sources": [ { "url": "https://github.com/Mintplex-Labs/anything-llm", "shows": "40+ LLM providers, RAG, multi-user permissioning, no-code agents with web browsing", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "v1.13.0 (May 26, 2026); MIT-licensed all-in-one local-first Desktop + Docker AI app (document chat, RAG, agents). Confirmed live on GitHub June 2026." }, { "product": "NextChat", "org": "NextChat", "type": "software", "description": "Lightweight, cross-platform ChatGPT/Gemini/Claude web client that runs as a single Next.js app deployable to Vercel in one click. Picked for its minimalism and easy personal deployment when full-fat frontends like Open WebUI are overkill; ~88k GitHub stars makes it the single most-starred OSS chat UI repo by raw count.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core, public source);enterprise-tier:NextChat Enterprise (permission controls, team resource mgmt, internal knowledge base/RAG)", "confidence": "high", "note": "OSI MIT core with a separate proprietary Enterprise Edition for multi-user permissions and internal-KB RAG -> open_core (calibrated to the LiteLLM open_core anchor).", "sources": [ { "url": "https://github.com/ChatGPTNextWeb/NextChat", "shows": "MIT license; cross-platform chat client; Enterprise Edition (permissions, team mgmt, internal knowledge base)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "88.2k GitHub stars and one-click Vercel deploy indicate broad lightweight use, but no verified download/MAU figure from a primary source this run, and last release ~6-12mo ago. Capped at 3 on stars fallback; momentum appears to have slowed.", "sources": [ { "url": "https://github.com/ChatGPTNextWeb/NextChat", "shows": "88.2k stars; v2.16.1 latest release; cross-platform deployment", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth (Claude/DeepSeek/GPT-4/Gemini/Azure/Baidu/ByteDance/Alibaba etc.); plugins (v2.15.0+); multimodal (vision); MCP support; artifacts; realtime chat; multi-user + RAG only in Enterprise Edition", "confidence": "medium", "note": "Solid lightweight cross-platform client with plugins/MCP/vision, but RAG and multi-user are gated to Enterprise; thinner OSS feature surface than Open WebUI/LibreChat/AnythingLLM.", "sources": [ { "url": "https://github.com/ChatGPTNextWeb/NextChat", "shows": "multi-provider, plugins, vision, MCP, artifacts; RAG/multi-user in Enterprise Edition", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "v2.16.1 latest release (~mid-2025; release notes cite OpenAI o3/o4-mini + Claude 3.7 support); MIT core. ~6-12mo since last release as of June 2026 -> freshness flag. Cross-platform (web/desktop/mobile) chat client; Enterprise Edition adds permissions/RAG." }, { "product": "HuggingChat / chat-ui", "org": "Hugging Face", "type": "software", "description": "Open-source chat UI that powers HuggingChat (huggingface.co/chat), giving free public access to open-weights models like Llama, Mistral, Command R+ and others. Differentiator is being the reference SvelteKit chat UI shipped and maintained by Hugging Face itself, with first-class support for TGI/transformers endpoints; ~11k GitHub stars but powers a high-traffic free consumer endpoint.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no-feature-gated-core(self-hostable SvelteKit app, MongoDB backend)", "confidence": "high", "note": "Fully OSI (Apache-2.0); the public codebase is exactly what powers the hosted HuggingChat, self-hostable end to end.", "sources": [ { "url": "https://github.com/huggingface/chat-ui", "shows": "Apache-2.0; 'open source codebase powering HuggingChat'; v0.10.0", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Hosted HuggingChat served 1M+ users and 100k+ assistants over its lifetime (HF announcement) before the July 2025 sunset; relaunched as HuggingChat Omni and live again on hf.co/chat. chat-ui repo itself is a modest 10.7k stars (self-host base is niche). Placed at 3 on the disclosed lifetime user reach of the hosted product; the discontinuity + relaunch lowers confidence.", "sources": [ { "url": "https://huggingface.co/chat", "shows": "live HuggingChat Omni interface, Omni router, 123+ open source models, MCP", "accessed": "2026-06-04" }, { "url": "https://www.startuphub.ai/ai-news/startup-news/2025/hugging-face-closes-huggingchat-to-make-way-for-next-generation-ai-tools", "shows": "HuggingChat served 1M+ users / 100k+ assistants before July 2025 sunset", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth (115-123 open models via 15 inference providers, Omni smart-router on Arch-Router-1.5B); multimodal input; MCP tools; OpenAI-compatible API; multi-user (hosted). Current branch removed legacy provider-specific integrations and GGUF discovery", "confidence": "medium", "note": "Good model breadth + smart routing + MCP, but the rebuilt v0.10 deliberately stripped RAG/provider integrations present in the legacy branch, so the self-host feature surface is leaner than Open WebUI/LibreChat. Mid-tier for the category.", "sources": [ { "url": "https://github.com/huggingface/chat-ui", "shows": "OpenAI-compatible API, MCP tools, multimodal, Omni routing; legacy provider integrations + GGUF discovery removed", "accessed": "2026-06-04" }, { "url": "https://huggingface.co/chat", "shows": "Omni router selecting among 123+ open models, MCP enabled", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "chat-ui v0.10.0 (May 11, 2026), Apache-2.0, powers HuggingChat at hf.co/chat. Original HuggingChat was sunset July 1, 2025, then relaunched as 'HuggingChat Omni' (Omni Router + 115-123 open models, 15 inference providers, MCP). Confirmed live June 2026." }, { "product": "Jan", "org": "Jan", "type": "software", "description": "Cross-platform desktop app (Windows/macOS/Linux) for running open source models 100% offline, with a clean ChatGPT-style UI, optional cloud provider integrations, and a local OpenAI-compatible API server on localhost:1337. Picked over alternatives for its polished native desktop experience and offline-first stance; 5.3M+ downloads and ~43k GitHub stars by May 2026.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI);source:public(github.com/janhq/jan);no-feature-gated-core;self-host:primary(desktop app, 100% offline-capable)", "note": "Fully OSI-licensed (Apache-2.0), full source public, no proprietary feature-gated core, a true open chat UI.", "sources": [ { "url": "https://github.com/janhq/jan/blob/dev/LICENSE", "shows": "Apache-2.0 license text", "accessed": "2026-06-04" }, { "url": "https://github.com/janhq/jan", "shows": "public source; Apache-2.0; v0.8.2 (1 Jun 2026)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Vendor site reports 5.7M+ cumulative downloads of the desktop app; 42.9k GitHub stars corroborates but is not the basis. Cumulative-download volume places it in the 1-10M band, though these are lifetime installs not active users.", "sources": [ { "url": "https://jan.ai/", "shows": "5.7M+ downloads stated on vendor homepage", "accessed": "2026-06-04" }, { "url": "https://github.com/janhq/jan", "shows": "42.9k stars (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth: yes (local GGUF + OpenAI/Anthropic/Google/Mistral/DeepSeek/Qwen/Kimi cloud providers); multimodal: limited; RAG: limited; plugins/extensions: yes; multi-user: no (single-user desktop); OpenAI-compatible local API: yes", "confidence": "medium", "note": "Solid open desktop chat UI with broad model selection and a local OpenAI-compatible server, but single-user and lighter on RAG/multimodal than the frontier consumer apps in this category. Mid-tier on the chat-UI feature matrix.", "sources": [ { "url": "https://jan.ai/", "shows": "open + cloud model providers, OpenAI-compatible API, local-first", "accessed": "2026-06-04" }, { "url": "https://github.com/janhq/jan", "shows": "feature set: local LLMs, cloud integration, custom assistants, OpenAI-compatible API", "accessed": "2026-06-04" } ] }, "maturity": 3.7, "mature": false, "version_note": "Jan v0.8.2 (released 1 Jun 2026). Open-source local-first ChatGPT alternative (desktop chat UI) by Menlo Research / janhq; runs local GGUF models plus pluggable cloud providers, ships an OpenAI-compatible local API server. Verified live on GitHub + jan.ai June 2026." }, { "product": "Continue", "org": "Continue", "type": "software", "description": "Open-source AI code assistant for VS Code and JetBrains IDEs offering inline completions, chat, and edit modes against any model (OpenAI, Anthropic, Gemini, Ollama, vLLM). Differentiates from Copilot by being model-agnostic and self-hostable, and from Cursor by living inside existing IDEs; YC-backed with ~$5M seed and ~33k GitHub stars.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:Apache-2.0(OSI);source:public(github.com/continuedev/continue);clients:VSCode+JetBrains+CLI;hub:continue.dev(hosted config/model hub adjacent, core OSS)", "note": "Apache-2.0, full source public for the IDE extensions and CLI; no feature-gated core in the editor itself (a hosted Hub exists alongside but the assistant is OSS).", "sources": [ { "url": "https://github.com/continuedev/continue", "shows": "Apache-2.0; public source; AI code assistant (chat/edit/autocomplete + agent)", "accessed": "2026-06-04" }, { "url": "https://marketplace.visualstudio.com/items?itemName=Continue.continue", "shows": "open source AI code agent extension listing", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "VS Code Marketplace shows ~3.21M installs of the Continue extension; plus JetBrains plugin and CLI installs on top. Install volume places it solidly in the 1-10M band. 33.5k GitHub stars corroborates only.", "sources": [ { "url": "https://marketplace.visualstudio.com/items?itemName=Continue.continue", "shows": "3,209,395 installs", "accessed": "2026-06-04" }, { "url": "https://github.com/continuedev/continue", "shows": "33.5k stars (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth: yes (model-agnostic, any provider/local); chat: yes; inline edit: yes; autocomplete/tab: yes; agent mode: yes; RAG over codebase: yes; multimodal: partial; multi-IDE: VSCode+JetBrains+CLI", "confidence": "medium", "note": "Leading open source in-IDE AI assistant; broad model-agnostic coverage with chat/edit/autocomplete/agent and codebase RAG. Strong feature breadth for an open editor assistant, just below the polished closed frontier (Copilot/Cursor).", "sources": [ { "url": "https://marketplace.visualstudio.com/items?itemName=Continue.continue", "shows": "agent, chat, edit, autocomplete features", "accessed": "2026-06-04" }, { "url": "https://github.com/continuedev/continue", "shows": "model-agnostic AI code assistant feature set", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Continue (continuedev/continue); VS Code extension v1.2.22 (Mar 2026), JetBrains plugin, and open source `cn` CLI. Open-source AI code assistant, chat + inline edit + autocomplete in the IDE, now also marketing 'AI code agent' (agent mode) and CI checks. Verified live on GitHub + VS Marketplace June 2026." }, { "product": "SillyTavern", "org": "SillyTavern", "type": "software", "description": "Self-hosted chat frontend originally forked from TavernAI, focused on character/roleplay use cases with deep support for prompt engineering, lorebooks, group chats, and any OpenAI-compatible or local backend. Dominant frontend in the open roleplay/character-chat community (~28k GitHub stars) and the de facto UI for users moving off Character.ai to local models.", "openness": { "score": 5, "class": "open_source", "confidence": "high", "components": "license:AGPL-3.0(OSI, copyleft);source:public(github.com/SillyTavern/SillyTavern);self-host:only(local install);no-managed-tier;extensions:third-party", "note": "AGPL-3.0 (OSI-approved copyleft), full source public, self-host only, fully open. No proprietary/SaaS tier.", "sources": [ { "url": "https://github.com/SillyTavern/SillyTavern", "shows": "AGPL-3.0; public source; v1.18.0 (3 May 2026); local UI for LLM/image/TTS backends", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No published download/install or active-user counts found (self-hosted desktop app, no telemetry). Falling back to 28.9k GitHub stars + 300+ contributors as the only honest signal, capped at level 3 per the stars-fallback rule. Likely a large enthusiast/roleplay community but unverified.", "sources": [ { "url": "https://github.com/SillyTavern/SillyTavern", "shows": "28.9k stars, 300+ contributors; no install/usage count published", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth: yes (many LLM APIs: KoboldAI/OpenAI/Claude/Mistral/local); multimodal: yes (image-gen + TTS integration); RAG/lorebooks: yes; plugins/extensions: yes (third-party); multi-user: no (single-user local)", "confidence": "medium", "note": "Feature-rich for its niche, broad backend coverage, image-gen/TTS, extensions, character/roleplay tooling (Visual Novel mode). Specialized toward roleplay rather than general-purpose assistant work; mid-tier on the general chat-UI matrix.", "sources": [ { "url": "https://github.com/SillyTavern/SillyTavern", "shows": "multi-backend, image-gen, TTS, extensions, Visual Novel mode", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "SillyTavern v1.18.0 (released 3 May 2026). Self-hosted local chat front-end (NodeJS) for text-gen LLMs, originally a TavernAI fork (Feb 2023); connects to many backends (KoboldAI, OpenAI, Claude, Mistral, etc.) with character/roleplay focus, image-gen + TTS integration, extensions. Verified live on GitHub June 2026." }, { "product": "Thunderbolt", "org": "MZLA Technologies", "type": "software", "description": "Open-source, self-hosted AI client from MZLA Technologies (the Mozilla Foundation subsidiary behind Thunderbird), positioned as a sovereign alternative to Microsoft Copilot, ChatGPT Enterprise, and Claude Enterprise. Provides chat, search, research, and task automation across web and native apps (Linux, Windows, macOS, iOS, Android), integrating Anthropic, OpenAI, Mistral, OpenRouter, and local Ollama models with no vendor lock-in. The differentiator is data control - organizations run their own infrastructure rather than renting access.", "openness": { "score": 5, "class": "open_source", "components": "license:MPL-2.0(OSI);source:public(github.com/thunderbird/thunderbolt);self-host:primary(enterprise self-hosting);governance:MZLA-Technologies(Mozilla-Foundation-subsidiary)", "confidence": "high", "note": "OSI-licensed (MPL-2.0), full source public from launch, designed primarily for self-hosting - a true open AI client.", "sources": [ { "url": "https://github.com/thunderbird/thunderbolt", "shows": "public source repo under MPL-2.0; \"AI You Control\" self-hosted client", "accessed": "2026-06-17" }, { "url": "https://www.phoronix.com/news/Mozilla-Thunderbolt", "shows": "MZLA launches Thunderbolt as an open source enterprise AI client (Apr 2026)", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 2, "signal_type": "reported_traction", "confidence": "low", "note": "Launched April 2026 and still in active development / mid-security-audit; no usage figures disclosed yet. Early-stage but backed by the Mozilla/Thunderbird brand. Conservative reported_traction pending real adoption signal.", "sources": [ { "url": "https://www.theregister.com/2026/04/16/mozilla_thunderbolt_enterprise_ai_client/", "shows": "launch coverage; positioned vs Copilot/ChatGPT Enterprise/Claude Enterprise", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth: Anthropic/OpenAI/Mistral/OpenRouter + local Ollama; surfaces: chat, search, research, task automation; platforms: web + native Linux/Windows/macOS/iOS/Android; self-host: primary; multi-user: enterprise-oriented", "confidence": "medium", "note": "Broad feature surface (multi-provider, multi-platform, agentic task automation, self-host) comparable to the leading open chat UIs, but newly launched and unproven; medium confidence pending maturity.", "sources": [ { "url": "https://www.ghacks.net/2026/04/20/mozillas-mzla-technologies-launches-thunderbolt-an-open-source-self-hosted-enterprise-ai-client/", "shows": "feature set: chat/search/research/task automation across web + native apps, multi-provider + Ollama", "accessed": "2026-06-17" } ] }, "maturity": 2.6, "mature": false, "version_note": "Launched April 2026 under MPL-2.0; in active development and mid-security-audit as of mid-2026. Self-hosted multi-provider enterprise AI client." }, { "product": "Morphic", "org": "Morphic", "type": "software", "description": "Open-source AI-powered answer engine with a generative UI - answers render rich, source-credited inline components (images, grids, headings) live from a streamed JSON spec rather than plain markdown. Supports multiple model providers (OpenAI, Anthropic, Google, Ollama) and search backends (Tavily, SearXNG, Brave, Exa), with chat history, shareable result URLs, file upload, and guest mode. The differentiator is the generative-UI answer surface; it is Docker-deployable for self-hosting.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(github.com/miurla/morphic);self-host:primary(Docker);no-feature-gated-core", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0), full source public, Docker-deployable for self-hosting - a true open generative-search UI.", "sources": [ { "url": "https://github.com/miurla/morphic", "shows": "Apache-2.0 license; ~8.9k stars; open generative-UI answer engine", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~8.9k GitHub stars by mid-2026; no download/usage telemetry available for a self-hosted web app, so stars_fallback (capped at level 3). Popular as a reference open answer-engine template (also a Vercel template).", "sources": [ { "url": "https://github.com/miurla/morphic", "shows": "8.9k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth: OpenAI/Anthropic/Google/Ollama; search backends: Tavily/SearXNG/Brave/Exa; generative UI: rich inline components from streamed JSON; chat history + shareable URLs; file upload; guest mode; multi-user: limited", "confidence": "medium", "note": "Distinctive generative-UI answer surface with multi-provider/multi-search support, but scoped to search-and-answer rather than the full chat-UI feature set (lighter on multimodal, plugins, enterprise auth). Mid-tier for the category.", "sources": [ { "url": "https://github.com/miurla/morphic", "shows": "feature set: generative UI, multi-provider, multi-search, history, sharing, file upload, guest mode", "accessed": "2026-06-17" } ] }, "maturity": 3.0, "mature": false, "version_note": "Apache-2.0; ~8.9k GitHub stars by mid-2026. Built on Next.js + the Vercel AI SDK. Self-hostable open generative-search UI." }, { "product": "Perplexica", "org": "ItzCrazyKns", "type": "software", "description": "Open-source, privacy-focused AI answer engine - a self-hostable alternative to Perplexity. It pairs a bundled SearXNG metasearch layer with local or cloud LLMs to return AI-generated answers with cited sources, all runnable on your own machine so searches stay local. Supports multiple model providers (Ollama, OpenAI, Anthropic, Gemini, Groq, any OpenAI-compatible endpoint), web/academic/discussion/image/video search modes, speed/balanced/quality tiers, and file-upload RAG. The differentiator is full self-hosting and privacy for the Perplexity-style answer-with-sources experience.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(github.com/ItzCrazyKns/Vane);self-host:primary(single bundled Docker image);no-feature-gated-core", "confidence": "high", "note": "Fully OSI-licensed (MIT), complete source public, self-hosting is the primary distribution via a bundled Docker image - a true open answer engine. No premium tier or gated functionality identified.", "sources": [ { "url": "https://github.com/ItzCrazyKns/Vane", "shows": "MIT license; ~35k stars; open Perplexity-alternative answer engine (formerly Perplexica)", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K+ Docker pulls", "signal_type": "usage_volume", "confidence": "medium", "note": "~35k GitHub stars plus 100k+ Docker pulls on the current itzcrazykns1337/vane image by mid-2026 - genuine install volume (usage_volume, not stars-only), but well below the 1M+ band, so mid-tier.", "sources": [ { "url": "https://hub.docker.com/r/itzcrazykns1337/vane", "shows": "100K+ Docker pulls for the self-host image", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth: Ollama/OpenAI/Anthropic/Gemini/Groq + any OpenAI-compatible API; search: bundled SearXNG metasearch with web/academic/discussion/image/video modes + speed/balanced/quality tiers; RAG: file upload (PDF/text/image); multimodal: image inputs; multi-user: none (single-user personal app)", "confidence": "medium", "note": "Strong provider breadth, bundled metasearch, search modes, and file-upload RAG; loses a point for no built-in multi-user/auth and limited multimodal generation. Scoped to search-and-answer.", "sources": [ { "url": "https://github.com/ItzCrazyKns/Vane", "shows": "feature set: multi-provider, bundled SearXNG, search modes, file-upload RAG", "accessed": "2026-06-17" } ] }, "maturity": 3.3, "mature": false, "version_note": "MIT-licensed; ~35k GitHub stars and 100k+ Docker pulls by mid-2026. NOTE the project was renamed Perplexica -> Vane (~Mar 2026) to avoid name proximity to Perplexity AI; the github.com/ItzCrazyKns/Perplexica URL now redirects to .../Vane. Still widely known and listed as \"Perplexica\". Self-host via a single bundled Docker image (Next.js frontend + API + private SearXNG)." }, { "product": "Khoj", "org": "Khoj AI", "type": "software", "description": "Open-source, self-hostable \"AI second brain\" / personal assistant. Chat with any cloud or local LLM, get answers grounded in your own documents (PDF, Markdown, org-mode, Word, Notion) and the web, run semantic search, build custom agents with their own knowledge bases, schedule automations, and do deep research. The differentiator is privacy-first personal RAG surfaced across many clients - browser, Obsidian, Emacs, desktop, mobile, and WhatsApp - with everything runnable on your own infrastructure.", "openness": { "score": 5, "class": "open_source", "components": "license:AGPL-3.0(OSI);source:public(github.com/khoj-ai/khoj);self-host:primary(Docker+pip);cloud-tier-sunset-2026-04(no remaining feature-gating)", "confidence": "high", "note": "OSI-licensed (AGPL-3.0), full source public, self-hostable via Docker or pip. The paid Khoj Cloud tier was discontinued 2026-04-15 and the team frames self-hosting as the complete replacement, so the earlier open-core characteristics no longer apply to the shipping product.", "sources": [ { "url": "https://github.com/khoj-ai/khoj", "shows": "AGPL-3.0 license; ~35k stars; self-hostable AI personal assistant", "accessed": "2026-06-17" }, { "url": "https://app.khoj.dev/", "shows": "official notice that Khoj Cloud was sunset 2026-04-15 and Khoj \"remains fully open source\"", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "~26k PyPI downloads/month", "signal_type": "usage_volume", "confidence": "medium", "note": "~35k GitHub stars plus ~26k PyPI downloads/month (pypistats) - real install volume (usage_volume, not stars-only), but modest in absolute terms, so mid-tier. Hosted cloud is now discontinued.", "sources": [ { "url": "https://pypistats.org/packages/khoj", "shows": "~26,054 downloads last month", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth: OpenAI/Anthropic/Gemini/Llama/Qwen/Mistral/DeepSeek + local Ollama; RAG over personal docs (PDF/Markdown/org-mode/Word/Notion/images); multimodal: image gen + voice/TTS; agents with custom knowledge bases; scheduled automations + deep research; multi-user server; clients: browser/Obsidian/Emacs/desktop/mobile/WhatsApp", "confidence": "medium", "note": "Broad feature surface across providers, personal RAG, agents, automations, multimodality and many client integrations - on par with the leading open self-hosted assistants. Top-tier for the category.", "sources": [ { "url": "https://github.com/khoj-ai/khoj", "shows": "feature set: multi-provider, personal-doc RAG, agents, automations, multimodal, multi-client", "accessed": "2026-06-17" } ] }, "maturity": 3.3, "mature": false, "version_note": "AGPL-3.0; ~35k GitHub stars and ~26k PyPI downloads/month by mid-2026. Khoj Cloud (the former paid hosted tier) was sunset 2026-04-15; the team directs all users to self-host and states Khoj \"remains fully open source\", so no feature-gating remains in the shipping product. Self-host via Docker or pip." }, { "product": "SurfSense", "org": "SurfSense", "type": "software", "description": "Open-source, privacy-focused AI research assistant positioned as a self-hostable alternative to NotebookLM, Perplexity, and Glean. It connects an LLM to your personal or team knowledge base plus a wide range of external sources and lets you chat over them with hybrid RAG retrieval. The differentiator is unlimited sources and notebooks with no data caps, full freedom to bring your own LLM/embedding/reranker models (including local ones), and a deep connector catalog - all self-hostable so data never leaves your infrastructure.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(github.com/MODSetter/SurfSense);self-host:primary(Docker/Compose);no-feature-gated-core-found", "confidence": "high", "note": "Permissive OSI license (Apache-2.0), complete source public, self-host-first via Docker/Compose. No paid tier or gated features found on the repo or site (mild uncertainty - treated as fully open until a commercial tier appears).", "sources": [ { "url": "https://github.com/MODSetter/SurfSense/blob/main/LICENSE", "shows": "Apache License 2.0", "accessed": "2026-06-17" }, { "url": "https://github.com/MODSetter/SurfSense", "shows": "~15k stars; open privacy-focused NotebookLM/Perplexity alternative; Docker self-host", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~15k GitHub stars by mid-2026; no download/usage telemetry available for a self-hosted web app, so stars_fallback (capped at level 3). Rapid star growth but unproven real-world deployment.", "sources": [ { "url": "https://github.com/MODSetter/SurfSense", "shows": "~14.9k stars, ~1.4k forks", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "model breadth: 100+ LLMs via LiteLLM/OpenAI spec + local (Ollama/vLLM), 6000+ embedding models, rerankers; connectors: 27+ (SearXNG/Tavily search, Drive/OneDrive/Dropbox, Slack/Teams/Discord, Notion/Confluence/Obsidian, Linear/Jira, Gmail, YouTube, GitHub); RAG: hybrid semantic + full-text with rank fusion; multimodal: podcast/video/image generation; multi-user: real-time collab + RBAC", "confidence": "low", "note": "Unusually broad connector, model, and collaboration surface on paper, but per-feature depth and quality are unverified (README-sourced) and the project is far less proven than open-webui-class tools - so 4, not top, with low confidence.", "sources": [ { "url": "https://github.com/MODSetter/SurfSense", "shows": "feature/connector matrix: 100+ LLMs, 27+ connectors, hybrid RAG, multimodal gen, RBAC collab", "accessed": "2026-06-17" } ] }, "maturity": 3.3, "mature": false, "version_note": "Apache-2.0; ~15k GitHub stars by mid-2026. Self-host via Docker/Compose (PostgreSQL + pgvector). Connector count and per-feature depth are taken from the maintainer-authored README and not independently verified. Newer/smaller project than the leading open chat UIs." }, { "product": "ChatGPT", "org": "OpenAI", "type": "software", "description": "OpenAI's flagship consumer chat product, web, desktop, and mobile apps for conversing with GPT-5/o-series models, with built-in tools (search, code interpreter, image gen, voice, memory). The category-defining UI by a wide margin: 900M weekly active users and 50M paying subscribers as of Feb 2026, on a $25B annualized run rate.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "source:closed;self-host:none;license:proprietary(SaaS app + API)", "note": "Proprietary, SaaS-only consumer app; no source, no self-host. Closed counterpart in the ui_api shelf.", "sources": [ { "url": "https://openai.com/index/how-people-are-using-chatgpt/", "shows": "OpenAI's proprietary ChatGPT product page/usage report", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "OpenAI officially announced ~900M weekly active users (27 Feb 2026), up from 800M (Oct 2025), with 50M paying subscribers. Mass-market scale, far above the >10M threshold. Attributed to the ChatGPT surface.", "sources": [ { "url": "https://techcrunch.com/2026/02/27/chatgpt-reaches-900m-weekly-active-users/", "shows": "OpenAI official announcement of 900M weekly active users, Feb 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "model breadth: frontier (GPT-5 family + variants); RAG/file upload: yes; multimodal: yes (vision, voice, image-gen); plugins/tools: yes (tools, code interpreter, web search, connectors); multi-user: yes (Team/Enterprise); memory + custom GPTs", "confidence": "high", "note": "Frontier consumer chat product within ui_api: full multimodal, code execution, web search, custom GPTs, memory, Team/Enterprise multi-user. Sets the bar for the category's chat-UI feature matrix.", "sources": [ { "url": "https://openai.com/index/how-people-are-using-chatgpt/", "shows": "ChatGPT capabilities/usage report", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "ChatGPT (OpenAI) consumer chat app, web + iOS/Android + desktop. Proprietary/closed UI scored as the closed counterpart in this category. Live and verified June 2026; default model GPT-5 family. (The model SKU is scored separately under base/finetuned categories; this record scores the chat surface.)" }, { "product": "GitHub Copilot", "org": "GitHub (Microsoft)", "type": "software", "description": "AI pair-programmer integrated into VS Code, JetBrains, Neovim, and Visual Studio, offering inline code completion, chat, and code review against multiple model backends (OpenAI, Anthropic, Google). Differentiator is enterprise distribution via GitHub and tight IDE integration; ~4.7M paid subscribers and ~$1B ARR as of early 2026.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "source:closed;self-host:none;license:proprietary(subscription SaaS)", "note": "Proprietary, closed-source subscription product; no self-host. Closed counterpart in the ui_api shelf.", "sources": [ { "url": "https://github.com/features/copilot", "shows": "proprietary GitHub Copilot product page; closed source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "Microsoft CEO confirmed 20M all-time Copilot users (earnings call, 30 Jul 2025); free tier opened to 150M GitHub developers (Dec 2024); ~4.7M paid subscribers by Jan 2026. All-time user base exceeds the >10M mass-market threshold. (All-time vs active distinction noted; still clearly level 5.)", "sources": [ { "url": "https://techcrunch.com/2025/07/30/github-copilot-crosses-20-million-all-time-users/", "shows": "GitHub-confirmed 20M all-time users, announced by Satya Nadella Jul 2025", "accessed": "2026-06-04" }, { "url": "https://github.com/features/copilot", "shows": "'millions of individual users and tens of thousands of business customers'", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "model breadth: yes (multi-model: GPT/Claude/Gemini selectable); chat: yes; inline/autocomplete: yes; agent mode: yes; codebase context/RAG: yes; multimodal: partial; multi-IDE + GitHub-native; enterprise multi-user: yes", "confidence": "high", "note": "Most widely adopted, feature-complete AI developer assistant: chat + inline + agent mode + multi-model selection, deeply IDE/GitHub-integrated, enterprise controls. Frontier of the in-IDE-assistant slice of ui_api.", "sources": [ { "url": "https://github.com/features/copilot", "shows": "inline suggestions, chat, agent, code explanations; 'world's most widely adopted AI developer tool'", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "GitHub Copilot (GitHub / Microsoft), proprietary in-IDE AI coding assistant (inline suggestions, chat, agent mode) across VS Code/JetBrains/etc. Closed counterpart in ui_api. Verified live on github.com/features/copilot June 2026." }, { "product": "Claude.ai", "org": "Anthropic", "type": "software", "description": "Anthropic's consumer chat product (web, desktop, mobile) for interacting with Claude Sonnet/Opus models, with Projects, Artifacts (live-rendered code/docs), and Computer Use. ~30M monthly active users and adding 1M+ daily signups as of March 2026, third-largest US mobile chatbot.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "source:closed;self-host:none;license:proprietary(SaaS app + API)", "note": "Proprietary, SaaS-only consumer app; no source, no self-host. Closed counterpart in the ui_api shelf.", "sources": [ { "url": "https://claude.com/product/overview", "shows": "Anthropic's proprietary Claude consumer product page (artifacts, projects, web search, code execution)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "active_users", "confidence": "low", "note": "Anthropic does NOT disclose official Claude.ai MAU. Third-party estimates (no primary vendor figure available) put consumer MAU ~18.9M web at start of 2026 rising toward ~30M by mid-2026, i.e. comfortably above 1M but well below the >10M-billion scale of ChatGPT/Gemini app. Placed at level 4 on best-available estimate; confidence low because the headline number is an estimate, not a vendor primary disclosure (flagged).", "sources": [ { "url": "https://claude.com/product/overview", "shows": "consumer chat product exists (web/desktop/mobile apps)", "accessed": "2026-06-04" }, { "url": "https://www.demandsage.com/claude-ai-statistics/", "shows": "third-party estimate ~18.9M web MAU early 2026 (corroborating, not primary)", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "model breadth: frontier (Opus/Sonnet/Haiku); RAG/file upload: yes (PDF/Word/Excel/image); multimodal: yes (vision, voice mode); tools/plugins: yes (MCP connectors, web search, code execution, Google Workspace); artifacts + projects; multi-user: yes (Team/Enterprise)", "confidence": "high", "note": "Frontier consumer chat product within ui_api: Artifacts, Projects, web search, code execution, MCP connectors, voice, multimodal file analysis, Team/Enterprise. Among the category's most capable chat surfaces.", "sources": [ { "url": "https://claude.com/product/overview", "shows": "Artifacts, Projects, Web Search, Voice, Code Execution, integrations, multimodal upload", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "Claude.ai (Anthropic) consumer chat app, web + macOS/Windows + iOS/Android. Proprietary/closed UI scored as the closed counterpart. Verified live June 2026 (product page now at claude.com/product/overview, redirected from anthropic.com/claude). Uses Claude Opus/Sonnet/Haiku." }, { "product": "Gemini", "org": "Google", "type": "software", "description": "Google's consumer Gemini chat app on web, Android, iOS, and integrated across Google Workspace, with Live (voice), Deep Research, and image/video generation. 900M+ monthly active users by Q1 2026 (up from 750M at end of 2025), fastest-growing standalone LLM in absolute terms.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "source:closed;self-host:none;license:proprietary(SaaS app)", "note": "Proprietary, SaaS-only consumer app; no source, no self-host. Closed counterpart in the ui_api shelf.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "Google's proprietary Gemini app/model product page", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "Google CEO Sundar Pichai officially stated the Gemini app surpassed 750M monthly active users (Q4 2025 earnings, 4 Feb 2026), up from 450M earlier in 2025. Mass-market scale, far above >10M (broader AI Overviews surface reaches ~2B, but the app itself is the cited figure).", "sources": [ { "url": "https://techcrunch.com/2026/02/04/googles-gemini-app-has-surpassed-750m-monthly-active-users/", "shows": "Pichai official statement: Gemini app >750M MAU, Feb 2026", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "model breadth: frontier (Gemini 3.x family); RAG/file upload: yes; multimodal: yes (vision, audio, image-gen, voice/Live); tools: yes (web/Search grounding, Workspace, extensions); multi-user: yes (Workspace/Enterprise); deep Google ecosystem integration", "confidence": "high", "note": "Frontier consumer chat product within ui_api: full multimodal (Live voice, image-gen), Search grounding, Workspace integration, extensions, enterprise. Sets the category bar alongside ChatGPT.", "sources": [ { "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/", "shows": "Gemini app multimodal capabilities + Search/AI Mode integration", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "Gemini app (Google), consumer chat app (web + Android/iOS), proprietary/closed UI scored as the closed counterpart. Verified live June 2026; powered by Gemini 3.x model family. (Model SKUs scored separately; this record scores the app surface.)" }, { "product": "Microsoft Copilot", "org": "Microsoft", "type": "software", "description": "Microsoft's consumer/enterprise Copilot chat surface on Windows, web, mobile, and embedded across Microsoft 365 (Word, Excel, Teams, Outlook). 15M paid M365 Copilot seats and 33M active users as of Q2 FY2026, though usage share has slipped vs ChatGPT and Gemini.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS/embedded);source:closed;self-host:none;model:OpenAI+Microsoft(closed)", "confidence": "high", "note": "Fully proprietary, SaaS/embedded, the closed counterpart on the UI/API shelf.", "sources": [ { "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/activity-reports/microsoft-copilot-usage?view=o365-worldwide", "shows": "Microsoft 365 Copilot Chat is a Microsoft-operated proprietary product (admin usage reporting docs)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "~420M monthly active users across all Copilot surfaces (Windows/Edge/M365/Bing/mobile) reported Q1 2026, with ~160M enterprise-licensed; far above the >10M threshold. Actively-engaged M365 seats are much lower (~33M), but the consumer + embedded surface reach is mass-market. Headline aggregated figure traces to Microsoft disclosures via aggregators; primary per-surface breakdown not directly fetched, so medium confidence.", "sources": [ { "url": "https://www.businessofapps.com/data/microsoft-copilot-statistics/", "shows": "~420M MAU all surfaces Q1 2026; ~160M enterprise-licensed (aggregator compiling Microsoft disclosures)", "accessed": "2026-06-04" }, { "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/activity-reports/microsoft-365-copilot-usage?view=o365-worldwide", "shows": "Microsoft's own active-usage reporting surface for M365 Copilot (confirms scale of deployed product)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "chat-UI matrix: broad model access (GPT family + Microsoft models), deep RAG/grounding over M365 Graph (mail/docs/Teams), multimodal (vision/image gen), Copilot Pages/canvas, plugins/connectors + Copilot Studio extensibility, enterprise multi-user/admin. Missing: model-agnostic provider breadth (locked to OpenAI/Microsoft).", "confidence": "medium", "note": "Feature-rich chat UI with class-leading enterprise grounding/integration; not frontier-defining within this category because it is locked to one model provider and not self-hostable.", "sources": [ { "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/activity-reports/microsoft-365-copilot-usage?view=o365-worldwide", "shows": "M365 Copilot chat features incl. Graph grounding across apps", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Microsoft Copilot consumer + M365 Copilot chat assistant surfaces (Windows, Edge, M365, Bing, mobile), live June 2026 per Microsoft Learn admin/usage docs. Proprietary chat UI over OpenAI + Microsoft models; human-in-the-loop assistant (Copilot Studio agents are a separate product). Scored as the chat-UI surface." }, { "product": "Perplexity", "org": "Perplexity AI", "type": "software", "description": "Search-grounded consumer chat product that returns cited answers with live web results, available on web, mobile, and as a Comet browser. 45M+ MAU and ~$200M ARR with a $21B valuation in 2026; the consumer surface (perplexity.ai) is human-driven Q&A, distinct from the Perplexity API (Cat 11) and the agentic Comet browser.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS);source:closed;self-host:none;models:multi-provider(closed orchestration)", "confidence": "high", "note": "Proprietary SaaS chat/answer engine; closed counterpart on the shelf.", "sources": [ { "url": "https://www.perplexity.ai/", "shows": "consumer AI answer/search chat product, proprietary SaaS (no source/self-host)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "CEO Aravind Srinivas (cited Apr 2026) reports 100M+ MAU across all products; standalone chatbot ~33M MAU; ~780M queries/mo (May 2025) growing >20% MoM to an estimated 1.2-1.5B/mo by mid-2026. Even the standalone-chat figure (33M) clears >10M. Headline user count is CEO-attributed via aggregators (Sacra), not a directly-fetched primary disclosure, medium confidence.", "sources": [ { "url": "https://www.businessofapps.com/data/perplexity-ai-statistics/", "shows": "100M+ MAU all products / ~33M standalone chatbot (CEO-attributed, compiled)", "accessed": "2026-06-04" }, { "url": "https://www.perplexity.ai/", "shows": "live consumer product confirming scale of surface", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "chat-UI matrix: multi-model access (selectable frontier models incl. GPT/Claude/Gemini + own Sonar), strong web-grounded RAG with citations (its core differentiator), multimodal input, file upload, Spaces/collections, multi-user/enterprise. Search-grounded answering is best-in-class for the category.", "confidence": "medium", "note": "Differentiated, capable consumer chat UI (citation-grounded answers, model choice); high feature breadth but not the single category frontier-definer.", "sources": [ { "url": "https://www.perplexity.ai/", "shows": "multi-model selection, web-grounded cited answers, file/multimodal features", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Perplexity consumer AI answer/search chat product (web + apps), live June 2026 (perplexity.ai homepage confirms product). Proprietary chat UI routing over multiple closed + open models; human-in-the-loop search/answer assistant. Comet browser + Computer agent are adjacent surfaces (agentic, scored elsewhere if at all)." }, { "product": "Proton Lumo", "org": "Proton", "type": "software", "description": "Privacy-first AI assistant from Proton with zero-access encryption, a no-logs policy, and a \"ghost mode\" that discards chats on close. Runs on open-weight models from Proton's European data centers, excluding third-party providers like OpenAI from inference. Web and mobile clients are open source (GPLv3), though the backend service, routing layer, and model serving are not public. The differentiator is confidentiality - no account is required and conversations are never used for training.", "openness": { "score": 3, "class": "open_core", "components": "clients:open(GPLv3 - web in ProtonMail/WebClients, iOS/Android in ProtonLumo);service:closed(backend, routing, model-serving not public);license:GPLv3(clients)", "confidence": "high", "note": "Open clients (GPLv3) over a proprietary hosted service - the backend, routing, and model-serving layer are not published. Marketed as \"fully open source,\" but support and third-party reviews note that only the clients are open, so open_core is the honest class.", "sources": [ { "url": "https://github.com/ProtonLumo/ios-lumo", "shows": "iOS client source under GPLv3", "accessed": "2026-06-17" }, { "url": "https://isitreallyfoss.com/projects/lumo/", "shows": "only the Lumo mobile/web apps are open; service + model interaction code not public", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "reported_traction", "confidence": "medium", "note": "Launched July 2025 with Lumo 1.1 following; distributed across Proton's large account base (100M+ Proton accounts) and available on web + iOS + Android with no account required. No Lumo-specific usage numbers disclosed, so reported_traction at a mid level.", "sources": [ { "url": "https://proton.me/blog/lumo-ai", "shows": "Proton's launch announcement for Lumo", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "model breadth: open-weight models (no third-party providers); RAG/file upload: yes (incl. Proton Drive); web search: yes; reasoning: yes (Lumo 1.1); multimodal: limited; multi-user: single-user; privacy: zero-access encryption + ghost mode", "confidence": "medium", "note": "Capable privacy assistant with file upload, web search, and a reasoning mode, but confined to open-weight models and the Proton ecosystem; lighter on multimodal/plugins than frontier consumer apps. Mid-tier on the chat-UI feature matrix.", "sources": [ { "url": "https://proton.me/blog/lumo-1-1", "shows": "Lumo 1.1 adds faster, advanced reasoning", "accessed": "2026-06-17" } ] }, "maturity": 3.0, "mature": false, "version_note": "Launched July 2025; Lumo 1.1 added advanced reasoning. Clients are GPLv3 (web client lives in ProtonMail/WebClients; iOS/Android in ProtonLumo), but the service, models, and routing are not open sourced, so the \"fully open source\" marketing claim is contested." }, { "product": "Duck.ai", "org": "DuckDuckGo", "type": "software", "description": "DuckDuckGo's free, anonymous AI chat surface. Lets users chat with multiple third-party models without an account - Anthropic's Claude 4.5 Haiku, Meta's Llama 3.3 70B, Mistral Small 3, and OpenAI's GPT-4o mini on the free tier, with frontier models (GPT-5.4, Claude Sonnet 4.6, Llama 4 Maverick) for subscribers. Strips personal metadata (including IP) before prompting providers, and conversations are not used to train models. The differentiator is privacy - a proprietary anonymizing proxy in front of otherwise-closed model APIs.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "source:closed;self-host:none;license:proprietary(SaaS app + browser integration)", "note": "Proprietary anonymizing proxy in front of closed model APIs; no source, no self-host. Closed privacy-chat surface in the ui_api shelf.", "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/duckai/ai-chat-privacy", "shows": "DuckDuckGo's proprietary Duck.ai privacy/help documentation", "accessed": "2026-06-17" } ], "bucket": "closed" }, "adoption": { "level": 3, "signal_type": "reported_traction", "confidence": "low", "note": "Backed by DuckDuckGo's large user base and integrated into its browser, but no Duck.ai-specific usage figures are published. Conservative reported_traction pending a disclosed active-user signal.", "sources": [ { "url": "https://spreadprivacy.com/ai-chat/", "shows": "DuckDuckGo's launch of free, anonymous AI Chat across web + browser", "accessed": "2026-06-17" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "model breadth: Claude 4.5 Haiku, Llama 3.3 70B, Mistral Small 3, GPT-4o mini (free); frontier models for subscribers; RAG: no; plugins/tools: no; multimodal: limited; multi-user: n/a (anonymous, no accounts); privacy: metadata stripped, no training", "confidence": "high", "note": "Anonymizing multi-model chat proxy - strong on privacy and model choice, but a deliberately minimal surface (no RAG, plugins, agents, or multimodal generation). Lower-tier on the chat-UI feature matrix.", "sources": [ { "url": "https://duckduckgo.com/duckduckgo-help-pages/duckai/chat-models", "shows": "available models (Claude 4.5 Haiku, Llama 3.3 70B, Mistral Small 3, GPT-4o mini; subscriber models)", "accessed": "2026-06-17" } ] }, "maturity": 2.7, "mature": false, "version_note": "Proprietary; web-based and integrated into the DuckDuckGo browser. No source, no self-host - the closed counterpart to the privacy-chat surfaces in this shelf." }, { "product": "Maple AI", "org": "OpenSecret", "type": "software", "description": "End-to-end encrypted, private AI chat assistant. Conversations are encrypted on the user's device and processed inside hardware-isolated Trusted Execution Environments (AWS Nitro Enclaves + Nvidia GPU TEEs) so that even the operator cannot read user data, with remote attestation that users can verify against open source builds. Maple is the flagship app built on the open OpenSecret confidential-computing platform (same team), serves a menu of open-weight models, and ships as web, iOS, Android, and desktop apps. The differentiator is cryptographically verifiable privacy rather than a policy promise.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(Maple-client)+AGPL-3.0(OpenSecret-platform),both-OSI;source:public(github.com/OpenSecretCloud);models:open-weight;hosted-tier:paid(managed-enclaves)", "confidence": "medium", "note": "Both halves of the stack are open under standard OSI licenses (Maple client MIT, OpenSecret platform AGPL-3.0) and the served models are open-weight - unusually clean for a \"private AI\" product. Scored open_source because the privacy guarantee is only credible because the code is open and attestable. A defensible alternative is open_core, since the delivered offering is a paid hosted service over managed enclaves.", "sources": [ { "url": "https://github.com/OpenSecretCloud/Maple", "shows": "MIT-licensed Maple client (Tauri/Rust/TS), active; multi-platform", "accessed": "2026-06-17" }, { "url": "https://github.com/OpenSecretCloud/opensecret", "shows": "AGPL-3.0 OpenSecret platform; AWS Nitro enclave / TEE confidential-computing architecture", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "reported_traction", "confidence": "low", "note": "Launched Jan 2025; vendor reports \"5x user growth in the last six months\", >90% month-over-month paid retention, and \"billions of tokens\" in year one, but no absolute user count is published. Self-reported figures only, hence low confidence.", "sources": [ { "url": "https://blog.trymaple.ai/one-year-of-private-ai-how-maple-ai-made-encryption-easy/", "shows": "launch date (Jan 2025); 5x growth, >90% retention, billions of tokens processed", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "models: 6+ open-weight (gpt-oss-120b, kimi-k2.5, deepseek-r1, llama-3.3-70b, qwen3-vl, gemma-3); E2EE chat in TEEs with remote attestation; image/vision (gated to paid tiers); voice/transcription; OpenAI-compatible API (Maple Proxy); platforms: web/iOS/Android/macOS/desktop; teams/multi-seat", "confidence": "medium", "note": "Broad multi-model, multi-platform private chat with vision, API, and teams. Short of top because it is a focused private-chat product without a confirmed agentic tool-use/web-browsing ecosystem.", "sources": [ { "url": "https://blog.trymaple.ai/maple-ai-model-guide-with-example-prompts/", "shows": "model lineup (open-weight) and tier/feature structure", "accessed": "2026-06-17" } ] }, "maturity": 3.3, "mature": false, "version_note": "Maple client is MIT-licensed; the underlying OpenSecret platform is AGPL-3.0 - both fully open so anyone can rebuild and compare hashes against the live attestation. Launched Jan 2025. Serves open-weight models (gpt-oss-120b, kimi-k2.5, deepseek-r1, llama-3.3-70b, qwen3-vl, gemma-3). The consumer offering at trymaple.ai is a paid hosted service - openness applies to the code/architecture, not free unlimited usage." }, { "product": "Privatemode", "org": "Edgeless Systems", "type": "software", "description": "Confidential-computing AI inference service from Edgeless Systems. Prompts are encrypted on the user's device and processed inside hardware-based Trusted Execution Environments (confidential VMs + Nvidia H100 GPUs) so data stays encrypted even during inference and the provider cannot see it, with end-to-end remote attestation making the privacy guarantee verifiable in hardware. It serves a rotating menu of open-weight models behind a chat UI and an OpenAI/Anthropic-compatible API, and is positioned for regulated industries (finance, law, healthcare).", "openness": { "score": 2, "class": "source_available", "components": "license:custom-source-available(audit-only)+MIT-subdirs(proxy/web/internal-oss);source:TCB-public(github.com/edgelesssys/privatemode-public);service:proprietary-hosted;models:third-party-open-weight", "confidence": "high", "note": "Not open source: the security-critical Trusted Computing Base is publicly readable (and the client proxy + web app are MIT), but the main license permits only viewing/compiling for audit, and the operational service is a proprietary hosted offering. More open than a black-box API, less than open-core.", "sources": [ { "url": "https://github.com/edgelesssys/privatemode-public", "shows": "TCB source (attestation agent, proxy, web app, SDK) under a custom audit-only license with MIT-licensed subdirectories", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "signal_type": "reported_traction", "confidence": "low", "note": "Announced Feb 18 2025; offers a free tier, pay-as-you-go, and enterprise plans, and is co-marketed with Capgemini for regulated industries, but no customer or usage numbers are published. New and unquantified.", "sources": [ { "url": "https://www.edgeless.systems/blog/what-is-privatemode", "shows": "launch date (Feb 18 2025); Edgeless Systems as maker; confidential-inference positioning", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "models: rotating menu of open-weight third-party models (chat/coding/embedding/speech); interfaces: chat UI (no sign-up) + OpenAI/Anthropic-compatible API (streaming, tool calling, structured outputs, prompt caching); confidential compute: confidential VMs + H100, end-to-end remote attestation, reproducible TCB; integrations: Claude Code/OpenCode/VS Code/JetBrains/n8n/AnythingLLM", "confidence": "medium", "note": "Broad model menu plus a drop-in OpenAI-compatible API, chat UI, and strong integration story. Not top-tier because it is a confidential inference relay for others' models rather than a frontier model or full platform.", "sources": [ { "url": "https://www.privatemode.ai", "shows": "model list, chat UI + API, integrations, confidential-compute guarantees", "accessed": "2026-06-17" } ] }, "maturity": 2.6, "mature": false, "version_note": "Launched Feb 2025. The privatemode-public repo holds the auditable Trusted Computing Base under a custom source-available license (view/compile for audit only); the privatemode-proxy, web app, and internal/oss subdirectories are dual-licensed MIT. The operational service itself is proprietary/hosted. Serves third-party open-weight models (e.g. gpt-oss-120b, Gemma, Kimi K2, Qwen embeddings, Whisper/Voxtral) - trains none of its own." }, { "product": "Confer", "org": "Confer Labs", "type": "software", "description": "End-to-end encrypted AI chat assistant from Confer Labs (founded by Moxie Marlinspike, creator of Signal) whose core claim is that the operator cannot read, train on, or hand over your conversations. Prompts are encrypted on-device and sent via Noise-protocol pipes into a Trusted Execution Environment on a confidential VM, processed by open source LLMs, then re-encrypted before leaving; chat history is encrypted with passkey-derived keys. The differentiator is cryptographically enforced, verifiable privacy backed by remote attestation, reproducible builds, and an append-only transparency log.", "openness": { "score": 2, "class": "source_available", "components": "license:none-declared(no LICENSE file in either public repo);source:partial(confer-proxy + confer-image public,client-app private);self-host:no(depends on vendor attestation infra);models:open source(undisclosed)", "confidence": "high", "note": "Marketed as open source, but neither public repo declares any license (GitHub reports license:null), so re-use rights are undefined, and the end-user client is not public. Source is published for verifiability/reproducible builds, not for self-deployment. Accurate classification is source-available, not open_source.", "sources": [ { "url": "https://confer.to/blog/2026/01/private-inference/", "shows": "architecture (TEE, Noise pipes, dm-verity, reproducible nix/mkosi builds, transparency log); names the confer-proxy and confer-image repos", "accessed": "2026-06-17" }, { "url": "https://github.com/ConferLabs/confer-proxy", "shows": "public proxy repo with no LICENSE file (no declared license)", "accessed": "2026-06-17" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "signal_type": "reported_traction", "confidence": "low", "note": "Young product (repos created Dec 2025, launched late 2025/early 2026); ~70-75 stars per repo and a reported partnership \"bringing foundational AI privacy to Meta\", but no user/usage numbers. Notability is driven mainly by the founder's reputation.", "sources": [ { "url": "https://confer.to/blog/", "shows": "product positioning and feature timeline (memory, encrypted folders, connectors, Meta partnership)", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "E2EE chat with TEE inference; remote attestation + reproducible builds + transparency log; encrypted chat history (passkey-derived keys); encrypted folders; cross-conversation memory; emerging connectors; models: multiple open source (task-routed via vLLM, names undisclosed); self-host: no; web client", "confidence": "medium", "note": "Strong, genuinely novel on the privacy/crypto dimension, but a relatively standard chat feature set, no self-hosting, and undisclosed models cap it at mid-tier.", "sources": [ { "url": "https://confer.to/blog/2026/01/private-inference/", "shows": "feature set: E2EE chat, TEE inference, attestation, transparency log, encrypted history", "accessed": "2026-06-17" } ] }, "maturity": 2.3, "mature": false, "version_note": "Confer publishes its inference/proxy stack and reproducible confidential-VM image (vLLM-based) openly for verifiability, but NEITHER repo carries a LICENSE file (GitHub reports no declared license), and the end-user client app is not public - so despite \"open source\" marketing the accurate classification is source-available. The service is not self-hostable; its security model depends on Confer's attestation infrastructure. Repos created Dec 2025." }, { "product": "Poe", "org": "Quora", "type": "software", "description": "Multi-model chat aggregator letting users talk to GPT-5.2, Claude Opus 4.6, Gemini 3 Pro, image/video generators, and 1M+ user-created bots from one interface, with a points-based unified pricing model. Differentiator is breadth (200+ models in one app) plus a creator economy of custom bots; recently added 200-person group chats. Run by Quora.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS);source:closed;self-host:none;models:multi-provider aggregation(closed)", "confidence": "high", "note": "Proprietary multi-model chat aggregator; closed counterpart on the shelf.", "sources": [ { "url": "https://poe.com/about", "shows": "proprietary multi-model chat aggregator (no source/self-host), Quora-affiliated", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "low", "note": "CEO/Quora figures (compiled by multiple aggregators) report ~18M Poe MAU in 2026 (~40% using paid features; ~$65M subscription ARR). 18M is comfortably in the >10M 'mass market' band -> level 5. Quora publishes no primary Poe MAU page, so the exact figure remains aggregator-sourced (confidence low), but the order of magnitude is consistent across sources and clearly clears the >10M threshold. [Corrected: prior record had level 4 with an invalid '10M-100M' reach band that contradicted the 18M figure.]", "sources": [ { "url": "https://originality.ai/blog/poe-statistics", "shows": "~18M Poe MAU, ~$65M subscription revenue (compiled from Quora reporting)", "accessed": "2026-06-04" }, { "url": "https://poe.com/about", "shows": "live product confirming consumer-scale aggregator surface", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "chat-UI matrix: very broad model breadth (text + image + video + audio across many providers in one UI), user-created bots / custom bot builder, multimodal, points-based unified billing, multi-platform clients. Lighter on enterprise/RAG vs Copilot/Perplexity.", "confidence": "medium", "note": "Strongest pure model-breadth in the consumer aggregator slot (text/image/video/audio + custom bots); high feature breadth, not the frontier-definer (LiteLLM anchors the gateway side at C4).", "sources": [ { "url": "https://poe.com/about", "shows": "multi-provider model access incl. image/video/audio + user-created bots", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Poe by Quora, multi-model chat aggregator app (web + mobile/desktop), live June 2026 (poe.com confirms). Single UI fronting many providers' bots (ChatGPT, Claude, Gemini, DeepSeek) + image/video/audio models + millions of user-created bots. Human-in-the-loop chat aggregator; proprietary." }, { "product": "OpenRouter", "org": "OpenRouter", "type": "software", "description": "Commercial API gateway that exposes 400+ models from 60+ providers (OpenAI, Anthropic, Google, Meta, DeepSeek, plus open-weights hosts) behind a single OpenAI-compatible endpoint with automatic failover and unified billing. The dominant closed-source LLM aggregator: ~$50M annualized revenue and reportedly raising $120M at a $1.3B valuation (CapitalG-led) in early 2026.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(hosted gateway);source:closed(SDK present, core service closed);self-host:none", "confidence": "high", "note": "Proprietary hosted routing gateway, the closed counterpart to open gateways (LiteLLM); listed as closed in the recipe.", "sources": [ { "url": "https://openrouter.ai/", "shows": "hosted unified API gateway across 400+ models/60+ providers; proprietary service (no self-host)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Vendor homepage (primary) reports 100T monthly tokens processed and 8M+ global users; ~250k+ apps. Token volume corroborated by external coverage (~25T tokens/week, ~1.5 quadrillion/yr run-rate; Menlo, TechCrunch on $1.3B valuation / $113M raise May 2026). Reach placed at 1M-10M on the 8M user figure; usage-volume signal is genuinely massive.", "sources": [ { "url": "https://openrouter.ai/", "shows": "vendor headline: 100T monthly tokens, 8M+ global users, 400+ models / 60+ providers", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2026/05/26/openrouter-more-than-doubles-valuation-to-1-3b-in-a-year/", "shows": "corroborates ~100T tokens/mo and $1.3B valuation (May 2026)", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "gateway matrix: provider/model coverage 400+ models across 60+ providers (category-leading breadth); automatic routing/fallback/load-balancing across providers; per-request cost tracking + credits; auth/keys + data policies; OpenAI-compatible drop-in API; provider/model ranking analytics. Hits every gateway feature-matrix dimension at the top of the category.", "confidence": "high", "note": "Frontier-defining for the gateway slot, broadest provider coverage + routing/fallback at the largest measured token scale; the closed counterpart that sets the bar LiteLLM (C4) approaches on the open side.", "sources": [ { "url": "https://openrouter.ai/", "shows": "400+ models, 60+ providers, routing/fallback, cost tracking, OpenAI-compat", "accessed": "2026-06-04" } ] }, "maturity": 4.3, "mature": false, "version_note": "OpenRouter unified LLM API gateway/router, live June 2026 (openrouter.ai homepage). Routes/aggregates API calls across 400+ models and 60+ providers behind one OpenAI-compatible API; not a chat UI but a routing gateway, squarely the ui_api gateway slot alongside the LiteLLM anchor. Proprietary hosted service." }, { "product": "Character.AI", "org": "Character.AI", "type": "software", "description": "Consumer chat product centered on user-created character personas, dialogue roleplay, and conversational entertainment. ~20M MAU and ~6M daily visitors with two hours/day average engagement, one of the stickiest consumer AI surfaces; founders' team was acqui-hired by Google in 2024 but the product continues to operate.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS);source:closed;self-host:none;models:own+licensed(closed)", "confidence": "high", "note": "Proprietary consumer chat app; closed counterpart on the shelf.", "sources": [ { "url": "https://www.businessofapps.com/data/character-ai-statistics/", "shows": "Character.AI is a proprietary consumer companion-chat app (product/company profile)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "10M-100M", "signal_type": "active_users", "confidence": "low", "note": "Reported ~20M MAU and ~180M monthly site visits (Apr 2026); peaked ~28M MAU mid-2024 then declined. Clears >10M but figures are aggregator-compiled (no first-party MAU disclosure fetched) and sources disagree (20M vs higher visit-based counts), so low confidence. Reach mapped to 10M-100M on monthly-visit volume; level 4 on the conservative MAU read.", "sources": [ { "url": "https://www.businessofapps.com/data/character-ai-statistics/", "shows": "~20M MAU, ~180M monthly visits, peak ~28M mid-2024 (compiled)", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "chat-UI matrix: narrow/specialized: strong character-creation + persona/roleplay + voice + multi-user community, but no model selection (single-provider, own models), limited RAG/grounding, limited multimodal/document tooling vs general assistants. Deep in its niche, narrow on the general chat-UI feature set.", "confidence": "medium", "note": "Specialized companion-chat UI; rich in persona/voice features but narrow on the general feature matrix (no model breadth, weak grounding); low capability relative to the general-assistant frontier in this category.", "sources": [ { "url": "https://www.businessofapps.com/data/character-ai-statistics/", "shows": "companion/roleplay chat product with character-creation focus", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Character.AI consumer companion/roleplay chat app (web + mobile), live June 2026. Proprietary chat UI over its own models + (post-Google-deal) licensed models; human-in-the-loop character chat. MAU peaked ~28M mid-2024, declined amid competition." }, { "product": "Grok", "org": "xAI", "type": "software", "description": "xAI's consumer chat product with a standalone app (grok.com) plus deep integration into X (Twitter), competing on real-time data access and a less-filtered tone. ~64M MAU by early 2026 and ~17.8% US chatbot share, third behind ChatGPT and Gemini, with the fastest single-year share gain of any consumer chatbot.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS/embedded in X);source:closed;self-host:none;model:xAI Grok(closed weights)", "confidence": "high", "note": "Proprietary chat UI (and its underlying model is closed); closed counterpart on the shelf.", "sources": [ { "url": "https://x.ai/news/grok-4-1", "shows": "xAI Grok is a proprietary assistant/product (official xAI release notes)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "low", "note": "App reported ~60-64M MAU and ~8-10M DAU early 2026, plus default availability to X Premium subscribers within X's ~350-400M MAU base. Clears >10M as a surface. Figures are aggregator-compiled (xAI does not publish a primary MAU page) and the X-embedded reach is hard to attribute cleanly, so low confidence on the exact number; reach band is robust.", "sources": [ { "url": "https://seoprofy.com/blog/grok-ai-statistics/", "shows": "~60-64M Grok app MAU, ~8-10M DAU early 2026; X Premium default access (compiled)", "accessed": "2026-06-04" }, { "url": "https://x.ai/news/grok-4-1", "shows": "official product confirming the consumer Grok assistant surface", "accessed": "2026-06-04" }, { "url": "https://x.com/xai/status/1990530499752980638", "shows": "xAI official post: Grok 4.1 frontier model, free on grok.com, X and mobile apps (confirms the consumer Grok assistant surface)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "chat-UI matrix: frontier-class own model (reasoning/multi-agent), real-time X-grounded search (differentiator), image generation, voice mode, multimodal, large context; multi-user via X/SuperGrok. Locked to xAI models (no provider breadth), lighter enterprise/RAG-over-private-data story.", "confidence": "high", "note": "Capable general assistant UI with a unique real-time X-search edge; high feature breadth but single-provider-locked, so not the category frontier-definer. [verifier: Grok 4.1 features (real-time X-grounded search, multimodal, reasoning, free on grok.com/X/apps) confirmed via xAI's own release + X post; x.ai/news/grok-4-1 is live (403 to bot-fetch only). Capability evidence now primary.]", "sources": [ { "url": "https://x.ai/news/grok-4-1", "shows": "Grok assistant features: reasoning model, real-time search, multimodal", "accessed": "2026-06-04" } ] }, "maturity": 4.7, "mature": false, "version_note": "Grok consumer AI chat assistant by xAI, the app/UI surface (grok.com + X integration + mobile), live June 2026 (x.ai). Scored here as the chat-UI product, NOT the Grok 4.x model SKU (the model is scored in base_pretrained as Grok 4.20). Proprietary; human-in-the-loop assistant with image gen + X-grounded search." }, { "product": "Le Chat", "org": "Mistral AI", "type": "software", "description": "Mistral's consumer/enterprise chat product (chat.mistral.ai) running on Mistral Medium/Large with Flash Answers, web search, image gen, and an enterprise on-prem deployment option. 1M downloads in 14 days at app launch; ~10.8M monthly desktop visits in March 2026 with 21% MoM growth, Europe's flagship sovereign-AI chat surface.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS app);source:closed(the Le Chat product; underlying Mistral models are open-weights but the chat UI/service is not);self-host:none", "confidence": "high", "note": "The Le Chat product/service is proprietary SaaS (closed), distinct from Mistral's open-weight models scored elsewhere; closed counterpart on the shelf.", "sources": [ { "url": "https://techcrunch.com/2025/02/19/mistrals-le-chat-tops-1m-downloads-in-just-14-days/", "shows": "Le Chat is Mistral's proprietary consumer chat app (product profile)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "active_users", "confidence": "medium", "note": "~1.7M app downloads by May 2026 (TechCrunch reported 1M in 14 days at Feb 2025 launch); Mistral states EU monthly active recipients below 45M (a DSA ceiling, not a measured MAU); ~10.8M desktop visits to mistral.ai Mar 2026. Honest read places real active users in the 1-10M band, level 4. The <45M is a regulatory cap not an actual count, so not used as the headline.", "sources": [ { "url": "https://techcrunch.com/2025/02/19/mistrals-le-chat-tops-1m-downloads-in-just-14-days/", "shows": "1M downloads in 14 days at launch (primary-press milestone)", "accessed": "2026-06-04" }, { "url": "https://www.businessofapps.com/data/perplexity-ai-statistics/", "shows": "context: peer consumer-chat scale for calibration", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "chat-UI matrix: multimodal (vision, image gen via Flux), web search, code interpreter/canvas, document/Flash answers (fast inference), connectors/agents, enterprise (Le Chat Enterprise) + flash-fast responses on Cerebras. Locked primarily to Mistral models (limited provider breadth).", "confidence": "medium", "note": "Feature-broad general assistant UI (search, multimodal, code, enterprise) with a speed differentiator; high breadth but single-provider-leaning, not the category frontier-definer.", "sources": [ { "url": "https://techcrunch.com/2025/02/19/mistrals-le-chat-tops-1m-downloads-in-just-14-days/", "shows": "Le Chat consumer assistant feature set (web search, multimodal, fast answers)", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Le Chat by Mistral AI, consumer AI chat assistant (web + mobile; app rebranded toward 'Mistral Vibe' by 2026), live June 2026 (mistral.ai). Proprietary chat UI primarily over Mistral's own models; human-in-the-loop assistant. ~1.7M app downloads by May 2026; EU MAU stated below 45M." }, { "product": "DeepSeek Chat", "org": "DeepSeek", "type": "software", "description": "Chinese consumer chat app from DeepSeek (Hangzhou) on chat.deepseek.com plus iOS/Android apps, exposing DeepSeek V3/R1 models with thinking mode and web search. ~131.5M MAU in January 2026 (90% MoM surge), second only to ChatGPT in global AI DAU rankings, driven by free access to frontier-tier models.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "client:proprietary(no public source for chat.deepseek.com app);backend:hosted-SaaS;note:underlying models open(MIT) but the chat application/service is closed", "note": "The DeepSeek chat surface is a proprietary hosted service; openness of the V3/V4 weights it serves is a separate (model-layer) score. Scored as closed counterpart per ui_api recipe.", "sources": [ { "url": "https://www.deepseek.com/en/", "shows": "Free hosted chat at chat.deepseek.com serving DeepSeek-V4 Preview; no source release for the chat app itself", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "DeepSeek's consumer app was one of the most-downloaded global AI apps after the early-2025 surge; the hosted chat surface remains a top-tier global assistant. No clean current MAU figure published on the vendor's own page, so confidence medium; reach >10M is well-supported by app-store ranking history. Attributed to the chat surface, not the model SKU.", "sources": [ { "url": "https://play.google.com/store/apps/details?id=com.deepseek.chat&hl=en_US", "shows": "Official DeepSeek AI Assistant Android app listing (consumer chat surface, mass-market distribution)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "chat UI feature set: single-vendor model breadth (DeepSeek V3/V4 family only); file upload + long-context; web/app multimodal; agent capabilities (V4 Preview); no multi-provider model hub; consumer single-user (no enterprise multi-user/RAG-over-corp-data)", "confidence": "medium", "note": "Capable single-vendor consumer chat UI with strong underlying model, but narrow on the ui_api feature axes (one provider, limited enterprise/RAG/plugin breadth) vs frontier multi-model chat UIs. Not a category frontier-definer.", "sources": [ { "url": "https://www.deepseek.com/en/", "shows": "Chat product features: file reading, long-context, V4 agent capabilities, single-vendor model", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "Hosted consumer chat UI at chat.deepseek.com (web + iOS/Android apps), currently fronting DeepSeek-V4 Preview (stronger agent + reasoning). Verified live June 2026. Note: the underlying DeepSeek-V3/V4 model weights are open (MIT) and scored separately in base_pretrained; this record scores the CHAT PRODUCT/UI itself, which is a proprietary free hosted service, not open source software." }, { "product": "Meta AI", "org": "Meta", "type": "software", "description": "Meta's consumer AI assistant, a chat surface embedded directly into WhatsApp, Instagram, Facebook, and Messenger, plus a standalone web app at meta.ai and native iOS/Android apps. Powered by Llama 4/5 across most surfaces with Muse Spark (Meta Superintelligence Labs' new closed-weight model, launched April 8, 2026) backing the standalone app and rolling out to Family-of-Apps in mid-2026. 1B+ monthly active users across Meta's apps, the only consumer AI assistant with native distribution into ~4B existing users; the standalone Meta AI iPhone app has held a top-of-chart App Store position alongside ChatGPT, Claude, and Gemini since the Muse Spark launch.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "client:proprietary(closed consumer app/embedded surface);backend:hosted-SaaS;note:underlying Llama weights are restricted (non-OSI) and model-layer scored elsewhere; the assistant product is closed", "note": "Meta AI the assistant/UI is a closed proprietary consumer product; scored as closed counterpart. (Llama-the-model openness is a separate restricted-class score.)", "sources": [ { "url": "https://www.demandsage.com/meta-ai-users/", "shows": "Meta AI described as proprietary consumer assistant embedded across Meta apps + meta.ai", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "high", "note": "Zuckerberg (May 2025 shareholder meeting, primary) stated Meta AI passed 1B monthly active users; reported growing to ~1.2B MAU by Q1 2026, ~63% of interactions on WhatsApp. Far above the >10M threshold. CAVEAT: a portion of these MAU reflect surfaced/embedded exposure inside WhatsApp/IG rather than deliberate assistant use (one analyst, Kantrowitz, argues engaged usage is far lower than OpenAI's). Attributed honestly to the embedded surface, not a standalone assistant count.", "sources": [ { "url": "https://www.facebook.com/yahoofinance/posts/zuck-says-meta-ai-has-more-than-one-billion-monthly-active-users-/1194397235888386/", "shows": "Zuckerberg statement: Meta AI has more than one billion monthly active users", "accessed": "2026-06-04" }, { "url": "https://finance.yahoo.com/news/meta-virtually-no-ai-users-144058392.html", "shows": "Analyst caveat: embedded MAU overstate engaged assistant usage vs OpenAI's ~900M", "accessed": "2026-06-04" }, { "url": "https://techcrunch.com/2025/05/29/meta-ai-now-has-1b-monthly-active-users/", "shows": "Primary press: Zuckerberg at May 2025 shareholder meeting says Meta AI has ~1B monthly active users", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "consumer assistant feature set: single-vendor (Llama family) model breadth; multimodal (text + image gen); embedded across messaging surfaces; consumer single-user (no enterprise RAG / multi-provider model hub / org-data grounding); limited plugin/tool breadth vs frontier chat UIs", "confidence": "medium", "note": "Broad-reach consumer assistant but narrow on the ui_api feature axes, one provider, consumer-only, limited enterprise/RAG/plugin depth. Not a category frontier-definer on capability.", "sources": [ { "url": "https://www.demandsage.com/meta-ai-users/", "shows": "Feature scope: embedded multimodal consumer assistant across Meta apps, Llama-powered", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "Meta's consumer AI assistant embedded across WhatsApp, Instagram, Messenger, Facebook + standalone meta.ai app. Proprietary; powered by Meta's Llama family (the underlying weights are use-restricted and scored separately at the model layer). Verified live June 2026." }, { "product": "Doubao", "org": "ByteDance Seed / Volcano Engine", "type": "software", "description": "ByteDance's consumer AI chat app (doubao.com plus iOS/Android) powered by the Doubao Seed model family, with native multimodal support (text, image, audio, video) and a full-duplex real-time voice mode that streams ASR, reasoning, and TTS concurrently for natural barge-in conversation. The largest Chinese consumer AI chat product by daily usage: ~100M DAU baseline with a peak of ~145M DAU during the February 2026 Lunar New Year, ~345M MAU per QuestMobile Q1 2026 (other trackers put it at 220-226M MAU around the same period). Served 1.9B+ queries as the exclusive AI partner of the 2026 CCTV Spring Festival Gala.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "client:proprietary(closed consumer app);backend:hosted-SaaS on Volcano Engine;models:proprietary Doubao(closed);license:Proprietary", "note": "Doubao is a closed proprietary consumer assistant; no public source for the app and the Doubao models are not open-weight. Closed counterpart per ui_api recipe.", "sources": [ { "url": "https://en.wikipedia.org/wiki/Doubao", "shows": "Doubao is a proprietary ByteDance AI chatbot app (iOS/Android/web) on Volcano Engine", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "active_users", "confidence": "medium", "note": "Doubao reached ~330M total users by May 2026 (en.wikipedia.org/wiki/Doubao); China's most popular AI chatbot; ~60M MAU as of Nov 2024 per Wikipedia. China-market app trackers report ~330M users as of May 2026. NOTE (verifier): the 330M figure is cited by Wikipedia as TOTAL users, not a measured MAU; the only Wikipedia-stated MAU is ~60M (Nov 2024). Either way the surface clears >10M. Headline counts are aggregator/Wikipedia-compiled (no single ByteDance primary MAU disclosure), so confidence medium. Volcano Engine reportedly serves ~120T tokens/day.", "sources": [ { "url": "https://en.wikipedia.org/wiki/Doubao", "shows": "Doubao ~330M users by May 2026; China's most popular AI chatbot; 120T tokens/day", "accessed": "2026-06-04" }, { "url": "https://pandaily.com/bytedance-s-doubao-introduces-paid-pro-tier-as-user-base-dip-jun2026", "shows": "MAU ~330M May 2026, down 6.1M after paid-tier signal; corroborating figure", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "consumer assistant feature set: single-vendor (Doubao models) breadth; multimodal; mobile-first consumer chat; beta AI e-commerce/agent features; consumer single-user (no enterprise RAG / multi-provider hub); China-market focused", "confidence": "medium", "note": "Feature-rich consumer assistant with multimodal + emerging agentic e-commerce, but narrow on the ui_api feature axes (single provider, consumer-only). Not a category frontier-definer on capability.", "sources": [ { "url": "https://news.aibase.com/news/26405", "shows": "Doubao multimodal consumer assistant adding agentic AI e-commerce/order features", "accessed": "2026-06-04" } ] }, "maturity": 4.4, "mature": false, "version_note": "ByteDance's consumer AI assistant (chatbot app: iOS/Android/web), launched Aug 2023, running on Volcano Engine (Doubao models). Proprietary closed product. Introduced a paid Pro subscription tier ~May/Jun 2026. Verified live June 2026 (Wikipedia + China-market app trackers)." }, { "product": "Glean Assistant", "org": "Glean", "type": "software", "description": "Enterprise AI coworker that grounds answers, drafts, slides, voice notes, and task execution in company context across apps and permissions. Glean positions it as the front-door work AI layer for employees, with product screenshots and messaging focused on proactive answers, content generation, and action-taking; the company also has a public $7.2B valuation headline and Gartner/G2 recognition, which signals meaningful enterprise adoption.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "client:proprietary;deployment:single-tenant SaaS;source:closed;license:Proprietary", "note": "Glean Assistant is a proprietary enterprise SaaS, no public source, no open-core/self-host OSS tier. Closed.", "sources": [ { "url": "https://www.glean.com/product/assistant", "shows": "Proprietary SaaS enterprise assistant; RAG over Enterprise Graph; not open source", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Glean surpassed $300M ARR (28 May 2026, vendor press), 15 months after $100M; nearly doubled Fortune 500 customer count YoY; 85%+ of customers use it across 5+ departments; 45% wDAU/wMAU engagement ratio; operating in 28 countries. ARR/enterprise-seat scale implies low-millions of enterprise seats; placed at 4 on reported enterprise traction. No raw seat/user count disclosed, so confidence medium and signal_type reported_traction (revenue + deployment breadth, not a measured user count).", "sources": [ { "url": "https://www.glean.com/press/glean-surpasses-300m-arr-unrivaled-enterprise-context-fuels-ai-adoption", "shows": "$300M ARR (May 2026); Fortune 500 count ~doubled YoY; 85%+ across 5+ depts; 45% wDAU/wMAU; 28 countries", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "enterprise chat-UI feature set: model breadth (Model Hub, multiple frontier LLMs + adaptive model selection); RAG (Enterprise Graph over 100+ connectors); multimodal output (images/slides/interactive pages); plugins/actions (workflow triggers, record updates, Skills); multi-user/enterprise (single-tenant, permissions-aware); strong across all five ui_api chat-UI axes; plus agentic sub-agent orchestration", "confidence": "high", "note": "Among the most feature-complete enterprise assistant UIs, multi-model hub, deep enterprise RAG, multimodal, actions, permissions-aware multi-user. Top-tier within ui_api for enterprise context; scored 4 (reserving 5 for an unambiguous category frontier-definer).", "sources": [ { "url": "https://www.glean.com/product/assistant", "shows": "Model Hub multi-LLM, Enterprise Graph RAG over 100+ connectors, multimodal output, task execution/actions, agentic sub-agents", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Enterprise AI assistant / 'AI coworker' grounded in the Glean Enterprise Graph (RAG over 100+ enterprise connectors), with a Model Hub (multiple frontier LLMs), multimodal output (images/slides/pages), and agentic sub-features (Skills, sub-agent orchestration). Proprietary single-tenant SaaS. May-2026 'enterprise AI coworker' launch verified live June 2026. BORDERLINE: agentic/Skills features touch orchestration_agents, but the product is primarily a human-in-driver enterprise chat/assistant UI grounded in company data, so kept in ui_api per the litmus (AI presented as a tool to the human)." }, { "product": "Glean Workplace Search AI", "org": "Glean", "type": "software", "description": "AI-powered workplace search across 100+ enterprise tools with permissions-aware results, knowledge-graph personalization, and generative answers. Glean’s own product pages and homepage emphasize that it searches everywhere employees work, then turns that context into concise answers and next steps; the company highlights a 141% ROI claim in its Forrester materials as an adoption signal.", "openness": { "score": 1, "class": "closed", "confidence": "high", "components": "client:proprietary;deployment:single-tenant SaaS;source:closed;license:Proprietary", "note": "Glean Search is proprietary enterprise SaaS; no public source, no OSS/self-host tier. Closed.", "sources": [ { "url": "https://www.glean.com/product/overview", "shows": "Glean Search as proprietary single-tenant SaaS enterprise search on Enterprise Graph", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Search is the foundational layer of the same Glean platform that crossed $300M ARR (May 2026) with near-doubled Fortune 500 count and 85%+ multi-department usage; effectively co-extensive with Glean's deployed seat base. Same caveat as Glean Assistant: no raw user count disclosed, so reported_traction at confidence medium. Adoption shared with the Assistant record (same platform), not an independent measurement.", "sources": [ { "url": "https://www.glean.com/press/glean-surpasses-300m-arr-unrivaled-enterprise-context-fuels-ai-adoption", "shows": "$300M ARR; Fortune 500 count ~doubled YoY; 85%+ across 5+ depts; search is platform foundation", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "enterprise search feature set: 100+ connector breadth (Slack/Teams/Zoom/ServiceNow/Zendesk/GitHub/...); Enterprise Graph + Personal Graph relevance/personalization; hybrid (semantic+keyword) search; permissions-aware results; single-tenant security; AI-assistant-grounding output", "confidence": "high", "note": "Best-in-class enterprise search on connector breadth, knowledge-graph relevance, and permissions-aware retrieval, a recognized leader in the enterprise-search-for-AI space. Scored 4 (top-tier; 5 reserved for an unambiguous frontier-definer).", "sources": [ { "url": "https://www.glean.com/product/overview", "shows": "Enterprise Graph + Personal Graph, hybrid search, 100+ connectors, permissions-aware enterprise search", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Glean Search ('the foundation of enterprise AI'), AI-powered enterprise/workplace search UI built on the Enterprise Graph + Personal Graph, hybrid search across 100+ connectors (Slack, Teams, Zoom, ServiceNow, Zendesk, GitHub, etc.), permissions-aware. Proprietary single-tenant SaaS. Verified live June 2026. The search layer underpins Glean Assistant; scored here as the human-facing search UI/API product." }, { "product": "LiteLLM", "org": "BerriAI", "type": "software", "description": "Python SDK and proxy server (AI gateway) that exposes a single OpenAI-format API across 100+ LLM providers (Bedrock, Azure, Vertex, Anthropic, Cohere, vLLM, etc.) with cost tracking, fallback routing, virtual keys, and guardrails. Picked as the de facto open API gateway for organizations that need provider-agnostic LLM access with spend controls; ~48k GitHub stars and adopted as enterprise gateway by many AI platform teams.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core);enterprise-dir:LiteLLM-Commercial-License;source:public", "confidence": "high", "note": "license:MIT(core);enterprise-dir:LiteLLM-Commercial-License;source:public", "sources": [ { "url": "https://github.com/BerriAI/litellm/blob/main/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/BerriAI/litellm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.litellm.ai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "Used-by 21.7k dependent repos on GitHub; ~49.2k stars, 1,342 releases. Treated as 100K-1M range on dependent/usage volume; stars not the basis.", "sources": [ { "url": "https://github.com/BerriAI/litellm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Unified gateway to 140+ LLM providers in OpenAI format with cost tracking, guardrails, load balancing, virtual keys/budgets, logging. No standard benchmark applies to a gateway; rated on breadth of provider coverage and production features.", "sources": [ { "url": "https://github.com/BerriAI/litellm", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.litellm.ai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.3, "mature": false, "version_note": "v1.84.5 (June 4, 2026)" }, { "product": "LM Studio", "org": "LM Studio", "type": "software", "description": "Desktop app (Windows/macOS/Linux) for discovering, downloading, and running local LLMs with a polished chat GUI, model browser, an OpenAI-compatible local server, headless mode, and native Apple MLX support. The smoothest local-LLM onboarding - but the app itself is closed-source.", "openness": { "score": 1, "class": "closed", "components": "app:proprietary-freeware(ToS forbids modify/reverse-engineer/redistribute);sdk+cli:MIT(public);core-engine:closed", "confidence": "high", "note": "Core app/engine is closed proprietary freeware; only peripheral CLI/SDKs are open.", "sources": [ { "url": "https://lmstudio.ai/app-terms", "shows": "ToS: no modification/reverse-engineering/redistribution", "accessed": "2026-06-17" } ], "bucket": "closed" }, "adoption": { "level": 3, "signal_type": "reported_traction", "confidence": "medium", "note": "Widely-used consumer app but no primary-verified install count; conservative level 3.", "sources": [ { "url": "https://lmstudio.ai/", "shows": "product positioning, free for home and work use", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "polished chat UI; model browser; OpenAI-compatible local server; headless mode; native Apple MLX", "confidence": "medium", "note": "Strong UX/polish on par with the open local UIs; loses on openness/extensibility (closed source).", "sources": [ { "url": "https://lmstudio.ai/", "shows": "feature set: chat UI, model browser, local server, MLX", "accessed": "2026-06-17" } ] }, "maturity": 3.3, "mature": false, "version_note": "Desktop app is proprietary freeware ('free for home and work use'; ToS forbids modification/reverse-engineering/redistribution). Only the CLI (lms) and JS/Python SDKs are MIT. Score 1 (not 0) because the SDK/CLI source is public." }, { "product": "GPT4All", "org": "Nomic AI", "type": "software", "description": "Free, cross-platform desktop app from Nomic AI for running LLMs privately on everyday laptops with no API or GPU required. Includes a chat UI, LocalDocs (private RAG over your files), an OpenAI-compatible local API server, and a Python client, with Nomic Vulkan GPU acceleration.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/nomic-ai/gpt4all", "shows": "MIT, ~77.4k stars, LocalDocs RAG + local API server", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~77k stars but reduced maintenance since early 2025 - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/nomic-ai/gpt4all", "shows": "~77.4k stars; last major release Feb 2025", "accessed": "2026-06-17" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "chat UI; LocalDocs private RAG; OpenAI-compatible local server; Vulkan GPU", "confidence": "medium", "note": "Solid core but behind faster-moving peers on newest-model support due to the maintenance slowdown.", "sources": [ { "url": "https://github.com/nomic-ai/gpt4all", "shows": "feature set: chat, LocalDocs, local server", "accessed": "2026-06-17" } ] }, "maturity": 3.0, "mature": false, "version_note": "MIT. ~77k stars. Reduced maintenance - last notable release Feb 2025, community reports of stalled development through 2025 (no official deprecation)." }, { "product": "OpenClaw", "org": "OpenClaw Foundation", "type": "software", "description": "Local-first, self-hosted personal AI assistant ('any OS, any platform'). It runs an agent gateway that answers on the messaging channels you already use - WhatsApp, Telegram, Slack, Discord, Signal, iMessage, Teams, Matrix and ~20 more - with companion apps for macOS/iOS/Android, voice (speak/listen), a live Canvas UI, and a skills/extensions SDK. Model-agnostic: the gateway routes to your chosen LLM backend.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI; body is MIT, GitHub NOASSERTION from custom copyright line);source:public;self-host:primary(local-first);no-feature-gated-core", "confidence": "high", "note": "Genuinely MIT open source; the NOASSERTION flag is a classifier artifact, not a restriction.", "sources": [ { "url": "https://github.com/openclaw/openclaw/blob/main/LICENSE", "shows": "MIT text, Copyright OpenClaw Foundation", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~379k stars and ~10.26M npm downloads/month - exceptional for a project under a year old.", "sources": [ { "url": "https://www.npmjs.com/package/openclaw", "shows": "npm openclaw, ~10.26M downloads/month", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "self-hosted agent gateway; 20+ messaging channels (WhatsApp/Telegram/Slack/Discord/Signal/iMessage/Teams/Matrix...); voice (speak/listen); WebChat + live Canvas UI; companion apps (macOS/iOS/Android); skills/extensions SDK; model-agnostic routing", "confidence": "high", "note": "Among the most ambitious open personal-assistant projects - unifies chat, voice, and many messaging surfaces; broader channel reach than Open WebUI/Jan/Khoj.", "sources": [ { "url": "https://api.github.com/repos/openclaw/openclaw", "shows": "real repo, 379k stars, TS, created 2025-11-24, active", "accessed": "2026-06-18" } ] }, "maturity": 5.0, "mature": true, "version_note": "MIT (repo LICENSE body is MIT; GitHub shows NOASSERTION due to a custom 'OpenClaw Foundation' copyright line; npm declares MIT). Community project led by Peter Steinberger (steipete). Created Nov 2025; ~379k stars, ~10.3M npm downloads/month." }, { "product": "LLM (Datasette)", "org": "Datasette", "type": "software", "description": "Command-line tool and Python library for interacting with large language models. Provides a unified CLI and API to send prompts to remote models (OpenAI, Anthropic, Gemini, Llama, etc.) and to local models, with a plugin architecture for adding new backends. Logs prompts and responses to a local SQLite database and supports embeddings and templated prompts. Part of Simon Willison's Datasette ecosystem (docs at llm.datasette.io).", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);PyPI;plugin-extensible;no feature-gated core", "confidence": "high", "note": "Apache-2.0 verified against LICENSE body; full source public; CLI/library with no gated tier (model providers need their own API keys, external to the tool).", "sources": [ { "url": "https://github.com/simonw/llm/blob/main/LICENSE", "shows": "Apache License 2.0 full text", "accessed": "2026-06-19" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "Established CLI/library distributed on PyPI (package 'llm', v0.31) with ~12.1k GitHub stars and a broad plugin ecosystem; widely used across Simon Willison's tooling. No exact monthly download figure captured, so placed at 3 on PyPI-presence + ecosystem signal.", "sources": [ { "url": "https://pypi.org/project/llm", "shows": "llm v0.31 published ~Apr 2026", "accessed": "2026-06-19" }, { "url": "https://github.com/simonw/llm", "shows": "~12.1k stars, plugin ecosystem", "accessed": "2026-06-19" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Model breadth: 100+ remote and local models via plugins; embeddings; prompt templates; SQLite logging of prompts/responses; Python API. CLI/library surface, not a full chat UI.", "confidence": "high", "note": "Strong on model breadth and extensibility, but a developer CLI/library rather than a multi-user surface; lacks the built-in RAG/multimodal/auth feature stack of top ui_api entries, so scored 3.", "sources": [ { "url": "https://llm.datasette.io/", "shows": "docs: unified CLI/Python API for remote+local models, plugins, embeddings, templates, SQLite logging", "accessed": "2026-06-19" } ] }, "maturity": 3.0, "mature": false, "version_note": "Apache-2.0 (verified LICENSE body). Latest release v0.31 (~24 Apr 2026, PyPI/GitHub). ~12.1k stars. Canonical repo under the personal simonw account; branded under the Datasette ecosystem. Verified 2026-06-19." } ] }, "telemetry_observability": { "label": "Telemetry & observability", "arc": "Product / UX", "layer": "product_ux", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "Phoenix", "org": "Arize AI", "type": "software", "description": "Open-source AI observability and evaluation toolkit by Arize AI. Provides tracing, embeddings analysis, experiment tracking, and LLM evaluation in a local-first workflow. Strong integrations with LlamaIndex and LangChain. Built by the team behind the Arize production ML monitoring platform. 9.8K GitHub stars.", "openness": { "score": 3, "class": "source_available", "components": "license:Elastic-License-2.0(ELv2, non-OSI: prohibits offering as a managed/hosted service to third parties);source:public;self-hostable;commercial:Arize-AX-SaaS-is-separate-closed-product", "confidence": "high", "note": "LICENSE file is ELv2 verbatim, not an OSI license; Arize markets Phoenix as 'the open source platform' (open_washed). Source-available + self-hostable, but not OSI open_source.", "sources": [ { "url": "https://github.com/Arize-ai/phoenix/blob/main/LICENSE", "shows": "Elastic License 2.0 (ELv2) full text", "accessed": "2026-06-04" }, { "url": "https://arize.com/phoenix/", "shows": "marketed as 'the open source platform'; 'ELv2 licensed. 9k+ GitHub stars'", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "arize-phoenix ~2.46M PyPI downloads/month; broad OTel-based framework integrations; ~10k GitHub stars corroborates only.", "sources": [ { "url": "https://pypistats.org/packages/arize-phoenix", "shows": "~2,458,129 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/Arize-ai/phoenix", "shows": "~10k stars; OTel tracing, evals, datasets, prompt mgmt, playground", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing(OTel-native), evals(LLM-as-judge response+retrieval), datasets/experiments, prompt management+playground, cost/latency, broad framework integrations (OpenAI/Claude/LangGraph/LlamaIndex/CrewAI)", "confidence": "high", "note": "Full-coverage observability/eval platform across all six feature-matrix dimensions; on par with MLflow/Langfuse anchors (C4).", "sources": [ { "url": "https://github.com/Arize-ai/phoenix", "shows": "tracing, evals, datasets, prompt management, playground, OTel integrations", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "arize-phoenix v17.2.0 (released 3 Jun 2026, GitHub); confirmed live June 2026. Open-source-positioned companion to the closed Arize AX platform." }, { "product": "Arize", "org": "Arize AI", "type": "software", "description": "Cloud-native ML and LLM observability platform (the commercial offering complementing the open source Phoenix). Production monitoring for model performance, data drift, embedding drift, and LLM traces at enterprise scale. Supports real-time alerting and root cause analysis. Raised $62M+ in funding. The closed counterpart to Phoenix, same team, enterprise-grade features.", "openness": { "score": 1, "class": "closed", "components": "weights:n/a;source:closed(proprietary SaaS);self-host:none for core AX features;note:open source Phoenix is a separate companion product, not the AX platform", "confidence": "high", "note": "Arize AX is closed/proprietary SaaS with no self-hostable core; only the separate Phoenix project is source-available.", "sources": [ { "url": "https://arize.com/", "shows": "'enterprise AI engineering platform'; closed AX with proprietary Alyx agent + adb datastore; Phoenix described as the separate self-hostable OSS counterpart", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Vendor discloses ~1 trillion spans/month and ~1 billion evals/month plus named enterprise customers (DoorDash, Roblox, PepsiCo, Atlassian, Booking.com, Reddit); no per-account user count, so reported_traction rather than active_users.", "sources": [ { "url": "https://arize.com/", "shows": "'1 trillion spans per month', '1 billion evals per month', named customers", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "observe(agent trace) + evaluate(spans/traces/sessions) + improve(prompt/config testing), Alyx embedded AI engineering agent, adb GenAI datastore, alerting/monitoring at trillion-span scale", "confidence": "medium", "note": "Frontier-tier closed observability platform for this category; enterprise-scale managed tracing/eval/monitoring exceeding the OSS feature sets. A 5 reflects category-leading closed capability.", "sources": [ { "url": "https://arize.com/", "shows": "Observe/Evaluate/Improve workflow, Alyx, adb, trillion-span scale", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": false, "version_note": "Arize AX, the enterprise AI engineering / LLM observability SaaS platform (distinct from open source Phoenix). Confirmed live via arize.com June 2026." }, { "product": "OpenLLMetry", "org": "Traceloop", "type": "software", "description": "OpenTelemetry-based instrumentation library for LLM applications. Auto-instruments LangChain, LlamaIndex, OpenAI SDK, and other popular frameworks. Exports telemetry data to any OTel-compatible backend (Jaeger, Grafana, Datadog, etc). The 'OpenTelemetry for LLMs' approach, standards-based, vendor-neutral. 7.1K GitHub stars.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no-feature-gated-core(SDK is fully OSS; Traceloop dashboard is an optional external export target, not gated core)", "confidence": "high", "note": "Apache-2.0 instrumentation library; exports to any OTel backend (Datadog, Honeycomb, etc.), so the OSS is genuinely standalone.", "sources": [ { "url": "https://pypi.org/project/traceloop-sdk/", "shows": "License: Apache-2.0; v0.61.0 (31 May 2026)", "accessed": "2026-06-04" }, { "url": "https://github.com/traceloop/openllmetry", "shows": "Apache-2.0; OTel-based GenAI instrumentation", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "traceloop-sdk ~3.15M PyPI downloads/month; 15+ LLM providers, 7 vector DBs, 25+ tested export destinations; ~7.2k stars corroborates only.", "sources": [ { "url": "https://pypistats.org/packages/traceloop-sdk", "shows": "~3,149,787 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/traceloop/openllmetry", "shows": "~7.2k stars; 15+ providers, 25+ destinations", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "deep OTel instrumentation(15+ LLM providers, 7 vector DBs, framework hooks), multi-destination export(25+ platforms); NO native prompt management, evals/LLM-as-judge, or datasets/annotation in the OSS layer", "confidence": "high", "note": "Strong on tracing + integrations but narrow vs full platforms; it is an instrumentation layer, not an eval/prompt/dataset suite. Below the MLflow/Langfuse/Phoenix/Opik feature breadth (C4).", "sources": [ { "url": "https://github.com/traceloop/openllmetry", "shows": "instrumentation/integration-focused feature set; evals/prompt mgmt are Traceloop-platform features not OSS SDK", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "traceloop-sdk v0.61.0 (released 31 May 2026, PyPI/GitHub); confirmed live June 2026. OTel-based instrumentation SDK; Traceloop hosted backend is the optional commercial destination." }, { "product": "Opik", "org": "Comet ML", "type": "software", "description": "Open-source platform for debugging, evaluating, and monitoring LLM applications, RAG systems, and agentic workflows. Provides comprehensive tracing, automated evaluations, and production-ready dashboards. Built by Comet ML (established ML experiment tracking company). Fast-growing at 19.4K GitHub stars, the second most-starred open source LLM observability tool after Langfuse.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;self-hostable-full-stack;commercial:Comet-cloud-hosting(not a gated core)", "confidence": "high", "note": "Apache-2.0, full platform self-hostable; cloud is managed hosting rather than a feature-gated proprietary core.", "sources": [ { "url": "https://pypi.org/project/opik/", "shows": "License: Apache 2.0; v2.0.57 (4 Jun 2026)", "accessed": "2026-06-04" }, { "url": "https://github.com/comet-ml/opik", "shows": "Apache-2.0; self-hosted + cloud deployment", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "opik ~6.19M PyPI downloads/month (highest in this batch); 60-70+ framework integrations; platform reports 40M+ traces/day; ~19.4k stars corroborates only.", "sources": [ { "url": "https://pypistats.org/packages/opik", "shows": "~6,192,885 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/comet-ml/opik", "shows": "~19.4k stars; 70+ integrations; 40M+ traces/day", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing(OTel-native + 70+ integrations), evals(datasets/experiments, pytest CI/CD), LLM-as-judge metrics(hallucination/moderation/relevance/context-precision), prompt playground, datasets, cost tracking", "confidence": "high", "note": "Covers all six feature-matrix dimensions; on par with MLflow/Langfuse anchors (C4). Not scored 5; closed Arize AX leads category capability.", "sources": [ { "url": "https://github.com/comet-ml/opik", "shows": "tracing, evals, LLM-as-judge, prompt management, datasets, OTel", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "opik v2.0.57 (released 4 Jun 2026, PyPI/GitHub); confirmed live June 2026. Self-hostable OSS platform; Comet.com cloud is optional hosting." }, { "product": "AgentOps", "org": "AgentOps AI", "type": "software", "description": "Python SDK for AI agent monitoring, LLM cost tracking, and benchmarking. Purpose-built for the agent era, tracks multi-step agent sessions, tool calls, and costs across frameworks (CrewAI, Agno, OpenAI Agents SDK, LangChain, Autogen). Lightweight instrumentation with two-line setup. 5.6K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(OSI, SDK);source:public(SDK + self-hostable app dir);commercial:app.agentops.ai-SaaS-dashboard-is-the-primary-product", "confidence": "medium", "note": "SDK is MIT/OSI and self-host is possible, but the hosted dashboard is the product surface, closer to open_core than pure open_source. Scored 4.", "sources": [ { "url": "https://github.com/AgentOps-AI/agentops", "shows": "MIT-licensed SDK; complementary SaaS dashboard at app.agentops.ai; self-host option", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "agentops ~585k PyPI downloads/month; multi-framework integrations (CrewAI, AG2, OpenAI Agents SDK, LangChain, Camel); ~5.6k stars corroborates only. 100K-1M monthly-download band.", "sources": [ { "url": "https://pypistats.org/packages/agentops", "shows": "~584,603 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/AgentOps-AI/agentops", "shows": "~5.6k stars; agent tracing + cost tracking + integrations", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "agent execution tracing + session replays, LLM cost tracking, multi-framework integrations, monitoring/debug dashboards, custom eval metrics; lighter on prompt versioning + dataset/annotation than full platforms", "confidence": "medium", "note": "Solid agent-tracing + cost + basic evals, but narrower feature matrix than MLflow/Langfuse/Opik/Phoenix; stale SDK (no 2026 release) reinforces a mid score.", "sources": [ { "url": "https://github.com/AgentOps-AI/agentops", "shows": "tracing, session replays, cost tracking, eval metrics, integrations", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "agentops v0.4.21 (last release 29 Aug 2025, PyPI), no 2026 release at scoring time (>9 months stale). SDK MIT-licensed; primary product is the app.agentops.ai SaaS dashboard." }, { "product": "Agenta", "org": "Agenta AI", "type": "software", "description": "Open-source LLMOps platform combining prompt playground, prompt management, LLM evaluation, and observability in a single tool. Covers the full lifecycle from prompt engineering through production monitoring. Self-hostable with a cloud option. 4.1K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(OSI, core);source:public;self-hostable;commercial:Agenta-Cloud-managed-tier(free tier available)", "confidence": "high", "note": "MIT/OSI core, self-hostable via GitHub, with a managed cloud tier, open_core. (Borderline open_source; scored open_core for the hosted commercial tier.)", "sources": [ { "url": "https://github.com/Agenta-AI/agenta", "shows": "MIT license; self-host + Agenta Cloud managed tier", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "high", "note": "agenta ~16k PyPI downloads/month (10K-100K band); ~4.2k GitHub stars corroborates only. Early-adopter scale, smallest in this batch.", "sources": [ { "url": "https://pypistats.org/packages/agenta", "shows": "~16,370 downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/Agenta-AI/agenta", "shows": "~4.2k stars; LLMOps platform", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "prompt management+versioning(branching/environments, 50+ models), evals(20+ evaluators, UI+API, test cases from prod/CSV), observability/tracing(OTel-native, OpenLLMetry/OpenInference compatible, cost/latency), multi-model playground", "confidence": "high", "note": "Broad feature matrix across prompt mgmt, evals, and OTel observability, comparable to Langfuse anchor (C4). Capability is decoupled from its lower adoption.", "sources": [ { "url": "https://github.com/Agenta-AI/agenta", "shows": "prompt management, 20+ evaluators, OTel tracing, playground", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "agenta v0.101.1 (released 4 Jun 2026, GitHub); confirmed live June 2026. LLMOps platform; self-hostable OSS + Agenta Cloud managed tier." }, { "product": "Laminar", "org": "Laminar AI", "type": "software", "description": "Open-source observability platform purpose-built for AI agents. Provides tracing, evaluation, and analytics for agentic workflows with an emphasis on multi-step agent debugging. YC S24. Rust-based backend for high performance. 2.9K GitHub stars and growing rapidly.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Rust core + TS UI);SDKs:Apache-2.0;no feature-gated core observed in repo;managed Laminar Cloud is hosting not a gated tier", "confidence": "medium", "note": "Repo and SDKs Apache-2.0 with full source; a hosted cloud exists but no enterprise-gated module split was visible in the repo (less explicit open-core boundary than Langfuse/LangWatch).", "sources": [ { "url": "https://github.com/lmnr-ai/lmnr", "shows": "Apache-2.0 license; Rust+TS source public; OTel-native tracing/evals/datasets features", "accessed": "2026-06-04" }, { "url": "https://pypi.org/project/lmnr/", "shows": "lmnr v0.7.52 (27 May 2026), open source LLM engineering platform SDK", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "low", "note": "pypistats reports ~17M lmnr downloads/mo, but this is anomalously high for a ~3k-star project and almost certainly CI/mirror-inflated (common for OTel auto-instrumentation SDKs pulled transitively); discounting to level 3 rather than treating 17M as honest end-user reach. GitHub ~3k stars corroborates a smaller real user base. Treat as directional.", "sources": [ { "url": "https://pypistats.org/packages/lmnr", "shows": "~17.06M lmnr downloads in last month (flagged as likely mirror-inflated)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Has: OTel-native tracing (Vercel AI SDK, Browser Use, LangChain, OpenAI/Anthropic/Gemini); evals (SDK+CLI, CI/CD); datasets + annotation UI; SQL query over traces; dashboards. Lighter on: prompt versioning/management and explicit cost tracking (not surfaced in repo).", "confidence": "medium", "note": "Solid tracing+evals+datasets coverage with a differentiated SQL/dashboard layer; below Langfuse/MLflow (C4 anchors) on prompt management and cost-tracking breadth, hence 3.", "sources": [ { "url": "https://github.com/lmnr-ai/lmnr", "shows": "feature list: tracing, evals SDK/CLI, datasets/annotation, SQL editor, dashboards", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "lmnr Python SDK v0.7.52 (27 May 2026); platform/repo lmnr-ai/lmnr at v0.1.46 (31 May 2026). Confirmed live on GitHub + PyPI June 2026." }, { "product": "OpenLIT", "org": "OpenLIT", "type": "software", "description": "OpenTelemetry-native AI engineering platform covering LLM observability, GPU monitoring, guardrails, evaluations, and prompt management. Integrates with 50+ LLM providers, vector databases, and agent frameworks. Vault for managing API keys. Differentiated by breadth of integrations and GPU-level monitoring. 2.5K GitHub stars.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;OTel-native SDKs(Python/TS/Go);no enterprise-gated core observed", "confidence": "high", "note": "Fully Apache-2.0, OpenTelemetry-native, no feature-gated core; vendor-neutral by design (exports to any OTel backend).", "sources": [ { "url": "https://github.com/openlit/openlit", "shows": "Apache-2.0 license; OTel-native SDKs; full feature source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~590K openlit PyPI downloads/mo; modest but real. As an OTel auto-instrumentation lib some downloads are CI/transitive, so placed conservatively at level 2 (early adopters). ~2.5k GitHub stars corroborates a smaller community vs Langfuse/MLflow.", "sources": [ { "url": "https://pypistats.org/packages/openlit", "shows": "~590,377 openlit downloads in last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Has: OTel-native tracing+metrics (LLMs, vector DBs, GPUs); 11 built-in eval types (hallucination/bias/toxicity); Prompt Hub (versioning); cost tracking incl custom/fine-tuned pricing; guardrails; secrets Vault; 50+ provider/framework integrations.", "confidence": "high", "note": "Broad feature matrix covering all six recipe dimensions (tracing, prompt mgmt, evals, datasets/annotation-adjacent, cost, OTel+integrations) plus GPU observability; comparable to Langfuse C4. Held at 4 (not 5) as no observability product is a clear frontier-definer over MLflow/Langfuse anchors.", "sources": [ { "url": "https://github.com/openlit/openlit", "shows": "feature list: tracing/metrics, 11 eval types, Prompt Hub, cost tracking, guardrails, vault, 50+ integrations", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "openlit PyPI v1.42.0 (8 May 2026); OTel GPU Collector 0.0.6 (3 Jun 2026). Confirmed live on GitHub + PyPI June 2026." }, { "product": "Langtrace", "org": "Scale3 Labs", "type": "software", "description": "Open-source, OpenTelemetry-based end-to-end observability tool for LLM applications. Provides real-time tracing, evaluations, and metrics for popular LLMs, frameworks, and vector databases. TypeScript and Python SDKs. Differentiated by strict OTel standards compliance and support for vector DB tracing. 1.2K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:app=AGPL-3.0(OSI, copyleft);SDKs=Apache-2.0(OSI);source:public;OTel-built;hosted Langtrace Cloud tier exists", "confidence": "high", "note": "App is AGPL-3.0 (OSI-approved but strong copyleft, which deters proprietary embedding) and SDKs Apache-2.0; a managed cloud tier exists. Scored open_core/4 rather than 5 given the AGPL copyleft + hosted commercial offering.", "sources": [ { "url": "https://github.com/Scale3-Labs/langtrace", "shows": "AGPL-3.0 app license, Apache-2.0 SDKs, OTel-based observability", "accessed": "2026-06-04" }, { "url": "https://github.com/Scale3-Labs/langtrace/releases", "shows": "app v4.0.11 release 17 Apr 2026 (active)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~34.5K langtrace-python-sdk downloads/mo; ~1.2k GitHub stars. Real but small footprint; early-adopter tier.", "sources": [ { "url": "https://pypistats.org/packages/langtrace-python-sdk", "shows": "~34,497 langtrace-python-sdk downloads in last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Has: OTel-native real-time tracing (LLMs, frameworks, vector DBs); evaluations; latency/cost/usage analytics; broad integrations (OpenAI/Anthropic/Cohere, LangChain/LlamaIndex, Pinecone/Chroma/Weaviate). Lighter/unclear on: prompt versioning and dataset/annotation workflows.", "confidence": "medium", "note": "Strong tracing+cost+integration breadth but thinner prompt-mgmt/datasets surface than Langfuse C4; scored 3.", "sources": [ { "url": "https://github.com/Scale3-Labs/langtrace", "shows": "feature list: tracing, evaluations, cost/latency analytics, OTel, wide integrations", "accessed": "2026-06-04" } ] }, "maturity": 2.5, "mature": false, "version_note": "Langtrace app v4.0.11 (17 Apr 2026); langtrace-python-sdk v3.8.12 (8 Apr 2026). Active and maintained as of June 2026 (initial fetch misread release year; corrected via release pages)." }, { "product": "LangWatch", "org": "LangWatch", "type": "software", "description": "Open-source platform for LLM evaluations and AI agent testing with production monitoring capabilities. Combines observability traces with automated quality checks and guardrails. Provides dashboards for cost, latency, and quality metrics. Positioned at the intersection of evaluation and production observability. 3.3K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(core)+MIT(SDKs);enterprise modules(SCIM,audit-logs,license/billing) under langwatch/ee/ require commercial license for production;source:public", "confidence": "high", "note": "Explicit open-core: Apache-2.0 core + MIT SDKs, with an /ee enterprise directory (SCIM, audit logs, billing) commercially licensed, same pattern as the Langfuse O4 anchor.", "sources": [ { "url": "https://github.com/langwatch/langwatch", "shows": "Apache-2.0 core, MIT SDKs, langwatch/ee/ enterprise modules requiring commercial license", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "No verified download/active-user figure located this run; ~3.3k GitHub stars is the only honest signal, so capped at level 2 via stars fallback (cannot exceed 3 on stars regardless).", "sources": [ { "url": "https://github.com/langwatch/langwatch", "shows": "~3.3k GitHub stars; no disclosed usage numbers", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Has: OTel/OTLP-native tracing; LLM evals (offline + production monitoring); agent simulations/scenario testing; prompt management with GitHub integration; dataset creation + annotation; AI Gateway (cost control, failover); broad framework/provider integrations.", "confidence": "high", "note": "Full feature matrix across all six recipe dimensions plus distinctive agent-simulation/scenario testing and a built-in gateway; on par with Langfuse C4. Held at 4 (no frontier-definer claim over the anchors).", "sources": [ { "url": "https://github.com/langwatch/langwatch", "shows": "feature list: tracing, evals, simulations, prompt mgmt, datasets/annotation, AI Gateway, OTel", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "langwatch/langwatch monorepo active June 2026 (skills component v0.6.0, 3 Jun 2026). Confirmed live on GitHub." }, { "product": "LangSmith", "org": "LangChain", "type": "software", "description": "Proprietary observability and testing platform from the LangChain team. Provides tracing, datasets, evaluations, prompt hub, and annotation queues. Tightly integrated with the LangChain ecosystem but supports any LLM application. The most widely adopted LLM observability tool, benefiting from LangChain's massive developer community. Closed-source SaaS with no self-hosted option.", "openness": { "score": 1, "class": "closed", "components": "platform:proprietary(no public source for LangSmith itself);self-hosting=licensed enterprise deployment of closed software, not OSS;OSS siblings(LangChain/LangGraph) are separate products not scored here", "confidence": "high", "note": "LangSmith the observability/eval platform is closed/proprietary; only the LangChain/LangGraph frameworks (separate registry entries) are open. Self-host/BYOC are licensed deployments of closed software, not source-available.", "sources": [ { "url": "https://www.langchain.com/langsmith", "shows": "LangSmith described as managed SaaS with BYOC/self-host enterprise options; open source items are LangChain/LangGraph frameworks, not LangSmith", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Default observability/eval platform for the very widely used LangChain ecosystem; vendor lists 25+ named enterprise customers (Klarna, Lyft, Cloudflare, LinkedIn, Coinbase, Nvidia, Workday, etc.). No quantitative user count disclosed, so reported_traction; placed at 4 on ecosystem reach + enterprise roster (directional).", "sources": [ { "url": "https://www.langchain.com/langsmith", "shows": "25+ named enterprise customers; positioned as the LangChain-ecosystem observability platform", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Has: framework-native tracing (Python/TS/Go/Java SDKs); monitoring with cost tracking, LLM-as-judge evals, alerts; evaluation/scoring; prompt + dataset workflows; deployment/fleet/sandbox tooling for agents.", "confidence": "medium", "note": "Mature, broad feature matrix across tracing/evals/prompt/datasets/cost plus agent deployment surface; comparable to the C4 anchors. Closed-source, but capability is assessed independently of openness.", "sources": [ { "url": "https://www.langchain.com/langsmith", "shows": "feature list: tracing, monitoring/cost, LLM-as-judge evals, evaluation, deployment/fleet/sandboxes", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "LangSmith proprietary SaaS by LangChain; managed cloud + BYOC + self-hosted enterprise deployment. Confirmed live (langchain.com/langsmith) June 2026." }, { "product": "Helicone", "org": "Helicone", "type": "software", "description": "LLM proxy that captures logs, costs, and latency with a one-line integration (swap your API base URL). Clean dashboard for monitoring request volumes, token usage, costs, and error rates. Positioned as the lowest-friction entry point for LLM monitoring, no SDK needed, just a proxy URL change. YC W23. Code is open source (Apache-2.0) but primarily used as a cloud service.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI);source:public;self-host supported;hosted Helicone Cloud with paid tiers(free=10k req/mo) = commercial SaaS layer atop OSS core", "confidence": "high", "note": "Apache-2.0 OSS core with a hosted commercial cloud (tiered pricing), classic open-core, same shelf as Langfuse/LangWatch (O4).", "sources": [ { "url": "https://github.com/Helicone/helicone", "shows": "Apache-2.0 license; full source; hosted cloud paid tiers + self-host", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No verified request/customer count disclosed (site says 'world's fastest-growing AI companies rely on Helicone' but no numbers); ~5.8k GitHub stars is the only honest signal, so capped at level 3 via stars fallback. Treat as directional.", "sources": [ { "url": "https://github.com/Helicone/helicone", "shows": "~5.8k GitHub stars; no disclosed usage figures", "accessed": "2026-06-04" }, { "url": "https://www.helicone.ai/", "shows": "no quantitative usage/customer numbers disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Has: observability + agent tracing; prompt management (version using production data); cost/latency/quality analytics; LLM routing via AI Gateway (100+ models) with automatic fallbacks; datasets & fine-tuning; caching.", "confidence": "high", "note": "Broad feature matrix with a strong gateway/routing + caching angle on top of tracing, prompt mgmt, cost, and datasets; on par with Langfuse C4. Held at 4 (no frontier-definer claim over anchors).", "sources": [ { "url": "https://github.com/Helicone/helicone", "shows": "feature list: observability/agent tracing, prompt mgmt, cost analytics, AI Gateway/routing, datasets/fine-tuning, fallbacks", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Helicone/helicone monorepo active June 2026 (5,477 commits on main). Confirmed live on GitHub; hosted cloud + self-host." }, { "product": "Datadog LLM Observability", "org": "Datadog", "type": "software", "description": "Enterprise LLM monitoring integrated into the Datadog APM platform. Traces LLM calls alongside existing infrastructure, application, and security monitoring. For organizations already using Datadog who want unified observability across their entire stack including AI/LLM components. Leverages Datadog's massive enterprise installed base.", "openness": { "score": 1, "class": "closed", "components": "platform:proprietary SaaS(no source);only the ddtrace instrumentation SDK is open;product itself closed", "confidence": "high", "note": "Closed proprietary SaaS; only Datadog's tracing client libraries are open, the LLM Observability product and backend are closed.", "sources": [ { "url": "https://docs.datadoghq.com/llm_observability/", "shows": "Datadog LLM Observability described as a proprietary cloud SaaS platform for monitoring LLM apps/agents", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "low", "note": "Rides the large installed base of Datadog APM (a major enterprise observability vendor); LLM Observability is a newer add-on with no disclosed standalone user/usage count. Placed at 4 on parent-platform enterprise reach as a directional estimate, not a measured figure for this module specifically.", "sources": [ { "url": "https://docs.datadoghq.com/llm_observability/", "shows": "shipped product within the Datadog enterprise platform; no standalone usage numbers in docs", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Has: end-to-end trace capture (tokens, errors, latency, workflows); built-in + custom evaluations; cost/latency/usage dashboards; sensitive-data scan/redaction + prompt-injection detection; topic clustering (Patterns); anomaly detection (Insights); framework integrations (OpenAI, LangChain, Bedrock, Anthropic) via auto-instrumentation.", "confidence": "medium", "note": "Broad enterprise-grade feature matrix (tracing, evals, cost, security/redaction, clustering, anomaly detection) integrated with the wider Datadog APM suite; comparable to C4 anchors. Closed, but capability scored independently.", "sources": [ { "url": "https://docs.datadoghq.com/llm_observability/", "shows": "feature list: tracing, evaluations, cost dashboards, data redaction/prompt-injection detection, Patterns clustering, Insights anomaly detection", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Datadog LLM Observability, proprietary SaaS module of the Datadog platform. Confirmed live (docs.datadoghq.com/llm_observability) June 2026." }, { "product": "Weave", "org": "Weights & Biases", "type": "software", "description": "Toolkit for developing and monitoring AI-powered applications, built by Weights & Biases. Provides tracing, evaluation, and dataset management for LLM applications. Leverages W&B's established ML experiment tracking platform and enterprise customer base. Code is open source (Apache-2.0) but primarily consumed via the W&B cloud platform. 1.1K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI, SDK/client repo wandb/weave);source:public;backend:commercial(W&B SaaS platform hosts traces/evals; full platform is proprietary, self-host is enterprise);eval+tracing-client:OSS", "confidence": "high", "note": "Apache-2.0 OSS instrumentation SDK (@weave.op tracing + eval scoring) but the trace store / dashboards / monitoring backend is the proprietary W&B SaaS platform, open-core, calibrated below MLflow O5 (fully-OSS core) and at/with Langfuse O4 (OSS core + commercial backend).", "sources": [ { "url": "https://github.com/wandb/weave", "shows": "Apache-2.0 license on the Weave SDK repo; LLM logging/tracing/eval toolkit", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~1.03M PyPI downloads in the last month for the `weave` package (primary: pypistats); rides the established W&B install base (ML teams adding LLM observability without leaving W&B). 1-10M monthly-download band.", "sources": [ { "url": "https://pypistats.org/packages/weave", "shows": "1,028,914 downloads last month for the weave package", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing(auto-logs OpenAI/Anthropic calls incl inputs/outputs/tokens/cost); evals(LLM-as-judge + code scorers, experiments vs datasets); datasets/annotation; cost tracking; integrations(OpenAI/Anthropic + framework hooks). Missing dedicated prompt-management/versioning depth vs MLflow/Langfuse.", "confidence": "medium", "note": "Competent end-to-end tracing+eval suite; rated 4 = at the MLflow/Langfuse anchor tier on feature breadth, just short of frontier (lighter prompt-versioning + OTel story than Langfuse/MLflow per 2026 comparisons).", "sources": [ { "url": "https://github.com/wandb/weave", "shows": "logging/debugging LLM traces + rigorous evaluations feature set", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "W&B Weave; OSS SDK v0.52.42 (Jun 2, 2026) confirmed live on GitHub (wandb/weave). LLM tracing + evaluation + monitoring layered onto the commercial W&B platform." }, { "product": "Braintrust", "org": "Braintrust", "type": "software", "description": "Enterprise AI product development platform with logging, tracing, evaluation, and prompt playground. Focuses on the eval-to-production workflow, iterate on prompts, evaluate with custom scorers, deploy with confidence, monitor in production. Strong enterprise positioning with SOC 2 compliance. Funded by a16z.", "openness": { "score": 1, "class": "closed", "components": "license:Apache-2.0(OSI, client SDKs only, Python/TS/Go/Ruby/C#);platform:proprietary(Brainstore custom DB, evals/tracing/automation all SaaS);self-host:enterprise-only(hybrid)", "confidence": "high", "note": "Open SDKs but a closed SaaS core (Brainstore backend, dashboards, automation are proprietary); no OSS core, so 'closed' (not open_core) like LangSmith/Galileo per recipe's closed examples. The Apache-2.0 client SDK is instrumentation only and does not open the product.", "sources": [ { "url": "https://github.com/braintrustdata/braintrust-sdk", "shows": "Apache-2.0 license on client SDK; platform itself at braintrust.dev", "accessed": "2026-06-04" }, { "url": "https://www.braintrust.dev/", "shows": "proprietary backend (Brainstore custom DB), SaaS + hybrid enterprise deployment", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~6.08M PyPI downloads in the last month for the `braintrust` package (primary: pypistats); named production users incl. Vercel, Notion, Coursera, Dropbox, Replit. 1-10M monthly-download band; strongest adoption of the closed-platform set here.", "sources": [ { "url": "https://pypistats.org/packages/braintrust", "shows": "6,080,857 downloads last month for the braintrust package", "accessed": "2026-06-04" }, { "url": "https://www.braintrust.dev/", "shows": "named customers Vercel, Notion, Coursera, Dropbox, Replit", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "tracing(real-time inspection across millions of logs, tool calls); evals(LLM/code/human scorers, experiments vs datasets); prompt mgmt + playground; automation(Topics pattern discovery, continuous scoring, quality gates blocking releases); datasets/annotation; OTel + MCP + framework-agnostic SDKs(5 langs). Quality-gate/automation depth exceeds the anchor tier.", "confidence": "high", "note": "Deepest eval+automation feature surface in this batch (CI-style quality gates, Topics auto-clustering, continuous scoring), a frontier-definer for eval-first LLM observability; rated 5 above the MLflow/Langfuse C4 anchors on feature breadth.", "sources": [ { "url": "https://www.braintrust.dev/", "shows": "tracing, evals (LLM/code/human), automation (Topics, quality gates), prompt mgmt, OTel/MCP integrations", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": false, "version_note": "Braintrust AI eval/observability platform (braintrust.dev); SDK v3.16.0 (Jun 3, 2026, Apache-2.0) confirmed live; platform backend (Brainstore) proprietary SaaS with hybrid enterprise deploy." }, { "product": "PromptLayer", "org": "PromptLayer", "type": "software", "description": "Prompt management and monitoring platform that logs every LLM API request. Maintains a complete history of prompts, responses, costs, and latency. Enables prompt versioning, A/B testing, and analytics. One of the earliest LLM monitoring tools (launched early 2023), focused on the prompt lifecycle rather than full tracing. 762 GitHub stars on the client library.", "openness": { "score": 1, "class": "closed", "components": "license:Apache-2.0(OSI, client library MagnivOrg/prompt-layer-library only);platform:proprietary-SaaS(API-key gated, hosted prompt registry/logging/evals);self-host:no(managed only)", "confidence": "high", "note": "Open client SDK over a closed hosted backend (prompt CMS, logging, evals are SaaS), no OSS core, so 'closed' alongside the Braintrust/LangSmith pattern; the open library is instrumentation only.", "sources": [ { "url": "https://github.com/MagnivOrg/prompt-layer-library", "shows": "Apache-2.0 official PromptLayer Python library; platform at promptlayer.com requiring API key", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "~289K PyPI downloads in the last month for the `promptlayer` package (primary: pypistats) → 100K-1M band. Recognized but lighter-weight (no-code prompt CMS for non-technical teams) vs the deeper platforms; no harder user count disclosed.", "sources": [ { "url": "https://pypistats.org/packages/promptlayer", "shows": "289,237 downloads last month for the promptlayer package", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "prompt mgmt/versioning(registry-based CMS, visual no-code editor, A/B testing); tracing(logs every request/response); evals(basic, regression sets); cost/monitoring(narrower than full-stack). Missing: deep LLM-as-judge eval + broad OTel/framework observability vs anchors.", "confidence": "medium", "note": "Strong prompt-lifecycle/versioning focus but shallower eval + production observability than MLflow/Langfuse (C4) per 2026 comparisons, rated 3, mid-tier.", "sources": [ { "url": "https://github.com/MagnivOrg/prompt-layer-library", "shows": "version/test/monitor prompts and agents with evals, tracing, regression sets", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "PromptLayer prompt-management + logging/observability SaaS (promptlayer.com), founded 2021, active 2026. Official `promptlayer` Python lib (Apache-2.0, 764 stars, MagnivOrg) confirmed live; platform itself is a managed SaaS." }, { "product": "New Relic AI Monitoring", "org": "New Relic", "type": "software", "description": "Enterprise AI/LLM monitoring integrated into the New Relic observability platform. Traces LLM interactions, tracks token usage and costs, monitors model performance and error rates. Supports OpenAI, AWS Bedrock, and other providers. For organizations already on New Relic who want AI observability unified with their existing APM, infrastructure, and log monitoring.", "openness": { "score": 1, "class": "closed", "components": "platform:proprietary-SaaS(closed);license:commercial(Standard/Pro/Enterprise tiers);ingestion:OpenTelemetry-compatible(open standard, not open product);source:closed", "confidence": "high", "note": "Fully proprietary SaaS observability platform; AI Monitoring is a commercial module. OTel ingestion support does not make the product open. Scored closed (1).", "sources": [ { "url": "https://newrelic.com/platform/ai-monitoring", "shows": "proprietary SaaS AI observability with Standard/Pro/Enterprise tiers; OTel ingestion", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "New Relic platform reported at ~6.6M platform users / 75,000+ businesses in 2026 (1-10M band). Figure is platform-wide, not AIM-specific, and the hard numbers came from aggregators (landbase/zoominfo); vendor product page confirms the AIM module and broad enterprise tiers but I did not fetch a primary user count; treat level as platform-attributed and directional.", "sources": [ { "url": "https://newrelic.com/platform/ai-monitoring", "shows": "AI Monitoring module of the broad New Relic observability platform; enterprise tiers", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing(full AI-stack distributed tracing, MCP request lifecycle, agent/tool call latency); cost(token usage tracking + alerts, model cost/perf/quality comparison); quality(bias/hallucination/toxicity detection, response analysis, user feedback); OTel + OpenAI/Pinecone/LangChain integrations; 'no new instrumentation'. Lacks dedicated prompt-versioning/dataset-eval depth of eval-first tools.", "confidence": "medium", "note": "Broad full-stack AI observability (traces, cost, quality, agentic spans) at the anchor C4 tier on feature breadth; not an eval/prompt-mgmt frontier-definer, so 4 not 5.", "sources": [ { "url": "https://newrelic.com/platform/ai-monitoring", "shows": "AI-stack tracing, token cost tracking, bias/hallucination/toxicity detection, OTel + LangChain/OpenAI/Pinecone integrations", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "New Relic AI Monitoring (AIM), the GenAI/LLM observability module of the proprietary New Relic platform; confirmed active 2026 (vendor product page + Feb 2026 platform announcement). Module of a general APM/observability suite." }, { "product": "Dynatrace AI Observability", "org": "Dynatrace", "type": "software", "description": "Enterprise AI observability integrated into the Dynatrace platform. Monitors LLM-powered applications with automatic distributed tracing, token tracking, cost analysis, and anomaly detection. Leverages Dynatrace's Davis AI engine for automated root cause analysis of LLM application issues. Targets large enterprises with existing Dynatrace deployments.", "openness": { "score": 1, "class": "closed", "components": "platform:proprietary-SaaS(Grail-powered, Davis AI);license:commercial;ingestion:OpenTelemetry+OpenLLMetry(via Traceloop)+GenAI-semconv(open standards, not open product);source:closed", "confidence": "high", "note": "Fully proprietary Dynatrace platform; AI Observability is a commercial app/module. OTel + OpenLLMetry ingestion are open standards consumed by a closed product. Scored closed (1).", "sources": [ { "url": "https://docs.dynatrace.com/docs/observe/dynatrace-for-ai-observability", "shows": "proprietary Dynatrace platform AI Observability; Grail/Davis AI; OTel + OpenLLMetry ingestion", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Dynatrace is a large public observability vendor, but I found no primary, AI-Observability-specific usage figure this run (vendor docs confirm the product exists but disclose no module adoption count). Declining to assign a level rather than attribute platform-wide numbers I could not source primarily.", "sources": [ { "url": "https://docs.dynatrace.com/docs/observe/dynatrace-for-ai-observability", "shows": "product exists and is part of the Dynatrace platform; no module usage count disclosed", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing(end-to-end prompt-flow tracing/debugging, root-cause, prompt latency + model-level metrics); cost(predictive cost mgmt via Davis AI, token/service-fee tracking, no-sampling full capture); OTel + OpenLLMetry(Traceloop) + GenAI semconv; providers OpenAI/Bedrock/NIM/Ollama/Gemini/Azure AI Foundry + LangChain/CrewAI. Lacks dedicated eval/LLM-as-judge + prompt-versioning depth.", "confidence": "medium", "note": "Strong full-stack AI observability (no-sampling capture, predictive cost, agentic tracing) at the anchor C4 tier; not an eval/prompt-mgmt frontier-definer, so 4.", "sources": [ { "url": "https://docs.dynatrace.com/docs/observe/dynatrace-for-ai-observability", "shows": "end-to-end tracing, token/cost metrics, predictive cost (Davis AI), OTel/OpenLLMetry, multi-provider + agentic framework support", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Dynatrace AI Observability, the GenAI/LLM/agentic observability capability of the proprietary Dynatrace platform; docs updated Jan 28, 2026 ('Latest Dynatrace') confirm it live. Module of a general full-stack APM suite." }, { "product": "WhyLabs", "org": "WhyLabs", "type": "software", "description": "AI observability platform for monitoring ML models and LLM applications in production. Provides data drift detection, performance monitoring, and LLM security guardrails. Built on the open source whylogs profiling library and LangKit LLM monitoring toolkit. Differentiated by statistical profiling approach, monitors distributions rather than individual requests, enabling cost-effective monitoring at scale.", "openness": { "score": 4, "class": "open_source", "components": "license:Apache-2.0(OSI, whylogs + LangKit + open sourced platform);source:public(github.com/whylabs);hosted-SaaS:discontinued;status:company-ceased-ops, platform open sourced on closure", "confidence": "high", "note": "Scored on what remains accessible: the OSS stack. whylogs (data logging) + LangKit (LLM monitoring) are Apache-2.0 OSI, and WhyLabs open sourced its platform on shutdown. Not 5 because the hosted monitoring backend is no longer maintained/operating and the OSS pieces are libraries rather than a turnkey live observability product.", "sources": [ { "url": "https://whylabs.ai/langkit", "shows": "'WhyLabs, Inc. is discontinuing operations'; platform/whylogs/LangKit open sourced", "accessed": "2026-06-04" }, { "url": "https://github.com/whylabs/whylogs", "shows": "whylogs Apache-2.0, 2.8k stars, public source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Hosted SaaS shut down (no active platform users). Remaining signal is the OSS libraries: whylogs ~2.8k GitHub stars, last release Dec 2024; stars_fallback caps at 3; placed at 2 given the dead commercial product and stalled release cadence. Honest signal is modest residual OSS usage, not a live platform.", "sources": [ { "url": "https://github.com/whylabs/whylogs", "shows": "2.8k stars, latest release v1.6.4 (Dec 3, 2024)", "accessed": "2026-06-04" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "whylogs(data logging/profiling, drift + data-quality monitoring, privacy-preserving) + LangKit(LLM text-quality/safety metrics: sentiment, toxicity, prompt-injection signals). No live tracing backend, no eval-experiment/prompt-mgmt platform (open sourced but unmaintained). Narrow vs anchor C4 tools.", "confidence": "medium", "note": "As living, usable capability it is a pair of monitoring libraries (drift + LLM safety metrics), not a full tracing/eval/prompt platform; with the hosted backend gone and releases stalled since Dec 2024, rated 2, well below the MLflow/Langfuse C4 anchors.", "sources": [ { "url": "https://github.com/whylabs/whylogs", "shows": "whylogs data-logging/drift/data-quality monitoring library feature set; release cadence stalled Dec 2024", "accessed": "2026-06-04" } ] }, "maturity": 2.0, "mature": false, "version_note": "WhyLabs ML/LLM observability. COMPANY DISCONTINUED: whylabs.ai states 'WhyLabs, Inc. is discontinuing operations' and open sourced the full platform plus whylogs and LangKit. The hosted SaaS observability platform is no longer operating; the OSS libraries remain on GitHub (whylogs Apache-2.0, 2.8k stars, latest release v1.6.4 Dec 2024)." }, { "product": "Galileo", "org": "Galileo", "type": "software", "description": "AI observability and reliability platform that combines offline evals, online production guardrails, and tracing, powered by their proprietary Luna-2 small judge models that detect hallucinations, tool errors, and policy violations at claimed 97% lower cost than LLM-as-judge. Positions as 'don't just monitor AI failures, stop them' and is consistently ranked in top-5 LLM observability platforms (Maxim, Firecrawl roundups). Series B-funded, with enterprise customers across financial services and healthcare.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary;source:closed;deploy:SaaS/VPC/on-prem(managed);no-OSS-core(open source contributions exist as separate tools/benchmarks, not the product)", "confidence": "high", "note": "Proprietary platform; recipe lists Galileo explicitly as a closed example. No OSI-licensed core product.", "sources": [ { "url": "https://galileo.ai/", "shows": "proprietary platform; SaaS/VPC/on-prem deploy, no open source core product", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Vendor states 'trusted by leading enterprises' but discloses no named customers or quantitative user/usage figures on the homepage. Level 3 reflects established commercial traction without a verified count; treat as directional.", "sources": [ { "url": "https://galileo.ai/", "shows": "'Trusted by leading enterprises' marketing claim, no quantitative figures", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing (agent behavior/failure-mode analysis) + evals (20+ out-of-box for RAG/agents/safety/security + custom) + production guardrails + cost optimization (Luna-2 eval models, sub-200ms, ~$0.02/M tokens) + data capture (synthetic/dev/prod datasets). Distinctive guardrail layer turning offline evals into runtime intervention.", "confidence": "medium", "note": "Strong feature coverage across the recipe matrix; the offline-eval-to-production-guardrail loop and purpose-built Luna eval models are differentiators. OTel support not confirmed on the homepage. Capped at 4 (MLflow/Langfuse anchors hold C4); not a frontier-definer over the OSS anchors.", "sources": [ { "url": "https://galileo.ai/", "shows": "evals, guardrails, tracing, cost optimization (Luna), data capture features", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Galileo AI observability & eval-engineering platform; verified live June 2026 at galileo.ai. Proprietary SaaS / VPC / on-prem; Luna-2 small eval models. Not open source." }, { "product": "Confident AI", "org": "Confident AI", "type": "software", "description": "Hosted SaaS layered on the open source DeepEval library, adds dataset management, regression tracking, human annotation, custom dashboards, online evals against live traffic, alerting, and incident response. DeepEval itself runs 2M evals/day with 3M monthly downloads and 12.6K GitHub stars; enterprise customers on the cloud product include Microsoft, AstraZeneca, AXA, and BCG. YC W25, $2.2M seed.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(platform);source:closed(platform);note:companion OSS framework DeepEval is Apache-2.0 but is a separate registry-scope product;self-host:enterprise-tier", "confidence": "high", "note": "The scored entity is the Confident AI platform itself, which is proprietary SaaS. DeepEval (the open Apache-2.0 framework, ~15.9k stars) is a distinct product; do not credit the platform with the framework's openness.", "sources": [ { "url": "https://www.confident-ai.com/", "shows": "proprietary cloud platform distinct from open source DeepEval; self-hosted enterprise option", "accessed": "2026-06-04" }, { "url": "https://github.com/confident-ai/deepeval", "shows": "DeepEval (the separate framework) is Apache-2.0; platform is the integrated paid companion", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Vendor discloses '500+ leading AI companies' incl. Panasonic, Samsung, Epic Games, Humach. Corroborated by the strong pull of its open source funnel DeepEval (~15.9k GitHub stars, used-by 1.3k repos), which drives platform signups. Level 3 on disclosed named-customer traction; no precise active-user/usage-volume number for the platform itself.", "sources": [ { "url": "https://www.confident-ai.com/", "shows": "'500+ leading AI companies' incl Panasonic, Samsung, Epic Games, Humach", "accessed": "2026-06-04" }, { "url": "https://github.com/confident-ai/deepeval", "shows": "~15.9k stars, used-by 1.3k repos (OSS funnel corroboration)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "tracing (full prod call capture: inputs/outputs/tool calls/latency/token cost) + evals (50+ research-backed DeepEval metrics incl G-Eval/LLM-as-judge) + dataset mgmt (auto-curation of traces) + prompt mgmt (git-based versioning/branching) + monitoring/alerts (regression detection) + red teaming (OWASP) + annotations. Covers the full recipe matrix.", "confidence": "medium", "note": "Broad coverage across all recipe dimensions; the deep DeepEval metric library (50+ incl LLM-as-judge) and red-teaming are strengths. Capped at 4 alongside the OSS anchors; not a frontier-definer over MLflow/Langfuse.", "sources": [ { "url": "https://www.confident-ai.com/", "shows": "tracing, evals, datasets, prompt mgmt, monitoring, red teaming, annotations", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Confident AI = the proprietary cloud quality platform built by the creators of DeepEval (the separate OSI Apache-2.0 OSS eval framework). Verified live June 2026. The platform layers tracing/datasets/monitoring on top of DeepEval; self-hosted enterprise option exists." }, { "product": "Vertex AI Model Observability", "org": "Google Cloud", "type": "software", "description": "Google Cloud's built-in production monitoring surface for Gen AI on Vertex AI, a predefined Cloud Monitoring dashboard tracking usage, throughput, token consumption, latency, and error rates (including 429 capacity errors) for Gemini and other Vertex AI Model Garden foundation models. Extended in 2026 by Vertex AI Agent Builder with sessions, traces, logs, agent performance dashboards, multi-turn auto-raters, online evaluation against live traffic, and a Unified Trace Viewer for debugging agent reasoning paths. Differentiated as the default zero-config observability surface for the >hundreds-of-thousands of organizations already on Vertex AI, with no separate SDK required for managed-model usage.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(Google Cloud managed service);source:closed;note:tracing is built on the open OTel standard but the observability product/service itself is proprietary and cloud-locked", "confidence": "high", "note": "Proprietary GCP managed offering (Agent Engine + Cloud Trace/Monitoring/Logging). OTel is the wire standard but the service is closed and tied to Google Cloud.", "sources": [ { "url": "https://docs.cloud.google.com/agent-builder/agent-engine/manage/tracing", "shows": "Cloud Trace tracing for Agent Engine as a managed Google Cloud service", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "A feature surface within Vertex AI Agent Engine on Google Cloud; no standalone usage figures published for the observability capability specifically. Level 3 reflects availability inside a large cloud platform's agent stack, not a measured user count; directional.", "sources": [ { "url": "https://docs.cloud.google.com/agent-builder/agent-engine/manage/tracing", "shows": "observability shipped as part of the managed Vertex AI Agent Engine platform", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "tracing (traces/spans for LLM calls, tool calls, agent decisions via Cloud Trace/OTel) + perf monitoring dashboard (token consumption, latency, error rates, tool calls) + quality/evaluation (integrated Gen AI Evaluation service incl User Simulator) + Cloud Logging. Cost/token tracking present; prompt management and dedicated annotation queues not evidenced as native.", "confidence": "medium", "note": "Solid tracing + monitoring + integrated eval, but the observability matrix is thinner than the dedicated OSS platforms on prompt versioning/datasets/annotation; scored 3 below the C4 anchors.", "sources": [ { "url": "https://docs.cloud.google.com/agent-builder/agent-engine/manage/tracing", "shows": "Cloud Trace traces/spans, perf dashboard, Gen AI Evaluation integration", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Vertex AI / Agent Engine observability: tracing via Cloud Trace (OpenTelemetry-built), Cloud Monitoring, Cloud Logging, plus integrated Gen AI Evaluation service. Proprietary Google Cloud managed service. Verified live June 2026 in GCP docs." }, { "product": "Azure AI Foundry Observability", "org": "Microsoft Azure", "type": "software", "description": "Microsoft's GenAI observability product inside Azure AI Foundry (rebranded to Microsoft Foundry in 2026), providing evaluations, distributed tracing, production monitoring, and content-safety logs for LLM apps and agents. Built on OpenTelemetry and integrated with Azure Monitor Application Insights, with built-in evaluators covering quality (coherence, fluency), RAG (groundedness, relevance), safety (hate/unfairness, violence, protected materials), and agent-specific metrics (tool-call accuracy, task completion). Supports tracing for LangChain, LangGraph, OpenAI Agents SDK, and the Microsoft Agent Framework. Evaluations, Monitoring, and Tracing went GA via Foundry Control Plane in 2026, with continuous evaluation and the Agent Monitoring Dashboard in preview; leverages Azure's enterprise footprint as the default observability layer for Azure-hosted GenAI workloads.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(Azure managed service);source:closed;note:tracing built on open OTel standard + Azure Monitor App Insights, but the observability product is proprietary and Azure-locked", "confidence": "high", "note": "Proprietary Microsoft Azure managed offering. OTel is the wire standard (trace data is portable) but the service itself is closed.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/concepts/observability", "shows": "Microsoft Foundry observability (eval/monitoring/tracing) as a managed Azure service built on OpenTelemetry + Azure Monitor", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "GA March 2026 as a feature surface within Microsoft Foundry on Azure; no standalone usage figures published for the observability capability specifically. Level 3 reflects availability inside a large cloud AI platform, not a measured user count; directional.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/concepts/observability", "shows": "observability GA (Mar 2026) shipped within the managed Microsoft Foundry platform", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "tracing (OTel distributed tracing of LLM calls, tool invocations, agent decisions, cross-agent handoffs; inputs/outputs/latency/token usage via Azure Monitor App Insights) + monitoring + evaluations; auto-instrumentation across Semantic Kernel, LangChain, LangGraph, OpenAI Agents SDK. Strong tracing + framework breadth; prompt mgmt/datasets/annotation not evidenced as first-class native features.", "confidence": "medium", "note": "Three pillars (eval/monitoring/tracing) with strong OTel + framework coverage, but matrix breadth on prompt versioning/datasets/annotation is thinner than the dedicated OSS platforms; scored 3 below the C4 anchors.", "sources": [ { "url": "https://learn.microsoft.com/en-us/azure/foundry/concepts/observability", "shows": "OTel tracing, monitoring, evaluations; auto-instrumentation for Semantic Kernel/LangChain/LangGraph/OpenAI Agents SDK", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Microsoft Foundry (formerly Azure AI Foundry) Observability: evaluations + monitoring + tracing reached general availability March 2026. OpenTelemetry-based tracing via Azure Monitor / Application Insights. Proprietary Azure managed service. Verified live June 2026 in Microsoft Learn docs." }, { "product": "MLflow", "org": "Databricks", "type": "software", "description": "Active 2026; Apache 2.0; observability/tracing for agents+LLMs+ML", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0;source:public;core-fully-OSS(Databricks-Managed-MLflow is third-party hosting, not gated core)", "confidence": "high", "note": "license:Apache-2.0;source:public;core-fully-OSS(Databricks-Managed-MLflow is third-party hosting, not gated core)", "sources": [ { "url": "https://github.com/mlflow/mlflow/blob/master/LICENSE.txt", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/mlflow/mlflow", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://mlflow.org/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 37.10M PyPI downloads in the last 30 days (verified on pypistats; the prior note already reported 25-30M+ but mislabeled the reach as 1M-10M). The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, langgraph ~58M at level 5). MLflow is the de facto open experiment-tracking / ML-lifecycle platform and clears the floor comfortably.", "sources": [ { "url": "https://pypistats.org/api/packages/mlflow/recent", "shows": "last_month downloads = 37,102,636", "accessed": "2026-06-25" }, { "url": "https://github.com/mlflow/mlflow", "shows": "mlflow/mlflow repo (experiment-tracking platform) confirmed live", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Single open platform combining distributed tracing, prompt versioning, automated evaluation, and trace replay for agents/LLMs plus classic ML tracking. No standard benchmark for observability; rated on feature matrix.", "sources": [ { "url": "https://mlflow.org/genai/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://mlflow.org/top-5-agent-observability-tools/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.5, "mature": true, "version_note": "Active 2026; Apache 2.0; observability/tracing for agents+LLMs+ML" }, { "product": "Langfuse", "org": "Langfuse", "type": "software", "description": "Open-source LLM engineering platform providing tracing, prompt management, evaluations, cost tracking, and datasets. Self-hostable or cloud-hosted. The leading open source LLM observability platform with broad integrations (OpenTelemetry, LangChain, OpenAI SDK, LiteLLM). YC W23, 27.7K GitHub stars.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(core:tracing/integrations/API/data-model/exports/evals/playground/SSO);commercial:enterprise-security(SCIM,audit-logs,data-retention);source:public", "confidence": "high", "note": "license:MIT(core:tracing/integrations/API/data-model/exports/evals/playground/SSO);commercial:enterprise-security(SCIM,audit-logs,data-retention);source:public", "sources": [ { "url": "https://github.com/langfuse/langfuse", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://langfuse.com/blog/2025-06-04-open-sourcing-langfuse-product", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://clickhouse.com/blog/clickhouse-acquires-langfuse-open-source-llm-observability", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Described as the most widely adopted open source LLM engineering platform; acquired by ClickHouse Mar 2026. No precise verified download/active-user count, rated 3 on reported traction.", "sources": [ { "url": "https://github.com/langfuse/langfuse", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://clickhouse.com/blog/clickhouse-acquires-langfuse-open-source-llm-observability", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "End-to-end tracing, prompt management, evals (incl. LLM-as-judge), datasets, annotation queues, playground; OpenTelemetry + LangChain/OpenAI/LiteLLM integrations. Rated on feature matrix.", "sources": [ { "url": "https://github.com/langfuse/langfuse", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://langfuse.com/docs/observability/overview", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.5, "mature": false, "version_note": "Active 2026; acquired by ClickHouse Mar 2026; MIT core" }, { "product": "TruLens", "org": "Snowflake", "type": "software", "description": "Open-source library for evaluation and tracing of LLM apps and AI agents. Its differentiator is 'feedback functions' - programmatic evaluators including the RAG triad (groundedness, context relevance, answer relevance) plus toxicity and others - combined with app instrumentation and tracing, making it more eval-first than gateway/observability-first peers.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(github.com/truera/trulens);commercial:Snowflake-Cortex-AI-Observability(separate hosted product);library:not-feature-gated", "confidence": "high", "note": "MIT library that is fully functional standalone; the Snowflake Cortex hosted product is separate, not a gate on the library.", "sources": [ { "url": "https://github.com/truera/trulens", "shows": "MIT, ~3.4k stars, LLM eval + tracing", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "medium", "note": "~38.7k PyPI downloads/month (pypistats) alongside ~3.4k stars.", "sources": [ { "url": "https://pypistats.org/packages/trulens", "shows": "38,729 downloads last month", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "feedback functions / LLM-as-judge (RAG triad, toxicity); instrumentation + tracing; experiment comparison across versions; Streamlit dashboard; OpenAI/HF/LiteLLM/Cortex providers", "confidence": "medium", "note": "Eval-first observability; lighter than Langfuse/Helicone on production gateway and prompt management.", "sources": [ { "url": "https://www.snowflake.com/en/blog/trulens-open-source-ai/", "shows": "Snowflake acquired TruEra; TruLens stays open source", "accessed": "2026-06-18" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT. Created by TruEra, which Snowflake acquired in 2024; now maintained by Snowflake (the GitHub org remains 'truera'). A separate hosted Snowflake Cortex AI Observability product exists; the library stays fully open. ~3.4k stars, ~39k PyPI downloads/month." } ] }, "agent_tools_protocols": { "label": "Agent tools & protocols", "arc": "Product / UX", "layer": "product_ux", "stage": { "num": 5, "name": "Mature Open Ecosystem" }, "gaps": [], "products": [ { "product": "Model Context Protocol", "org": "Anthropic", "type": "software", "description": "Open standard for connecting AI agents to external tools and data sources. Defines how agents discover, authenticate with, and invoke tools via a JSON-RPC protocol. Rapidly adopted across the agent ecosystem; supported by Claude Code, Cursor, VS Code Copilot, Windsurf, and dozens of IDE and agent frameworks. The emerging universal standard for agent-tool communication, with 86K+ stars on the reference server repo.", "openness": { "score": 5, "class": "open_source", "components": "spec+reference-SDKs:public;license:Apache-2.0(OSI, new code/spec)+MIT(OSI, legacy contributions)+CC-BY-4.0(docs);governance:Linux-Foundation/AAIF(open, co-founded Anthropic/Block/OpenAI)", "confidence": "high", "note": "Open protocol with OSI-licensed reference implementations and open foundation governance; no feature-gated core.", "sources": [ { "url": "https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/LICENSE", "shows": "Apache-2.0 (new code/spec) + MIT (legacy) + CC-BY-4.0 (docs) license text", "accessed": "2026-06-04" }, { "url": "https://en.wikipedia.org/wiki/Model_Context_Protocol", "shows": "Anthropic origin; donated to Linux Foundation AAIF Dec 2025", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "De facto standard agent tool protocol. ~97M monthly SDK downloads in Mar 2026 (up from ~2M at Nov 2024 launch); adopted natively by OpenAI (ChatGPT), Google DeepMind (Gemini), Microsoft, AWS; >16,000 MCP servers in the wild (Nerq Q1 2026 census indexed 17,468). Native client/agent support breadth is the headline MCP signal per recipe.", "sources": [ { "url": "https://en.wikipedia.org/wiki/Model_Context_Protocol", "shows": "adoption by OpenAI/Google DeepMind/Microsoft/AWS; SDKs across 7+ languages", "accessed": "2026-06-04" }, { "url": "https://zuplo.com/blog/one-year-of-mcp", "shows": "~97M monthly SDK downloads Mar 2026, 4,750% growth since launch (corroborating aggregator)", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "A wire protocol is not benchmarked; capability is not a meaningful axis per recipe. Openness + adoption carry the signal.", "sources": [] }, "maturity": 5.0, "mature": true, "version_note": "Open wire protocol for connecting LLMs/agents to tools and data; created by Anthropic (David Soria Parra, Justin Spahr-Summers), launched 25 Nov 2024. Reference spec + SDKs (Python/TypeScript/C#/Java/Go/Rust/Ruby) verified live on GitHub June 2026. Governance donated to the Linux Foundation's Agentic AI Foundation (AAIF) Dec 2025." }, { "product": "Firecrawl", "org": "Mendable", "type": "software", "description": "Web scraping and crawling API built specifically for LLM consumption. Turns any URL into clean markdown or structured data, handling JavaScript rendering, pagination, and anti-bot measures automatically. The most popular open source web-to-LLM pipeline with 123K GitHub stars, used by agent frameworks as the default scraping tool. Open-source with a hosted SaaS option.", "openness": { "score": 4, "class": "open_core", "components": "license:AGPL-3.0(OSI, self-host core);source:public;managed-cloud:firecrawl.dev with 'additional features' beyond OSS = open-core split", "confidence": "high", "note": "AGPL-3.0 OSI core is fully open, but the managed cloud carries features beyond the OSS build, making this open-core rather than pure open_source.", "sources": [ { "url": "https://github.com/mendableai/firecrawl/blob/main/LICENSE", "shows": "AGPL-3.0 license text, Sideguide Technologies Inc.", "accessed": "2026-06-04" }, { "url": "https://www.firecrawl.dev/pricing", "shows": "6 paid managed-cloud tiers (Free/Hobby/Standard/Growth/Scale/Enterprise)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No verified download/API-volume or customer count disclosed on repo or pricing page; only ~129k GitHub stars available as an honest signal. Capped at level 3 per stars_fallback rule. Widely integrated as an agent scrape tool (LangChain, etc.) but no primary usage figure found this run.", "sources": [ { "url": "https://github.com/mendableai/firecrawl", "shows": "~129k GitHub stars, v2.10 (15 May 2026)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "scrape + crawl + map + search + agent extract + batch; JS-rendered pages, rotating proxies, rate-limit handling; markdown/JSON/screenshot/structured output; P95 latency 3.4s, claims 96% web coverage", "confidence": "medium", "note": "Broad scrape/browse feature coverage (the recipe's search/scrape dimensions: coverage, freshness, structured output, rate limits), among the most capable open scrape tools; not a 5 (no benchmark, and frontier reserved for category-definers).", "sources": [ { "url": "https://github.com/mendableai/firecrawl", "shows": "feature list: search/scrape/crawl/map/agent/batch, LLM-ready output formats", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Web scrape/crawl/search API that returns LLM-ready data (markdown/JSON/structured). Self-hostable OSS + managed cloud (firecrawl.dev). v2.10 released 15 May 2026; verified live on GitHub June 2026. Copyright Sideguide Technologies Inc." }, { "product": "Playwright MCP", "org": "Microsoft", "type": "software", "description": "MCP server that gives AI agents full browser automation capabilities via Playwright. Agents can navigate pages, click elements, fill forms, take screenshots, and extract structured data from web pages, all through the MCP protocol. The standard open source approach for giving agents a browser, with 33K GitHub stars. Backed by Microsoft and tightly integrated with VS Code and Copilot.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Microsoft repo);no-feature-gated-core;no-managed-tier", "confidence": "high", "note": "Fully OSI-licensed Microsoft repo, no open-core split.", "sources": [ { "url": "https://github.com/microsoft/playwright-mcp", "shows": "Apache-2.0 license, v0.0.75 (7 May 2026)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "@playwright/mcp is a widely distributed npm package and the de facto browser-automation MCP server, shipped/recommended across major MCP clients (VS Code, Cursor, Claude Desktop). npm download page returned HTTP 403 this run so an exact weekly-download figure could not be fetched; level 4 rests on broad multi-client native integration plus 33.5k stars (corroborating, not basis). Confidence medium pending a clean download number.", "sources": [ { "url": "https://github.com/microsoft/playwright-mcp", "shows": "33.5k stars, integration list across VS Code/Cursor/Claude Desktop and many MCP clients", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "structured accessibility-snapshot navigation, click/type/fill-form, screenshots, network requests, console, tabs, file upload, code-eval; cross-browser via Playwright", "confidence": "medium", "note": "Deep, broad browser-control surface (the leading agentic browser tool); scored 4 on the scrape/browse feature matrix, strong coverage but no standardized benchmark and not the single category-definer.", "sources": [ { "url": "https://github.com/microsoft/playwright-mcp", "shows": "browser automation feature set via structured snapshots", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "MCP server providing browser automation to LLMs via Playwright using structured accessibility snapshots (not screenshots). Apache-2.0. Latest v0.0.75 (7 May 2026, 65 releases); verified live on GitHub June 2026. Integrates with VS Code, Cursor, Claude Desktop and other MCP clients." }, { "product": "FastMCP", "org": "Prefect", "type": "software", "description": "High-level Python framework for building MCP servers and clients with minimal boilerplate. Provides decorator-based tool definitions, automatic schema generation, and built-in testing utilities. The fastest way to create a production MCP server. 25K GitHub stars and growing rapidly. Becoming the de facto framework for MCP server development in Python.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no-feature-gated-core;maintained-by-Prefect", "confidence": "high", "note": "Fully OSI-licensed, no open-core split observed on the repo.", "sources": [ { "url": "https://github.com/jlowin/fastmcp", "shows": "Apache-2.0 license, v3.4.0 (3 Jun 2026), maintained by Prefect", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "fastmcp ~72.7M PyPI downloads in the last month; README reports ~1M downloads/day, used by ~10k dependent projects, and powering ~70% of MCP servers across languages. Mass-market reach for MCP server tooling.", "sources": [ { "url": "https://pypistats.org/packages/fastmcp", "shows": "~72.7M monthly PyPI downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/jlowin/fastmcp", "shows": "~25.5k stars, ~10k dependents, README claims ~1M downloads/day and ~70% of MCP servers", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "auto schema generation/validation, tool/resource/prompt primitives, server+client, auth, deployment, protocol lifecycle management; the standard high-level MCP server framework", "confidence": "medium", "note": "Most feature-complete framework for authoring the tool-protocol layer (MCP); scored 4 on protocol-tooling feature breadth; a 5 is reserved for the protocol/category-definer (MCP itself).", "sources": [ { "url": "https://github.com/jlowin/fastmcp", "shows": "framework feature set: schema gen/validation, server+client, auth, deployment", "accessed": "2026-06-04" } ] }, "maturity": 4.6, "mature": true, "version_note": "Pythonic framework for building MCP servers and clients (FastMCP 1.0 was upstreamed into the official MCP Python SDK in 2024; FastMCP 2.0/3.x is the standalone successor). Maintained by Prefect. Apache-2.0. v3.4.0 released 3 Jun 2026; verified live June 2026." }, { "product": "Qdrant", "org": "Qdrant", "type": "software", "description": "High-performance vector database and search engine built in Rust, designed for the next generation of AI applications. Stores vectors with rich payloads and supports advanced filtering, making it ideal for RAG retrieval pipelines that agents call to access knowledge bases. 31K+ GitHub stars, available as open source self-hosted or managed cloud. The leading Rust-native vector DB, competing with Milvus and Pinecone.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI, engine core);source:public(Rust);managed-tier:Qdrant Cloud (hosted, paid + free tier)", "confidence": "high", "note": "Engine core is Apache-2.0 OSI, but a managed Qdrant Cloud tier sits on top; recipe explicitly classes Qdrant Cloud as open_core, so scored 4 rather than the pure-OSS 5 the Milvus anchor carries.", "sources": [ { "url": "https://github.com/qdrant/qdrant/blob/master/LICENSE", "shows": "Apache-2.0 license text", "accessed": "2026-06-04" }, { "url": "https://github.com/qdrant/qdrant", "shows": "v1.18.2 (4 Jun 2026); Qdrant Cloud managed tier with free option", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. qdrant-client 23.06M PyPI downloads in the last 30 days (verified on pypistats; the prior note already reported ~26.1M but mislabeled the reach as 1M-10M). The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, langgraph ~58M at level 5); Qdrant's client volume clears it. It remains one above the Milvus anchor (A3) on download volume, a leading open vector DB for RAG/agent retrieval.", "sources": [ { "url": "https://pypistats.org/api/packages/qdrant-client/recent", "shows": "last_month downloads = 23,061,488", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "benchmark:ANN-Benchmarks", "value": "Included in ANN-Benchmarks (37 algorithms incl. Qdrant/Milvus/faiss/scann). Vendor benchmark: highest RPS and lowest latency in almost all scenarios vs Milvus/Weaviate/Elasticsearch/Redis at matched precision; advanced filtered-search query planning.", "confidence": "medium", "note": "Top-tier ANN engine on RPS/latency; tied with the Milvus anchor at C4 (frontier vector-DB capability). Headline comparison is vendor-run (qdrant.tech/benchmarks); ANN-Benchmarks corroborates inclusion but its interactive plots did not yield a clean ranking number this run, hence medium confidence.", "sources": [ { "url": "https://ann-benchmarks.com/", "shows": "Qdrant included among 37 benchmarked ANN algorithms", "accessed": "2026-06-04" }, { "url": "https://qdrant.tech/benchmarks/", "shows": "Qdrant highest RPS/lowest latency vs Milvus/Weaviate/Elasticsearch/Redis (vendor benchmark)", "accessed": "2026-06-04" } ] }, "maturity": 4.6, "mature": true, "version_note": "High-performance Rust vector database / vector search engine (dense, sparse, multi-vector, hybrid search, quantization). Apache-2.0 core + Qdrant Cloud managed tier (free tier available). v1.18.2 released 4 Jun 2026; verified live June 2026. Calibrated against frozen anchor Milvus (O5 open_source/A3/C4)." }, { "product": "Jina Reader", "org": "Jina AI", "type": "software", "description": "URL-to-LLM-input conversion service that turns any web page into clean, structured content optimized for language models. Prefix any URL with r.jina.ai to get markdown output. Simple, fast, and widely integrated into agent workflows as a lightweight alternative to full scraping tools. Open-source with 11K GitHub stars. Also offers search (s.jina.ai) and grounding capabilities.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(TypeScript, stateless mode self-hostable via Docker);managed-tier:r.jina.ai/s.jina.ai hosted SaaS (free tier + paid API key) on top of the same OSS core", "confidence": "high", "note": "Repo is Apache-2.0 with the full reader pipeline public and self-hostable; hosted Jina API is convenience hosting, not a gated core.", "sources": [ { "url": "https://github.com/jina-ai/reader", "shows": "Apache-2.0 license, public source, self-host Docker instructions, ~11k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "stars_fallback", "confidence": "low", "note": "No verified download or API-call volume found from a primary source. ~11k GitHub stars on the OSS repo plus a widely-used hosted r.jina.ai endpoint; capped at 3 on stars/reported-traction fallback in the absence of a measured usage figure.", "sources": [ { "url": "https://github.com/jina-ai/reader", "shows": "~11k stars, active maintenance (550 commits), production SaaS backing", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "URL->LLM-friendly markdown; handles HTML/PDF/MS-Office/images (AI captioning); web search via s.jina.ai; stateless + optional caching; self-host or hosted", "confidence": "medium", "note": "Solid single-purpose reader/scrape tool with multi-format coverage and a search mode; mid-tier on the {coverage, freshness, structured output, rate limits} feature matrix vs full agentic browse/search platforms.", "sources": [ { "url": "https://github.com/jina-ai/reader", "shows": "feature list: HTML/PDF/Office/image input, markdown output, search mode, caching", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "Open-source branch of the r.jina.ai / s.jina.ai URL-to-markdown reader service; repo jina-ai/reader re-synced with the SaaS codebase Apr 2026 (MongoDB layer decoupled, local Docker deploy enabled). Confirmed live on GitHub June 2026." }, { "product": "Agent2Agent Protocol", "org": "A2A Project (Google-initiated)", "type": "software", "description": "Open protocol enabling communication and interoperability between opaque agentic applications. Where MCP connects agents to tools, A2A connects agents to other agents, defining task delegation, status updates, and result passing between autonomous systems. Initiated by Google with 24K GitHub stars. Still early-stage but positioned as the complementary standard to MCP for multi-agent workflows.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(spec + reference SDKs in 6 languages);governance:Linux Foundation (vendor-neutral, Google-contributed)", "confidence": "high", "note": "Reference spec and SDKs are Apache-2.0 under neutral LF governance; for a protocol this is the openness ceiling.", "sources": [ { "url": "https://github.com/a2aproject/A2A", "shows": "Apache-2.0 license, LF-hosted open protocol, spec v1.0.1 + 6-language SDKs, ~24k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "Protocol adoption-breadth signal (per recipe): LF press (Apr 2026 one-year mark) reports 150+ supporting organizations, in-production use at Microsoft (Azure AI Foundry, Copilot Studio), AWS (Bedrock AgentCore), Salesforce, SAP, ServiceNow, and embedding into Google Cloud. Breadth across the three major clouds + enterprise production puts effective downstream reach in the 1-10M band; this is a breadth/traction count, not a measured per-user figure.", "sources": [ { "url": "https://www.linuxfoundation.org/press/a2a-protocol-surpasses-150-organizations-lands-in-major-cloud-platforms-and-sees-enterprise-production-use-in-first-year", "shows": "150+ orgs supporting; Azure/AWS Bedrock/Google Cloud production integrations", "accessed": "2026-06-04" }, { "url": "https://github.com/a2aproject/A2A", "shows": "6-language SDKs, ~24k stars corroborating developer uptake", "accessed": "2026-06-04" } ] }, "capability": { "score": null, "basis": "n/a", "value": null, "confidence": "high", "note": "A2A is a wire protocol for agent interoperability; per the recipe, capability is not a meaningful axis for a protocol. The signal lives in openness and adoption-breadth.", "sources": [ { "url": "https://github.com/a2aproject/A2A", "shows": "described as an open protocol/spec, not a benchmarkable runtime", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "A2A open protocol for inter-agent interoperability; contributed by Google, hosted by the Linux Foundation (launched June 2025). Spec v1.0.1 released May 28, 2026 (v1.0 = first stable spec, adds Signed Agent Cards). SDKs in Python/Go/JS/Java/.NET/Rust. Verified live on GitHub June 2026." }, { "product": "Composio", "org": "Composio", "type": "software", "description": "Agent tooling platform that provides 1000+ pre-built tool integrations with managed authentication, context management, and a sandboxed execution environment. Handles the hard parts of connecting agents to external services (OAuth flows, API pagination, rate limiting) so agent builders can focus on logic. 28K GitHub stars, YC-backed. The largest catalog of ready-to-use agent tools with built-in auth.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(OSI, SDK/core public on GitHub);source:public(Python + TypeScript SDKs);managed-tier:Composio hosted platform + Rube MCP server (auth, hosted tool execution) is the commercial layer", "confidence": "high", "note": "SDK/core is MIT and public; the hosted multi-app integration/auth platform is the proprietary tier, classic open-core.", "sources": [ { "url": "https://github.com/ComposioHQ/composio", "shows": "MIT LICENSE, public SDK source, 1000+ toolkits, ~28.6k stars", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "CORRECTION (adversarial verify 2026-06-04): prior record claimed no primary download figure and capped at stars_fallback level 3. pypistats now shows the composio PyPI package at ~3.17M downloads in the last month (a measured usage_volume signal in the 1M-10M band), so upgraded to level 4 / usage_volume and the stars_fallback cap is removed. Corroborated by ~28.6k GitHub stars and framework integrations (OpenAI, Anthropic, LangChain, LlamaIndex) plus the hosted platform.", "sources": [ { "url": "https://pypistats.org/packages/composio", "shows": "~3,166,911 (3.17M) monthly PyPI downloads", "accessed": "2026-06-04" }, { "url": "https://github.com/ComposioHQ/composio", "shows": "~28.6k stars, integrations across major agent frameworks", "accessed": "2026-06-04" }, { "url": "https://pypi.org/project/composio/", "shows": "active package, latest v0.13.1 (May 2026); confirms maintenance", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "1000+ toolkits / 500+ apps (Rube MCP), tool search, managed OAuth/auth, sandboxed workbench, multi-framework SDKs (OpenAI/Anthropic/LangChain/LlamaIndex)", "confidence": "medium", "note": "Broad tool-coverage and auth/sandbox feature set make it one of the most capable agent tool-integration layers; strong on the {coverage, structured output, breadth} matrix, just short of a frontier-defining 5.", "sources": [ { "url": "https://github.com/ComposioHQ/composio", "shows": "1000+ toolkits, tool search, auth, sandboxed workbench, Rube MCP 500+ apps", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Agent tool-integration SDK + managed platform: 1000+ toolkits, tool search, auth, sandboxed workbench; Rube MCP server connects 500+ apps. Python SDK v0.13.1 (May 14, 2026); @composio/claude-agent-sdk 0.9.2 (May 13, 2026). Verified live on GitHub/PyPI June 2026." }, { "product": "Tavily Search API", "org": "Tavily", "type": "software", "description": "Search API purpose-built for AI agents, returning clean, structured results optimized for LLM consumption rather than human browsing. Includes search, extract, crawl, and map endpoints. The default search provider in most agent tutorials and frameworks; integrated into LangChain, CrewAI, and AutoGen out of the box. YC-backed, rapidly growing as the canonical 'give your agent web search' solution.", "openness": { "score": 1, "class": "closed", "components": "license:Proprietary(API-only);source:closed(no self-hostable core; only thin client SDKs are public);managed:cloud SaaS with API keys, credits, rate limits", "confidence": "high", "note": "Closed search/scrape API (the recipe's closed counterpart to Milvus/Qdrant); only client SDKs are open, the search service itself is proprietary.", "sources": [ { "url": "https://docs.tavily.com/", "shows": "API keys, credits, rate limits, hosted endpoints, no self-hostable open core", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Tavily's own homepage states 2M+ developers and 300M+ monthly requests, billions of pages crawled (both figures re-confirmed on the live homepage 2026-06-04); it is the default search provider wired into LangChain/CrewAI/LlamaIndex agent stacks. Developer reach solidly in 1-10M band; request volume is far higher. Acquisition by Nebius corroborates enterprise traction.", "sources": [ { "url": "https://www.tavily.com/", "shows": "vendor-stated 2M+ developers, 300M+ monthly requests, billions of pages crawled", "accessed": "2026-06-04" }, { "url": "https://nebius.com/newsroom/nebius-announces-agreement-to-acquire-tavily-to-add-agentic-search-to-its-ai-cloud-platform", "shows": "Nebius agreement to acquire Tavily (Feb 2026), Tavily serving Fortune 500 / leading AI companies", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "search + extract + crawl + map + research endpoints; real-time freshness; structured/LLM-ready output; SLA + rate limits; default integration across major agent frameworks", "confidence": "medium", "note": "Full {coverage, freshness, structured output, rate limits} feature set purpose-built for agents and the de-facto default in agent frameworks; near-frontier among agent search APIs but not the single category-definer.", "sources": [ { "url": "https://docs.tavily.com/", "shows": "search/extract/crawl/map/research endpoints with rate limits and credits", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Proprietary web-access API for agents: search, extract, crawl, map, research endpoints; Python/JS SDKs. PRECISION (verify 2026-06-04): Nebius announced an agreement to acquire Tavily on 10 Feb 2026 for $275M (up to $400M on milestones), expected to close within weeks, an acquisition agreement, not necessarily a fully-closed deal at announcement. Verified live via docs.tavily.com + tavily.com June 2026." }, { "product": "Exa Search API", "org": "Exa", "type": "software", "description": "Neural search API that understands meaning, not just keywords. Returns links and full content based on semantic similarity, 'search the way an AI would.' Particularly strong for finding similar content, research papers, and niche topics where traditional keyword search fails. Well-funded (YC-backed, $22M+ raised), positioned as the semantic alternative to Tavily's more traditional search approach.", "openness": { "score": 1, "class": "closed", "components": "license:Proprietary(API-only);source:closed(neural search models + index proprietary; only client SDKs public);managed:cloud SaaS with API keys, usage-based pricing", "confidence": "high", "note": "Closed neural-search API; no self-hostable core, the index and ranking models are proprietary.", "sources": [ { "url": "https://exa.ai/about", "shows": "search-as-a-service, API dashboard, no self-host/open source option", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "Exa's own about page states it powers search for Cursor, Cognition, HubSpot, OpenRouter, Monday.com and 400,000+ developers. Developer reach in the 100K-1M band; large enterprise/agent customers but below the 1M-developer Tavily tier.", "sources": [ { "url": "https://exa.ai/about", "shows": "vendor-stated 400,000+ developers; named customers Cursor, Cognition, HubSpot, OpenRouter, Monday.com", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "neural/semantic web search + contents retrieval + Websets (search, verify-against-criteria, enrich entity lists); positioned for high-compute complex queries; scaling to hundreds of thousands of searches/sec (vendor)", "confidence": "medium", "note": "Strong on {coverage, structured output}; Websets' verify-and-enrich pipeline is a differentiated capability beyond plain search; near-frontier among agent search APIs.", "sources": [ { "url": "https://exa.ai/about", "shows": "neural search + Websets verify/enrich; high-compute search positioning", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Proprietary neural/web search API (formerly Metaphor): search + contents + Websets (verified, enriched entity lists). Series C $250M at ~$2.2B valuation (May 2026). Verified live via exa.ai docs + about page June 2026." }, { "product": "Browserbase", "org": "Browserbase", "type": "software", "description": "Cloud-hosted headless browsers designed specifically for AI agents. Handles authentication persistence, CAPTCHA solving, and anti-bot detection that self-hosted Playwright cannot easily manage. The managed alternative to running your own browser infrastructure; agents get a reliable, scalable browser as a service. YC-backed, also maintains Stagehand (22K stars), an open source browser agent SDK.", "openness": { "score": 1, "class": "closed", "components": "license:Proprietary(API-only managed browser infra);source:closed(cloud browser fleet/session infra);note:companion Stagehand SDK IS open source (Apache/MIT) but the Browserbase service being scored is the closed managed platform", "confidence": "high", "note": "The scored product is the managed browser-infrastructure service, which is closed/proprietary; only the separate Stagehand client SDK is open; do not let the open SDK reclassify the closed service.", "sources": [ { "url": "https://docs.browserbase.com/", "shows": "one API key for cloud browsers/search/fetch/runtime, managed proprietary platform; Stagehand SDK noted as the open source client", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "No primary usage figure for the Browserbase managed service itself was published. Best primary proxy is its own SDK, Stagehand, which Browserbase docs cite at 22k GitHub stars and 700k+ weekly downloads, a strong open-SDK adoption signal that approximates the platform's developer reach (100K-1M band). The closed service's own user/session count is undisclosed.", "sources": [ { "url": "https://docs.browserbase.com/", "shows": "vendor-stated Stagehand SDK 22k stars, 700k+ weekly downloads (proxy for Browserbase developer reach)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Browser API (session control), Search API, Fetch API (markdown), Functions (agent deploy/schedule), Model Gateway (100+ LLM providers); Stagehand: natural-language selectors, self-healing actions, caching at scale", "confidence": "medium", "note": "Broad browser-automation-for-agents feature set (control + search + fetch + deploy + model routing) plus a capable agent SDK; near-frontier among agent browser/scrape platforms.", "sources": [ { "url": "https://docs.browserbase.com/", "shows": "5-component platform: Browser/Search/Fetch/Functions/Model Gateway + Stagehand features", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Proprietary cloud headless-browser platform for agents: Browser API, Search API, Fetch API, Functions, Model Gateway. Ships the open source Stagehand SDK (browser-agent framework) on top. Verified live via docs.browserbase.com June 2026." }, { "product": "Perplexity Sonar API", "org": "Perplexity AI", "type": "software", "description": "Search-augmented LLM API that combines real-time web search with language model reasoning to produce sourced, grounded answers. Unlike Tavily (which returns raw search results), Perplexity returns synthesized answers with citations; agents get a research assistant, not just a search engine. Both a consumer product and a developer API. Raised $500M+, valued at $9B+, the most well-funded company in the AI search space.", "openness": { "score": 1, "class": "closed", "components": "license:Proprietary(API-only);source:closed(Sonar models, search index, ranking all proprietary);managed:cloud SaaS, token + per-request pricing", "confidence": "high", "note": "Fully closed search-grounded API; no weights, source, or self-host path.", "sources": [ { "url": "https://docs.perplexity.ai/", "shows": "Sonar/Search/Embeddings APIs, token + per-request pricing, proprietary hosted service", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "reported_traction", "confidence": "medium", "note": "No standalone Sonar-API request/developer count is published by a primary source; Perplexity docs confirm the API exists and is a core developer surface. Platform-level scale is large (45M+ MAU, ~1B+ queries/month per 2026 reporting) but that is the consumer app, not the API specifically. Rated 4 on reported developer-API traction within a clearly mass-scale parent platform; flagged that the headline platform numbers are aggregator-reported and not API-specific.", "sources": [ { "url": "https://docs.perplexity.ai/", "shows": "Sonar API is a documented developer surface (Agent/Search/Embeddings)", "accessed": "2026-06-04" }, { "url": "https://www.perplexity.ai/api-platform", "shows": "vendor API-platform page positioning Sonar for developers", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "search-grounded Q&A with real-time web retrieval; Sonar + Sonar Pro tiers (context-depth pricing); ranked Search API with filtering; Embeddings API for RAG; Agent API routes to external models (e.g. GPT-5.x)", "confidence": "medium", "note": "Strong {coverage, freshness, structured output}, grounded answers + raw search + embeddings in one API; near-frontier among agent search/retrieval APIs but a wrapper-plus-search rather than a category-defining 5.", "sources": [ { "url": "https://docs.perplexity.ai/", "shows": "Sonar/Sonar Pro grounded search, Search API filtering, Embeddings, Agent API model routing", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Proprietary search-grounded API: Sonar / Sonar Pro models, plus Search API (ranked web results) and Embeddings API; 2026 adds a multi-model Agent API marketplace. Verified live via docs.perplexity.ai June 2026." }, { "product": "Pinecone", "org": "Pinecone", "type": "software", "description": "Fully managed vector database purpose-built for AI applications at scale. Serverless architecture with automatic scaling, integrated sparse-dense hybrid search, and sub-100ms query latency. The most widely adopted closed-source vector database, used by thousands of production RAG systems. Raised $138M+ (Series B at $750M valuation). The default choice for teams that want managed vector search without operational overhead.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS);source:closed;self-host:none(BYOC runs Pinecone-managed in customer VPC, not OSS);managed:Pinecone-Cloud", "confidence": "high", "note": "Fully proprietary managed vector DB; no open source core (contrast Milvus/Qdrant which are OSI-licensed). BYOC is still closed-source software run in customer cloud.", "sources": [ { "url": "https://www.pinecone.io/pricing/", "shows": "proprietary tiered SaaS pricing (Starter/Builder/Standard/Enterprise/BYOC); no OSS/self-host license", "accessed": "2026-06-04" }, { "url": "https://docs.pinecone.io/guides/get-started/overview", "shows": "described as 'fully managed vector database', billions of items searched in ms", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Leading managed vector DB; production deployments cited at 1.4B vectors / 5,700 QPS for a single e-commerce customer and 135M vectors / 2,200 QPS load-tested for another. Broad enterprise/developer base via serverless + AWS Marketplace PAYG. No single hard user count published; placed at 4 on aggregate production usage_volume. (Some churn noted, e.g. Notion left over cost.)", "sources": [ { "url": "https://siliconangle.com/2025/12/01/pinecone-scales-vector-database-support-demanding-workloads/", "shows": "billion-scale production workloads, multi-thousand QPS customer deployments", "accessed": "2026-06-04" }, { "url": "https://aws.amazon.com/marketplace/pp/prodview-xhgyscinlz4jk", "shows": "Pinecone PAYG distribution via AWS Marketplace (broad enterprise availability)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:ANN-Benchmarks", "value": "P50 latency ~26-60ms at 1.4B-vector scale, 5,700 QPS; serverless auto-indexing, dedicated read nodes", "confidence": "medium", "note": "Top-tier managed ANN search at billion-vector scale with strong latency/QPS. No independent ANN-Benchmarks/VectorDBBench submission found (closed system, vendor-reported figures); rated 4 (peer to Milvus C4), reserving 5 for benchmark-proven frontier.", "sources": [ { "url": "https://www.infoq.com/news/2025/12/pinecone-drn-vector-workloads/", "shows": "dedicated read nodes for predictable vector workloads, serverless scaling", "accessed": "2026-06-04" }, { "url": "https://siliconangle.com/2025/12/01/pinecone-scales-vector-database-support-demanding-workloads/", "shows": "26-60ms P50 latency, thousands of QPS at billion-vector scale", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Pinecone serverless vector database, verified live June 2026 (pricing + docs). Proprietary managed SaaS; BYOC option added 2025/2026. Sub-segment: retrieval/RAG infra (vector DB)." }, { "product": "Serper", "org": "Serper", "type": "software", "description": "Fast, low-cost Google Search API designed for developers and AI agents. Returns structured Google SERP data (organic results, knowledge graph, related searches) via a simple REST API. Positioned as the cheapest way to give agents Google-quality search results, $0.001/query at scale. Widely used in LangChain and agent frameworks as a cost-effective alternative to Tavily for high-volume use cases.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(API-only);source:closed;managed:Serper-Cloud(key-gated)", "confidence": "high", "note": "Proprietary commercial SERP API; no source or self-host. Closed counterpart to OSS-adjacent search tools, like Tavily/Exa per the recipe.", "sources": [ { "url": "https://serper.dev", "shows": "commercial key-gated Google Search API, JSON output, 2,500 free queries then paid; no OSS license", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Popular low-cost SERP API widely wired into agent stacks (LangChain/CrewAI tool integrations), but vendor publishes no hard user/query-volume figure and no primary download/API-call count is verifiable. Declining to score adoption rather than rely on marketing 'fastest & cheapest' claims.", "sources": [ { "url": "https://serper.dev", "shows": "no published user/query-volume figures, only marketing speed/price claims", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "coverage: web/images/news/maps/shopping/video/scholar/patents/autocomplete; structured JSON output; ~1-2s latency; freshness=live Google index", "confidence": "medium", "note": "Broad search-type coverage with structured output and live freshness; solid feature matrix for a search API, but thinner than full agent-search tools (no built-in extraction/answer synthesis like Tavily/Exa). Rated 3 on coverage/freshness/structured-output dimensions.", "sources": [ { "url": "https://serper.dev", "shows": "9 search verticals, structured JSON, 1-2s latency", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Serper.dev Google Search API, verified live June 2026. Proprietary closed search API (web/images/news/maps/shopping/video/scholar). Sub-segment: search APIs (Tavily/Exa peer, closed counterpart)." }, { "product": "SearXNG", "org": "SearXNG", "type": "software", "description": "Free, privacy-respecting open source metasearch engine that aggregates results from up to 269 search services without tracking or profiling users. Though it predates the LLM era and does no inference itself, it has become the de facto self-hosted web-search backend for open AI answer engines and RAG - it exposes a JSON API and runs entirely on your own infrastructure (no commercial search keys, no tracking), and is integrated as a web-search provider by Open WebUI, Perplexica, Morphic, and LibreChat. It is the open peer to proprietary AI search APIs like Tavily, Serper, and Exa.", "openness": { "score": 5, "class": "open_source", "components": "license:AGPL-3.0(OSI);source:public(github.com/searxng/searxng);self-host:only(no official SaaS);reproducible-Docker-build;no-proprietary-backend", "confidence": "high", "note": "Fully open source under copyleft AGPL-3.0, complete source public, reproducible official Docker image, no proprietary backend or gated hosted tier - self-hosting (or community public instances) is the only distribution.", "sources": [ { "url": "https://github.com/searxng/searxng", "shows": "AGPL-3.0 license; ~32k stars; privacy metasearch engine, no tracking", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "50M+ total Docker pulls", "signal_type": "usage_volume", "confidence": "high", "note": "~32k GitHub stars plus 50M+ total Docker pulls (~1.5M in a single recent week) and ~70 maintained public instances - concrete deployment volume well into the high band, not stars-only. The de facto open web-search backend for self-hosted AI.", "sources": [ { "url": "https://hub.docker.com/r/searxng/searxng", "shows": "official image, 50M+ total pulls, ~1.5M weekly", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "engines aggregated: up to 269 search services; API: JSON (also CSV/RSS) via ?format=json, no auth required by default; programmatic access: simple HTTP GET; consumed as a web-search/RAG backend by Open WebUI, Perplexica, Morphic, LibreChat; self-hostable official Docker image", "confidence": "medium", "note": "The reference open implementation of the search-backend slot - broad source aggregation, clean structured JSON, fully self-hosted. Below top among AI search APIs because it is raw metasearch without the AI-native extraction/reranking that proprietary peers (Tavily/Exa) layer on.", "sources": [ { "url": "https://docs.searxng.org/dev/search_api.html", "shows": "?format=json search API; up to 269 search services aggregated", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "AGPL-3.0; ~32k GitHub stars and 50M+ total Docker pulls (~1.5M/week) by mid-2026. Community-run fork-successor of the older searx project. No official hosted SaaS - distributed as source/Docker for self-hosting, with a network of ~70 community public instances (searx.space). Not itself an AI product; it is the search/data backend AI tools call." }, { "product": "Google Grounding API", "org": "Google", "type": "software", "description": "Google's managed grounding-with-search capability within Vertex AI. Lets agents ground their responses in real-time Google Search results or enterprise data via Vertex AI Search. Tight integration with Gemini models. The enterprise closed alternative for search-augmented generation, leveraging Google's search index directly. Part of the broader Vertex AI platform used by large enterprises.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(GCP-managed API);source:closed;managed:Vertex-AI/Gemini-Agent-Platform", "confidence": "high", "note": "Fully proprietary Google Cloud managed grounding service; no open source component or self-host. Tied to Gemini + Google Search index.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/grounding/overview", "shows": "Grounding with Google Search listed as a managed grounding option in the Gemini Enterprise Agent Platform; proprietary GCP feature", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": null, "reach": null, "signal_type": "unknown", "confidence": "low", "note": "Bundled feature of Vertex AI / Gemini Enterprise Agent Platform with no standalone usage figure disclosed. The broad Vertex AI / Gemini surface is large, but no honest grounding-API-specific adoption signal is verifiable; not scoring on the parent platform's reach.", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/grounding/overview", "shows": "feature documented but no usage/customer figures for the grounding API itself", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "coverage: full Google Search index grounding; freshness=live web; structured grounding metadata + citations/grounding-support attribution returned to the model", "confidence": "medium", "note": "Grounds LLM output in the full Google Search index with citation/grounding-support metadata, best-in-class web freshness/coverage for agent grounding. Rated 4 on the coverage/freshness/structured-output matrix; capped below 5 as detailed per-feature confirmation came from the overview page only (deeper feature docs not fetched this run).", "sources": [ { "url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/grounding/overview", "shows": "grounds responses in Google Search/web within the agent platform", "accessed": "2026-06-04" } ] }, "maturity": null, "mature": false, "version_note": "Grounding with Google Search in Vertex AI / Gemini Enterprise Agent Platform, verified live June 2026 (Google Cloud docs). Proprietary managed grounding API that anchors Gemini responses in Google Search/web. Sub-segment: search/retrieval grounding tool for agents." }, { "product": "Apify", "org": "Apify", "type": "software", "description": "Full-stack web scraping and automation platform with a marketplace of 2000+ pre-built scrapers (Actors) for specific websites. Provides cloud infrastructure for running scrapers at scale, proxy management, and anti-bot handling. The most mature commercial web scraping platform now repositioned for AI agent workflows. Offers both a managed platform and Crawlee, their open source crawling framework (23K stars).", "openness": { "score": 2, "class": "open_core", "components": "platform:proprietary(Apify-Cloud,Actor-store,managed);oss-sdk:Crawlee(Apache-2.0,OSI, JS+Python);license:mixed(platform closed, SDK open)", "confidence": "high", "note": "The product proper (Apify platform/Actor store) is proprietary SaaS; Apify also maintains the OSS Crawlee scraping library (Apache-2.0). Scored open_core but low (2): the open piece is an adjacent SDK, not the platform core, so most value is gated behind the managed cloud.", "sources": [ { "url": "https://apify.com", "shows": "proprietary full-stack scraping platform with 35,000+ Actors; Crawlee called out as their OSS contribution", "accessed": "2026-06-04" }, { "url": "https://github.com/apify/crawlee", "shows": "Crawlee Apache-2.0 license (OSI), Apify's open source scraping library", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "Crawlee at 23.7k GitHub stars (corroborating, not basis); platform serves named enterprises (T-Mobile, Microsoft, Accenture) and a 15,000+ member Discord. 35,000+ published Actors implies a sizable active developer base. No precise account/API-call count published; placed at 3 on combined developer/usage signal across platform + OSS SDK.", "sources": [ { "url": "https://apify.com", "shows": "35,000+ Actors, named enterprise users (T-Mobile, Microsoft, Accenture), 15,000+ Discord members", "accessed": "2026-06-04" }, { "url": "https://github.com/apify/crawlee", "shows": "23.7k stars on the OSS Crawlee library (corroborating signal)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "coverage: 35,000+ prebuilt Actors; HTTP + headless (Playwright/Puppeteer/Selenium); proxy rotation/session mgmt; structured output; serverless custom Actors; MCP-client integration", "confidence": "high", "note": "Broad scrape/browse feature matrix, multi-engine crawling, proxy/anti-block, structured extraction, serverless deploy, and MCP integration for agents. Among the most complete scrape platforms; rated 4 (frontier-adjacent for the browser/scrape sub-segment).", "sources": [ { "url": "https://apify.com", "shows": "multi-engine crawling, proxy rotation, Actor store, MCP-client integration", "accessed": "2026-06-04" }, { "url": "https://github.com/apify/crawlee", "shows": "Crawlee multi-engine (Playwright/Puppeteer/Cheerio/JSDOM), proxy + session management", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Apify full-stack web scraping/extraction platform, verified live June 2026. Proprietary SaaS (35,000+ Actors store) with open source Crawlee library (Apache-2.0) as its scraping SDK. Sub-segment: browser/scrape infra for agents." }, { "product": "Cohere Rerank API", "org": "Cohere", "type": "software", "description": "Reranking API that takes a query and a list of documents and returns them reordered by relevance. Purpose-built for RAG pipelines; agents retrieve candidate documents from a vector DB, then call Rerank to get the most relevant ones before feeding them to the LLM. The leading standalone reranking service, used across thousands of production RAG systems. Significantly improves retrieval quality with a single API call.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(hosted-API; also cloud-marketplace deploy);source:closed;weights:not-released;managed:Cohere+Azure/AWS/Oracle", "confidence": "high", "note": "Proprietary closed reranking models served via API and cloud marketplaces; no open weights or source. Marketed as 'open' in search-relevance contexts but is not OSS.", "sources": [ { "url": "https://docs.cohere.com/docs/rerank-overview", "shows": "commercial Rerank API; models 4.0/3.5/3.0, no source or weight release", "accessed": "2026-06-04" }, { "url": "https://docs.cohere.com/changelog/rerank-v3.5", "shows": "Rerank v3.5 announced as a hosted proprietary model", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "De facto default managed reranker for RAG, distributed across Azure AI Foundry, AWS SageMaker JumpStart / OpenSearch, and Oracle GenAI in addition to Cohere's API. No hard call-volume/user count published; placed at 3 on broad multi-cloud distribution (reported_traction).", "sources": [ { "url": "https://aws.amazon.com/blogs/big-data/enhancing-search-relevancy-with-cohere-rerank-3-5-and-amazon-opensearch-service/", "shows": "Rerank 3.5 integrated/distributed via Amazon OpenSearch Service", "accessed": "2026-06-04" }, { "url": "https://ai.azure.com/catalog/models/Cohere-rerank-v3.5", "shows": "Rerank v3.5 available in Azure AI Foundry model catalog", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "SOTA on BEIR + multilingual retrieval (nDCG@10), 100+ languages with SOTA on 10 business languages; up to ~25% gain over embedding-only retrieval; multimodal/long-context in newer variants", "confidence": "medium", "note": "Vendor reports SOTA on BEIR and multilingual nDCG@10; widely treated as the reference managed reranker. Concrete nDCG numbers not disclosed on primary pages (claims are vendor-run, not an independent leaderboard), so rated 4 rather than 5. Capability basis is feature_matrix as no standard public reranker leaderboard applies.", "sources": [ { "url": "https://cohere.com/blog/rerank-3pt5", "shows": "vendor SOTA claims on BEIR + multilingual retrieval, ~25% gain over embedding search", "accessed": "2026-06-04" }, { "url": "https://docs.cohere.com/changelog/rerank-v3.5", "shows": "multilingual 100+ language reranking, nDCG@10 evaluation", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Cohere Rerank API, verified live June 2026 (Cohere docs). Latest model Rerank 4.0 (fast/pro variants); prior 3.5 and 3.0 (English + multilingual). Proprietary hosted reranking API for RAG/retrieval; also available via Azure AI Foundry, AWS SageMaker/OpenSearch, Oracle GenAI. Sub-segment: retrieval/RAG infra (reranker)." }, { "product": "Algolia AI Search", "org": "Algolia", "type": "software", "description": "Enterprise search-as-a-service platform that combines keyword, semantic, and AI-powered search with sub-50ms response times. Recently added AI agent integrations and MCP support, allowing agents to search product catalogs, documentation, and enterprise content. Powers search for 17K+ businesses including major e-commerce and SaaS platforms. The enterprise-grade search infrastructure increasingly used as an agent tool for domain-specific retrieval.", "openness": { "score": 1, "class": "closed", "components": "license:proprietary(SaaS, API-first);source:closed(search engine core);oss:client/UI-libraries-only(InstantSearch);managed:Algolia-Cloud", "confidence": "high", "note": "Closed proprietary hosted search engine; only client/UI integration libraries are open. Core ranking/index engine is not OSS or self-hostable.", "sources": [ { "url": "https://www.algolia.com/products/ai-search/", "shows": "proprietary API-first cloud search platform; OSS limited to client/integration libraries", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "18,000+ customers across 150+ countries; processes over 1 billion queries every 5 seconds at sub-20ms average latency. Named users incl. Stripe, Decathlon, Ubisoft. Query volume is enormous; end-user reach (consumers hitting Algolia-powered search) is effectively mass-scale, but as a B2B tool the direct customer base sits in the heavy-trending band; placed at 4 on verified query-volume.", "sources": [ { "url": "https://www.algolia.com/products/ai-search/", "shows": "18,000+ customers, 1B+ queries/5s, sub-20ms latency, named enterprise users", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "hybrid vector+keyword (NeuralSearch), NLP/typo-tolerance/autocomplete, AI ranking + personalization, sub-20ms latency at 1B+ queries/5s, multi-content-type", "confidence": "high", "note": "Mature hybrid search feature matrix (semantic + keyword, personalization, AI ranking) at very low latency and massive scale, frontier-adjacent for managed site/retrieval search. Rated 4; closed system so no independent ANN-Benchmarks number, basis is feature_matrix.", "sources": [ { "url": "https://www.algolia.com/products/ai-search/", "shows": "hybrid vector+keyword NeuralSearch, AI ranking, personalization, sub-20ms at 1B+ queries/5s", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Algolia AI Search, verified live June 2026. Proprietary API-first hosted search platform (keyword + semantic/vector hybrid, NeuralSearch). Open-source client libraries (InstantSearch etc.) but closed core. Sub-segment: search/retrieval infra. Note: primarily a human-facing site-search product; usable as an agent retrieval tool." }, { "product": "Browser Use", "org": "Browser Use", "type": "software", "description": "Open-source agent framework that makes websites accessible to AI agents, enabling autonomous web browsing, form filling, data extraction, and multi-step online task completion. Built on Playwright, it provides a structured way for LLMs to observe page state, plan actions, and execute browser interactions. 95K GitHub stars with 315 contributors, one of the fastest-growing projects in the agent space, filling the crucial gap of web-native agent capabilities.", "openness": { "score": 4, "class": "open_core", "components": "license:MIT(OSI, library core fully open);source:public;managed-tier:Browser-Use-Cloud(proxies/stealth/captcha/hosted browsers, proprietary)", "confidence": "high", "note": "Core library is fully MIT/OSI open (no feature-gated core); a separate proprietary Browser Use Cloud adds managed infra. Scored open_core at 4 because the OSS core is complete and the cloud is optional hosting, not a gated core.", "sources": [ { "url": "https://github.com/browser-use/browser-use", "shows": "MIT (OSI) license, full public source; Browser Use Cloud as optional managed tier", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~27M PyPI downloads in the last month (browser-use), strong real install/usage volume, not stars. 97.2k GitHub stars corroborate but are not the basis. Solidly in the 1-10M+ monthly-download band; one of the most-used agent browser-automation libraries.", "sources": [ { "url": "https://pypistats.org/packages/browser-use", "shows": "~26.96M PyPI downloads last month", "accessed": "2026-06-04" }, { "url": "https://github.com/browser-use/browser-use", "shows": "97.2k GitHub stars (corroborating)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "agent browser control: navigate/click/fill-forms/extract/multi-step tasks; LLM-driven DOM interaction; cloud infra adds proxy rotation, stealth fingerprinting, CAPTCHA solving, 1000+ integrations", "confidence": "medium", "note": "Leading OSS browser-automation-for-agents library with broad action coverage and a production cloud tier; widely adopted as the default agent browsing layer. Rated 4 on feature_matrix (multi-step web tasks, structured extraction, anti-block infra); no standard benchmark applies to a browser-control library.", "sources": [ { "url": "https://github.com/browser-use/browser-use", "shows": "agent browser automation feature set + cloud capabilities (proxies, stealth, CAPTCHA, integrations)", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Browser Use, verified live June 2026 (GitHub + PyPI). OSS Python library making websites accessible to AI agents (navigate/click/fill/extract). MIT license. Recategorized into agent_tools_protocols (browser/scrape sub-segment) per assignment. Managed Browser Use Cloud tier also exists." }, { "product": "Milvus", "org": "Zilliz", "type": "software", "description": "Cloud-native vector database built for scalable approximate nearest neighbor search at billion-scale. Supports hybrid search combining vector similarity with scalar filtering. The most-starred dedicated vector database on GitHub (44K stars) with strong enterprise adoption. Open-source core with Zilliz Cloud as the managed offering. The go-to choice for production RAG systems that need to scale.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0;source:public;LF-AI&Data-governed(Zilliz-Cloud is managed hosting, not gated core)", "confidence": "high", "note": "license:Apache-2.0;source:public;LF-AI&Data-governed(Zilliz-Cloud is managed hosting, not gated core)", "sources": [ { "url": "https://github.com/milvus-io/milvus/blob/master/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/milvus-io/milvus", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://en.wikipedia.org/wiki/Milvus_(vector_database)", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "high", "note": "Described as the default open source engine for billion-scale semantic search; LF AI & Data graduate. No precise verified deployment/download count, rated 3 on reported traction.", "sources": [ { "url": "https://github.com/milvus-io/milvus", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://zilliz.com/what-is-milvus", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "benchmark:ANN-Benchmarks", "value": null, "confidence": "high", "note": "High-performance cloud-native vector ANN search; covered by independent ANN-Benchmarks and first-party VectorDBBench. v2.6 adds Storage Format V2, advanced JSON, BM25 full-text search. Rated on ANN benchmark coverage / scale.", "sources": [ { "url": "https://github.com/milvus-io/milvus", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://milvus.io/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "v2.6.x stable (late 2025); v3.0 RC published May 2026; LF AI & Data project" }, { "product": "MCP Python SDK", "org": "Anthropic", "type": "software", "description": "Official Python implementation of the Model Context Protocol for building MCP clients and servers that give LLMs standardized access to tools, resources, and prompts. Bundles the FastMCP high-level framework, stdio/SSE/Streamable-HTTP transports, structured (Pydantic) output, and OAuth 2.1 auth.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT reference SDK.", "sources": [ { "url": "https://pypi.org/project/mcp/", "shows": "MIT, package 'mcp', author Anthropic PBC", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~23k stars; PyPI download volume not retrievable - stars_fallback (true usage likely higher as the reference SDK).", "sources": [ { "url": "https://github.com/modelcontextprotocol/python-sdk", "shows": "MIT, ~23.4k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "client+server; stdio/SSE/Streamable-HTTP; FastMCP; structured output; OAuth 2.1; elicitation", "confidence": "high", "note": "Most complete Python MCP toolkit.", "sources": [ { "url": "https://github.com/modelcontextprotocol/python-sdk", "shows": "feature set: client/server, transports, FastMCP, OAuth", "accessed": "2026-06-17" } ] }, "maturity": 3.8, "mature": false, "version_note": "MIT (package 'mcp', author Anthropic PBC). ~23k stars; very actively released." }, { "product": "MCP TypeScript SDK", "org": "Anthropic", "type": "software", "description": "Official TypeScript SDK for the Model Context Protocol (Node, Bun, Deno), providing server and client libraries with tools/resources/prompts, Streamable-HTTP + stdio transports, auth helpers, and Express/Hono/Node middleware. The most-downloaded MCP package by a wide margin.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT reference SDK.", "sources": [ { "url": "https://www.npmjs.com/package/@modelcontextprotocol/sdk", "shows": "MIT, ~154M downloads/30d", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 5, "signal_type": "usage_volume", "confidence": "high", "note": "~154M npm downloads in the last 30 days.", "sources": [ { "url": "https://www.npmjs.com/package/@modelcontextprotocol/sdk", "shows": "~154.2M downloads/last-30-days", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "client+server; Node/Bun/Deno; Streamable-HTTP/stdio; auth + framework middleware", "confidence": "high", "note": "Feature parity with the Python SDK; the canonical JS/TS MCP implementation.", "sources": [ { "url": "https://github.com/modelcontextprotocol/typescript-sdk", "shows": "feature set: client/server, multi-runtime, transports", "accessed": "2026-06-17" } ] }, "maturity": 5.0, "mature": true, "version_note": "MIT (@modelcontextprotocol/sdk). ~13k stars, ~154M npm downloads/last-30-days." }, { "product": "MCP Inspector", "org": "Anthropic", "type": "software", "description": "Developer tool for testing and debugging MCP servers, with a React web UI and a CLI mode. Includes a Node proxy supporting stdio/SSE/Streamable-HTTP, request history and live visualization, bearer-token auth, multi-server config, and DNS-rebinding protection.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/modelcontextprotocol/inspector", "shows": "MIT license file", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 4, "signal_type": "usage_volume", "confidence": "high", "note": "~847k npm downloads in the last 30 days.", "sources": [ { "url": "https://www.npmjs.com/package/@modelcontextprotocol/inspector", "shows": "~847k downloads/last-30-days", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "visual + CLI MCP debugging; all transports; request history/visualization; auth; multi-server config", "confidence": "high", "note": "Best-in-class for its debugging niche; narrower scope than the SDKs by design.", "sources": [ { "url": "https://github.com/modelcontextprotocol/inspector", "shows": "feature set: web UI + CLI debugging, all transports", "accessed": "2026-06-17" } ] }, "maturity": 4.0, "mature": false, "version_note": "MIT. ~10k stars, ~847k npm downloads/last-30-days." }, { "product": "GitHub MCP Server", "org": "GitHub (Microsoft)", "type": "software", "description": "GitHub's official MCP server (Go) that lets AI agents interact with the GitHub platform in natural language, exposing 20+ toolsets - Repositories, Issues, Pull Requests, Actions, Code Security, Dependabot, Discussions, Projects, Copilot, and more. Runs locally via Docker/binary or via GitHub's remote hosted endpoint.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT, first-party GitHub server.", "sources": [ { "url": "https://github.com/github/github-mcp-server", "shows": "MIT, ~30.8k stars, official GitHub server", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~31k stars; no download metric for Go/Docker distribution - stars_fallback (first-party backing implies higher real usage).", "sources": [ { "url": "https://github.com/github/github-mcp-server", "shows": "~30.8k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "20+ toolsets: repos, issues, PRs, Actions, code security, Dependabot, Discussions, Projects, Copilot", "confidence": "high", "note": "The most comprehensive GitHub-platform tool surface among MCP servers.", "sources": [ { "url": "https://github.com/github/github-mcp-server", "shows": "toolset list", "accessed": "2026-06-17" } ] }, "maturity": 3.8, "mature": false, "version_note": "MIT (c) 2025 GitHub. ~31k stars. Distributed as a Go binary/Docker image (no npm/pypi)." }, { "product": "Slack MCP Server", "org": "Dmitry Korotovsky", "type": "software", "description": "Actively maintained community Slack MCP server (Go) connecting AI agents to Slack workspaces over stdio/SSE/HTTP, with a 'stealth mode' needing no bot install or workspace permissions plus an OAuth mode. Exposes 18 tools (history, search, posting, reactions, channel/user management).", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/korotovsky/slack-mcp-server", "shows": "MIT, ~1.7k stars, v1.3.0", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 2, "signal_type": "stars_fallback", "confidence": "medium", "note": "~1.7k stars (smaller community project) - stars_fallback.", "sources": [ { "url": "https://github.com/korotovsky/slack-mcp-server", "shows": "~1.7k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "18 tools: history/replies/search, posting, reactions, channel/user/usergroup management, saved items; stealth + OAuth modes", "confidence": "medium", "note": "Broad Slack read/write surface with an unusual no-permission stealth mode.", "sources": [ { "url": "https://github.com/korotovsky/slack-mcp-server", "shows": "18-tool feature set", "accessed": "2026-06-17" } ] }, "maturity": 2.8, "mature": false, "version_note": "MIT. ~1.7k stars. There is no first-party official Slack MCP server - the original reference server was archived (servers-archived), making this the most popular maintained one." }, { "product": "Notion MCP Server", "org": "Notion", "type": "software", "description": "Notion's official MCP server (TypeScript), exposing ~22 tools over the Notion API for agents: search the workspace, query/retrieve/create/update data sources, read and write pages and blocks, manage properties and comments - with token-optimized responses tailored to agents.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT, first-party Notion server.", "sources": [ { "url": "https://github.com/makenotion/notion-mcp-server", "shows": "MIT, ~4.4k stars, official Notion server", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~4.4k stars plus npm distribution - stars_fallback (first-party backing implies higher real usage).", "sources": [ { "url": "https://github.com/makenotion/notion-mcp-server", "shows": "~4.4k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "~22 tools across search, data sources, pages, blocks, comments; agent-optimized output", "confidence": "medium", "note": "Comprehensive single-product Notion API coverage.", "sources": [ { "url": "https://github.com/makenotion/notion-mcp-server", "shows": "22-tool feature set", "accessed": "2026-06-17" } ] }, "maturity": 3.4, "mature": false, "version_note": "MIT. ~4.4k stars. First-party Notion (makenotion) server." }, { "product": "Filesystem MCP Server", "org": "Anthropic", "type": "software", "description": "The official reference Filesystem MCP server (TypeScript) in the modelcontextprotocol/servers monorepo, providing secure file operations within configurable allowed directories - read/write/edit files, directory trees, search, media reads - with the MCP Roots protocol for dynamic access scoping.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT reference server.", "sources": [ { "url": "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem", "shows": "MIT, reference filesystem server", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "low", "note": "Bundled reference server; monorepo ~87k stars are shared across servers - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/modelcontextprotocol/servers", "shows": "~87.4k monorepo stars, filesystem among active reference servers", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "granular read/write/edit; directory tree; search; media reads; sandboxed allowed-directory access; Roots support", "confidence": "high", "note": "Full local-filesystem surface with sandboxed access controls.", "sources": [ { "url": "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem", "shows": "tool list", "accessed": "2026-06-17" } ] }, "maturity": 3.4, "mature": false, "version_note": "MIT. One of the active reference servers; the monorepo has ~87k stars shared across all bundled servers, so per-server adoption isn't separable." }, { "product": "Postgres MCP Pro", "org": "Crystal DBA", "type": "software", "description": "Actively maintained PostgreSQL MCP server (Python) by Crystal DBA, 'Postgres MCP Pro', that goes beyond read-only querying to database performance and health tooling for agents: schema/object inspection, execute_sql with access modes, EXPLAIN with hypothetical-index simulation, slow-query analysis, index advising, and health checks.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT.", "sources": [ { "url": "https://github.com/crystaldba/postgres-mcp", "shows": "MIT, ~2.9k stars, Postgres MCP Pro", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~2.9k stars plus PyPI distribution - stars_fallback (capped at 3).", "sources": [ { "url": "https://github.com/crystaldba/postgres-mcp", "shows": "~2.9k stars", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "9 tools: schema/object inspection, execute_sql w/ access modes, EXPLAIN + hypothetical indexes, slow-query analysis, index advising, health checks", "confidence": "medium", "note": "Richest open Postgres MCP surface, well beyond the archived read-only reference server.", "sources": [ { "url": "https://github.com/crystaldba/postgres-mcp", "shows": "9-tool feature set", "accessed": "2026-06-17" } ] }, "maturity": 3.4, "mature": false, "version_note": "MIT. ~2.9k stars. The original reference Postgres server was archived (read-only since May 2025) and only did read-only queries, so this is the canonical maintained one." }, { "product": "Docling", "org": "IBM", "type": "software", "description": "Open-source SDK and CLI that parses PDF, DOCX, PPTX, XLSX, HTML, EPUB, images, and audio into unified structured output (Markdown, lossless JSON, HTML, DocTags) for gen-AI and RAG pipelines. Unlike web-URL-to-Markdown services (Firecrawl, Jina Reader), it does deep local document understanding of rich binary docs - layout/reading-order analysis, table-structure recognition, formula/code extraction, OCR, and VLM integration - running locally as a library.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(github.com/docling-project/docling);self-host:primary;no-feature-gated-core", "confidence": "high", "note": "MIT, local-first document parsing, fully self-hostable with integrated open-weight models.", "sources": [ { "url": "https://github.com/docling-project/docling", "shows": "MIT, ~61.8k stars, multi-format document parser", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~62k stars (exceptionally high) but PyPI download volume not surfaced - stars_fallback (capped at 3; true usage almost certainly higher).", "sources": [ { "url": "https://github.com/docling-project/docling", "shows": "~61.8k stars", "accessed": "2026-06-18" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "parses PDF/DOCX/PPTX/XLSX/HTML/EPUB/images/audio -> Markdown/JSON/HTML/DocTags; layout + reading-order; table-structure recognition; formula/code extraction; OCR; VLM integration", "confidence": "high", "note": "Deepest local document understanding among RAG ingestion tools; web-URL parsers lack rich binary-doc depth.", "sources": [ { "url": "https://github.com/docling-project/docling", "shows": "feature set: multi-format, layout, tables, OCR, VLM", "accessed": "2026-06-18" } ] }, "maturity": 3.8, "mature": false, "version_note": "MIT; originated at IBM Research, inducted into the Linux Foundation AI & Data Foundation (Docling Project) on 2025-04-29 under community governance. ~62k stars (exceptionally high)." }, { "product": "MarkItDown", "org": "Microsoft", "type": "software", "description": "Lightweight Python utility from Microsoft for converting files and office documents - PDF, DOCX, PPTX, XLSX, images (OCR), audio (transcription), HTML, CSV/JSON/XML, ZIP, EPub, even YouTube URLs - into Markdown optimized for LLMs and agents. Offers CLI, Python API, Docker, and a plugin system, with optional Azure Document Intelligence and LLM-vision for image descriptions.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI; repo LICENSE + pyproject, PyPI field null);source:public;no-feature-gated-core", "confidence": "high", "note": "Fully MIT; optional Azure/LLM features are pluggable, not gating.", "sources": [ { "url": "https://raw.githubusercontent.com/microsoft/markitdown/main/LICENSE", "shows": "MIT, Copyright Microsoft Corporation", "accessed": "2026-06-18" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "confidence": "medium", "note": "~155k stars; PyPI monthly volume not retrievable - stars_fallback (capped at 3; true usage higher given the Microsoft-backed package).", "sources": [ { "url": "https://github.com/microsoft/markitdown", "shows": "~155k stars, MIT, file->Markdown converter", "accessed": "2026-06-18" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "converts PDF/DOCX/PPTX/XLSX/images(OCR)/audio/HTML/CSV/JSON/XML/ZIP/EPub/YouTube -> Markdown; CLI + Python API + Docker; plugins; optional Azure Doc Intelligence + LLM vision", "confidence": "medium", "note": "Broadest format coverage among RAG ingestion tools; lighter on deep layout parsing than Docling.", "sources": [ { "url": "https://github.com/microsoft/markitdown", "shows": "format coverage + interfaces", "accessed": "2026-06-18" } ] }, "maturity": 3.4, "mature": false, "version_note": "MIT (PyPI license field is null but the repo LICENSE and pyproject declare MIT). ~155k stars. Lighter than Docling on layout/table depth but broader format coverage." } ] }, "ml_frameworks": { "label": "Core ML frameworks & libraries", "arc": "Infrastructure", "layer": "infrastructure", "stage": { "num": 5, "name": "Mature Open Ecosystem" }, "gaps": [], "products": [ { "product": "PyTorch", "org": "PyTorch Foundation (Linux Foundation)", "type": "software", "description": "PyTorch is a Python library for tensor computation with strong GPU acceleration and deep neural networks built on a tape-based autograd system. Originally developed at Meta, it is now governed by the PyTorch Foundation under the Linux Foundation. It is one of the two dominant deep learning frameworks and underpins most modern research and production model training.", "openness": { "score": 5, "class": "open_source", "components": "license:BSD-3-Clause(OSI);source:public;no feature-gated core", "confidence": "high", "note": "BSD 3-Clause license with multiple contributor copyright lines; full source public.", "sources": [ { "url": "https://github.com/pytorch/pytorch/blob/main/LICENSE", "shows": "BSD 3-Clause license body with Facebook/contributor copyright lines", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "88M+/mo", "signal_type": "usage_volume", "confidence": "high", "note": "88.6M PyPI downloads in the last 30 days.", "sources": [ { "url": "https://pepy.tech/projects/torch", "shows": "88,636,352 downloads in last 30 days", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Foundational deep learning framework: GPU-accelerated tensors, dynamic autograd, neural network primitives, distributed training.", "confidence": "high", "note": "Defines the model-training layer alongside TensorFlow.", "sources": [ { "url": "https://github.com/pytorch/pytorch", "shows": "README describes tensor computation with GPU acceleration and tape-based autograd", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. BSD 3-Clause license with multiple contributor copyright lines; full source public." }, { "product": "TensorFlow", "org": "Google", "type": "software", "description": "TensorFlow is an end-to-end open source platform for machine learning, offering stable Python and C++ APIs for building and deploying ML models across CPUs, GPUs, and TPUs. It was developed by Google Brain and remains maintained by Google with broad community involvement. It is one of the two dominant deep learning frameworks and powers large-scale research and production deployments.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 licensed with full public source.", "sources": [ { "url": "https://github.com/tensorflow/tensorflow", "shows": "Repo shows Apache License 2.0 and full source", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 21.06M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M, langgraph ~58M, fastmcp ~72.7M, all level 5); the prior level 4 here applied a stricter 5M-50M band used only in one ml_frameworks scoring batch, which placed these frameworks inconsistently below map peers of equal or lower volume. TensorFlow at 21M/mo is one of the two foundational training frameworks and clears the prevailing floor.", "sources": [ { "url": "https://pepy.tech/projects/tensorflow", "shows": "21,057,167 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Foundational ML platform: numerical computation graph engine, neural network training, multi-device (CPU/GPU/TPU) and edge deployment.", "confidence": "high", "note": "Defines the model-training layer alongside PyTorch.", "sources": [ { "url": "https://github.com/tensorflow/tensorflow", "shows": "README describes end-to-end ML platform with stable Python/C++ APIs", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed with full public source." }, { "product": "JAX", "org": "Google", "type": "software", "description": "JAX is a Python library for accelerator-oriented array computation and program transformation, combining NumPy-style arrays with composable function transforms (automatic differentiation, JIT compilation via XLA, and auto-vectorization). It is developed by the jax-ml organization (originating at Google) and is widely used for high-performance numerical computing and large-scale ML research.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 licensed with full public source.", "sources": [ { "url": "https://github.com/jax-ml/jax", "shows": "Repo shows Apache-2.0 license and full source", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 18.92M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M, langgraph ~58M, all level 5); the prior level 4 applied a stricter 5M-50M band used only in one ml_frameworks batch, which sat below map peers of equal volume. JAX at ~19M/mo is the foundation of a major ML ecosystem (Flax, the Google research stack) and clears the prevailing floor.", "sources": [ { "url": "https://pypistats.org/api/packages/jax/recent", "shows": "last_month downloads = 18,921,715", "accessed": "2026-06-25" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Composable function transforms over array code: autodiff, XLA JIT compilation for GPU/TPU, vectorization; foundation of a major ML ecosystem.", "confidence": "high", "note": "Defines the functional/accelerated-compute layer for ML research.", "sources": [ { "url": "https://github.com/jax-ml/jax", "shows": "README describes accelerator-oriented array computation and program transformation", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed with full public source." }, { "product": "Keras", "org": "Google", "type": "software", "description": "Keras is a high-level deep learning framework billed as 'Deep Learning for humans'; Keras 3 is multi-backend, running on JAX, TensorFlow, PyTorch, and OpenVINO. It is maintained by the keras-team organization (closely tied to Google) and reports use by nearly three million developers. It provides an accessible API for building and training models while letting users choose the underlying backend.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 licensed with full public source.", "sources": [ { "url": "https://github.com/keras-team/keras", "shows": "Repo shows Apache-2.0 license and full source", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 21.53M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M at level 5); the prior level 4 applied a stricter 5M-50M band used only in one ml_frameworks batch, below map peers of equal volume. Keras is the standard high-level training API across TF/JAX/PyTorch and clears the prevailing floor.", "sources": [ { "url": "https://pepy.tech/projects/keras", "shows": "21,532,407 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "High-level multi-backend deep learning API for building/training models over JAX, TensorFlow, PyTorch, and OpenVINO.", "confidence": "high", "note": "Central API layer but sits atop lower-level frameworks rather than defining the compute layer.", "sources": [ { "url": "https://github.com/keras-team/keras", "shows": "README describes multi-backend deep learning framework", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed with full public source." }, { "product": "Flax", "org": "Google DeepMind", "type": "software", "description": "Flax is a neural network library and ecosystem for JAX designed for flexibility, providing network components (Linear, Conv, BatchNorm, attention) plus training utilities like checkpointing and serialization. It is developed and maintained by Google DeepMind as an open source project. Its newer Flax NNX API emphasizes Python reference semantics for building networks with standard Python objects.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 licensed with full public source.", "sources": [ { "url": "https://github.com/google/flax", "shows": "Repo displays Apache-2.0 license and full source", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "5M+/mo", "signal_type": "usage_volume", "confidence": "high", "note": "5.3M PyPI downloads in the last 30 days, just above the 5M Level-4 threshold.", "sources": [ { "url": "https://pepy.tech/projects/flax", "shows": "5,302,085 downloads in last 30 days", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Neural network library on top of JAX: layers, attention, training utilities, Flax NNX API.", "confidence": "high", "note": "Key NN layer for the JAX ecosystem but built atop JAX rather than foundational itself.", "sources": [ { "url": "https://github.com/google/flax", "shows": "README describes neural network library and ecosystem for JAX", "accessed": "2026-06-22" } ] }, "maturity": 4.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed with full public source." }, { "product": "Transformers", "org": "Hugging Face", "type": "software", "description": "Transformers is a model-definition framework for state-of-the-art machine learning across text, vision, audio, and multimodal models, supporting both inference and training. It is maintained by Hugging Face and provides a unified API to load and run over a million pretrained model checkpoints from the Hugging Face Hub. It is a foundational library underpinning much of the modern open source ML and LLM ecosystem.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE file is the standard, unmodified Apache 2.0 text.", "sources": [ { "url": "https://github.com/huggingface/transformers/blob/main/LICENSE", "shows": "Standard Apache License 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "150M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads of 150,245,308 exceed the 50M+ Level 5 threshold.", "sources": [ { "url": "https://pypistats.org/api/packages/transformers/recent", "shows": "last_month downloads = 150,245,308", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Unified framework for defining, loading, fine-tuning, and running 1M+ pretrained models across text, vision, audio, and multimodal tasks.", "confidence": "high", "note": "Centralizes model definitions across training frameworks and inference engines; foundational to the ecosystem.", "sources": [ { "url": "https://github.com/huggingface/transformers", "shows": "Describes multimodal model-definition framework, 1M+ checkpoints", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE file is the standard, unmodified Apache 2.0 text." }, { "product": "Datasets", "org": "Hugging Face", "type": "software", "description": "Datasets is a lightweight Hugging Face library providing one-line dataloaders for thousands of public datasets and efficient data preprocessing for ML workflows. It supports CSV, JSON, Parquet, audio, image, and video formats with streaming, memory-mapping, and multi-framework integration (PyTorch, TensorFlow, JAX). It is widely used to feed training and evaluation pipelines.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE file is the standard, unmodified Apache 2.0 text.", "sources": [ { "url": "https://github.com/huggingface/datasets/blob/main/LICENSE", "shows": "Standard Apache License 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "120M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads of 120,316,114 exceed the 50M+ Level 5 threshold.", "sources": [ { "url": "https://pypistats.org/api/packages/datasets/recent", "shows": "last_month downloads = 120,316,114", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Data loading and preprocessing library: one-line dataset loading from the Hub plus fast manipulation of multi-format, multi-modal data with streaming.", "confidence": "high", "note": "Broad and central to data pipelines but scoped to data handling rather than modeling.", "sources": [ { "url": "https://github.com/huggingface/datasets", "shows": "Describes one-line dataloaders and efficient preprocessing", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE file is the standard, unmodified Apache 2.0 text." }, { "product": "Diffusers", "org": "Hugging Face", "type": "software", "description": "Diffusers is a PyTorch library from Hugging Face providing state-of-the-art pretrained diffusion models for generating images, audio, and 3D molecular structures. It offers ready-to-use inference pipelines, interchangeable noise schedulers, and pretrained models that serve as building blocks for custom diffusion systems. It is the de facto open source library for working with diffusion-based generative models.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE file is the standard, unmodified Apache 2.0 text.", "sources": [ { "url": "https://github.com/huggingface/diffusers/blob/main/LICENSE", "shows": "Standard Apache License 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "7M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads of 7,406,705 fall in the 5M-50M Level 4 band.", "sources": [ { "url": "https://pypistats.org/api/packages/diffusers/recent", "shows": "last_month downloads = 7,406,705", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Diffusion-model toolkit: inference pipelines, swappable schedulers, and pretrained models for image, audio, and 3D generation.", "confidence": "high", "note": "Deep but domain-specific to diffusion/generative models rather than general-purpose ML.", "sources": [ { "url": "https://github.com/huggingface/diffusers", "shows": "Describes pipelines, schedulers, and pretrained diffusion models", "accessed": "2026-06-22" } ] }, "maturity": 4.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE file is the standard, unmodified Apache 2.0 text." }, { "product": "Accelerate", "org": "Hugging Face", "type": "software", "description": "Accelerate is a Hugging Face PyTorch library that lets users run raw PyTorch training scripts across CPUs, multi-GPU, and TPU configurations with mixed precision, by adding only a few lines of code. It abstracts the boilerplate of distributed setup and device placement while leaving the training loop under the user's control, and provides a CLI for configuration. It is widely used as the distributed-training backbone for other Hugging Face libraries.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE file is the standard, unmodified Apache 2.0 text.", "sources": [ { "url": "https://github.com/huggingface/accelerate/blob/main/LICENSE", "shows": "Standard Apache License 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 25.01M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M at level 5); the prior note bucketed it into a 5M-50M=L4 band used only in one ml_frameworks batch, below map peers of equal volume. Accelerate is the standard distributed-training launcher under the HF stack (TRL/PEFT depend on it) and clears the prevailing floor.", "sources": [ { "url": "https://pepy.tech/projects/accelerate", "shows": "25,013,751 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Distributed-training abstraction: launches and runs PyTorch training on CPU/GPU/multi-GPU/TPU with mixed precision via minimal code changes.", "confidence": "high", "note": "Central infrastructure for distributed training but scoped to orchestration, not modeling.", "sources": [ { "url": "https://github.com/huggingface/accelerate", "shows": "Describes few-line distributed/mixed-precision training across devices", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE file is the standard, unmodified Apache 2.0 text." }, { "product": "Gradio", "org": "Hugging Face", "type": "software", "description": "Gradio is an open source Python package for quickly building web demos and applications for machine learning models, APIs, or arbitrary Python functions without JavaScript, CSS, or web-hosting knowledge. It provides the Interface, Blocks, and ChatInterface classes plus built-in public sharing of demos. Originally an independent project, it is now maintained under Hugging Face after the company acquired Gradio.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE file is the standard, unmodified Apache 2.0 text.", "sources": [ { "url": "https://github.com/gradio-app/gradio/blob/main/LICENSE", "shows": "Standard Apache License 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 12.69M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M at level 5); the prior note explicitly bucketed it into a 5M-50M=L4 band that only this ml_frameworks batch used, below map peers of equal volume. Gradio is the default ML demo/UI framework (the standard backend for HF Spaces) and clears the prevailing floor. Conservative: it sits just above 10M, so the bump rests on volume plus ecosystem-standard status, not volume alone.", "sources": [ { "url": "https://pepy.tech/projects/gradio", "shows": "12,685,121 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "UI/demo framework: build shareable web interfaces and chatbots for ML models and Python functions with no frontend code.", "confidence": "high", "note": "Broad and widely used for demos and apps but scoped to the interface layer, not model training.", "sources": [ { "url": "https://github.com/gradio-app/gradio", "shows": "Describes Interface/Blocks/ChatInterface and public sharing", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE file is the standard, unmodified Apache 2.0 text." }, { "product": "Safetensors", "org": "Hugging Face", "type": "software", "description": "Safetensors is a file format and library for safely storing and loading tensors, designed as a secure alternative to Python pickle that avoids arbitrary code execution and supports fast zero-copy loading. It is implemented in Rust with Python bindings and works across PyTorch, TensorFlow, and other frameworks. Maintained by Hugging Face, it has become the default weight format for models on the Hugging Face Hub.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE body is verbatim Apache License 2.0; full Rust/Python source public.", "sources": [ { "url": "https://github.com/huggingface/safetensors/blob/main/LICENSE", "shows": "LICENSE file is the verbatim Apache License Version 2.0", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "78M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last-month downloads of 78,250,137 exceed the 50M+ threshold.", "sources": [ { "url": "https://pypistats.org/api/packages/safetensors/recent", "shows": "last_month = 78,250,137 PyPI downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Secure, zero-copy tensor serialization format and library used cross-framework.", "confidence": "high", "note": "Narrow scope (storage format) but central as the default Hub weight format.", "sources": [ { "url": "https://github.com/huggingface/safetensors", "shows": "README describes safe, zero-copy, multi-framework tensor format", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE body is verbatim Apache License 2.0; full Rust/Python source public." }, { "product": "Tokenizers", "org": "Hugging Face", "type": "software", "description": "Tokenizers is a library providing fast implementations of modern tokenization algorithms including Byte-Pair Encoding, WordPiece, and Unigram. It is written in Rust with bindings for Python, Node.js, and Ruby, and emphasizes high-throughput training and inference. Maintained by Hugging Face, it underpins tokenization in the Transformers library and many production NLP pipelines.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE body is verbatim Apache License 2.0; full Rust/Python source public.", "sources": [ { "url": "https://github.com/huggingface/tokenizers/blob/main/LICENSE", "shows": "LICENSE file is the verbatim Apache License Version 2.0", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "189M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last-month downloads of 189,770,639 exceed the 50M+ threshold.", "sources": [ { "url": "https://pypistats.org/api/packages/tokenizers/recent", "shows": "last_month = 189,770,639 PyPI downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Foundational, fast multi-algorithm tokenization with training, normalization, and pre/post-processing.", "confidence": "high", "note": "Broad feature set and central dependency of the Transformers ecosystem.", "sources": [ { "url": "https://github.com/huggingface/tokenizers", "shows": "README lists BPE/WordPiece/Unigram, normalization, multi-language bindings", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE body is verbatim Apache License 2.0; full Rust/Python source public." }, { "product": "huggingface_hub", "org": "Hugging Face", "type": "software", "description": "huggingface_hub is the official Python client for the Hugging Face Hub, enabling programmatic downloading and uploading of files, repository management, running inference, and search across models, datasets, and Spaces. It serves as the integration layer that other ML libraries use to host and retrieve content on the Hub. It is maintained by Hugging Face.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE body is verbatim Apache License 2.0; full source public.", "sources": [ { "url": "https://github.com/huggingface/huggingface_hub/blob/main/LICENSE", "shows": "LICENSE file is the verbatim Apache License Version 2.0", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "331M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last-month downloads of 331,796,503 exceed the 50M+ threshold.", "sources": [ { "url": "https://pypistats.org/api/packages/huggingface-hub/recent", "shows": "last_month = 331,796,503 PyPI downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Foundational client for Hub up/download, repo management, inference, and library integration.", "confidence": "high", "note": "Central dependency across the Hugging Face ecosystem.", "sources": [ { "url": "https://github.com/huggingface/huggingface_hub", "shows": "README describes downloading/uploading, repo management, inference, search", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE body is verbatim Apache License 2.0; full source public." }, { "product": "Optimum", "org": "Hugging Face", "type": "software", "description": "Optimum is an extension of Transformers, Diffusers, TIMM, and Sentence-Transformers that provides tools to optimize training and inference on targeted hardware. It supports accelerators and runtimes including ONNX Runtime, OpenVINO, Intel Gaudi, AWS Trainium/Inferentia, and NVIDIA TensorRT-LLM, and handles model export and quantization. It is maintained by Hugging Face.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE body is verbatim Apache License 2.0; full source public.", "sources": [ { "url": "https://github.com/huggingface/optimum/blob/main/LICENSE", "shows": "LICENSE file is the verbatim Apache License Version 2.0", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "1.7M+/month", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last-month downloads of 1,759,764 fall in the 500k-5M range.", "sources": [ { "url": "https://pypistats.org/api/packages/optimum/recent", "shows": "last_month = 1,759,764 PyPI downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Hardware-targeted optimization, export, and quantization across multiple accelerators/runtimes.", "confidence": "high", "note": "Broad multi-backend optimization layer, though dependent on the core Transformers stack.", "sources": [ { "url": "https://github.com/huggingface/optimum", "shows": "README lists ONNX Runtime, OpenVINO, Gaudi, Trainium/Inferentia, TensorRT-LLM support", "accessed": "2026-06-22" } ] }, "maturity": 3.4, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE body is verbatim Apache License 2.0; full source public." }, { "product": "SentencePiece", "org": "Google", "type": "software", "description": "SentencePiece is an unsupervised text tokenizer and detokenizer that implements subword units (Byte-Pair Encoding and unigram language model) directly from raw text, without language-specific pre-tokenization. It treats input as a stream of Unicode characters, enabling a language-independent, end-to-end pipeline useful for languages without clear word boundaries. It is developed and maintained by Google.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "LICENSE body is verbatim Apache License 2.0; full C++/Python source public.", "sources": [ { "url": "https://github.com/google/sentencepiece/blob/master/LICENSE", "shows": "LICENSE file is the verbatim Apache License Version 2.0", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 37.65M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, langgraph ~58M at level 5); the prior note bucketed it into a 5M-50M=L4 band used only in one ml_frameworks batch, leaving a 37M/mo tokenizer below map peers of equal or lower volume. SentencePiece is the de facto subword tokenizer underpinning most LLM/NMT pipelines and clears the prevailing floor.", "sources": [ { "url": "https://pepy.tech/projects/sentencepiece", "shows": "37,654,106 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Language-independent subword tokenization (BPE/unigram) trained directly from raw text.", "confidence": "high", "note": "Widely used tokenizer underpinning many LLM and NMT models, though narrow in scope.", "sources": [ { "url": "https://github.com/google/sentencepiece", "shows": "README describes BPE and unigram subword units, language-independent tokenization", "accessed": "2026-06-22" } ] }, "maturity": 4.6, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. LICENSE body is verbatim Apache License 2.0; full C++/Python source public." }, { "product": "ONNX", "org": "LF AI & Data (Linux Foundation)", "type": "software", "description": "ONNX (Open Neural Network Exchange) is an open standard format and specification for representing machine learning models, defining an extensible computation graph model plus built-in operators and standard data types. It enables interoperability so models can move between frameworks like PyTorch, TensorFlow, and scikit-learn. It is governed under the Linux Foundation as part of LF AI & Data.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 license body confirmed; full source public on GitHub.", "sources": [ { "url": "https://github.com/onnx/onnx/blob/main/LICENSE", "shows": "Apache License Version 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. 20.65M PyPI downloads in the last 30 days. The map's prevailing usage_volume level-5 floor is >10M/mo (cf. pydantic-ai ~31M, llama-index ~10.18M at level 5; the companion onnxruntime is already level 5 at ~74.7M); the prior level 4 applied the stricter ml_frameworks-batch band. ONNX is the de facto cross-framework model-exchange standard and clears the floor.", "sources": [ { "url": "https://pepy.tech/projects/onnx", "shows": "20,653,241 downloads in last 30 days", "accessed": "2026-06-25" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Foundational open model-exchange format and operator spec underpinning cross-framework interoperability.", "confidence": "high", "note": "Defines the graph model, operators, and data types used as the de-facto interchange standard.", "sources": [ { "url": "https://github.com/onnx/onnx", "shows": "README: open format with extensible computation graph, built-in operators", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 license body confirmed; full source public on GitHub." }, { "product": "Triton", "org": "Triton (triton-lang)", "type": "software", "description": "Triton is a Python-embedded language and compiler for writing highly efficient custom GPU kernels (deep-learning primitives), aiming for higher productivity than CUDA with more flexibility than other DSLs. It compiles via MLIR and targets NVIDIA GPUs and AMD GPUs, with CPU support in progress. Originally created by Philippe Tillet at OpenAI, it is now developed under the triton-lang organization. It is widely used as the kernel backend for projects like PyTorch's inductor.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no feature-gated core", "confidence": "high", "note": "MIT license body confirmed (Tillet/OpenAI copyright); full source public.", "sources": [ { "url": "https://github.com/triton-lang/triton/blob/main/LICENSE", "shows": "MIT License text, Copyright 2018-2020 Philippe Tillet / 2020-2022 OpenAI", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": "50M+/mo", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads ~64.7M.", "sources": [ { "url": "https://pypistats.org/api/packages/triton/recent", "shows": "last_month=64,708,650 downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "GPU kernel language and MLIR-based compiler for custom high-performance deep-learning operators.", "confidence": "high", "note": "Foundational kernel-authoring layer (e.g. PyTorch inductor backend) across NVIDIA/AMD GPUs.", "sources": [ { "url": "https://github.com/triton-lang/triton", "shows": "README: language+compiler for fast custom DL primitives, MLIR-based", "accessed": "2026-06-22" } ] }, "maturity": 5.0, "mature": true, "version_note": "Verified live 2026-06-22 via primary sources. MIT license body confirmed (Tillet/OpenAI copyright); full source public." }, { "product": "DeepSpeed", "org": "Microsoft", "type": "software", "description": "DeepSpeed is a deep-learning optimization library for efficient distributed training and inference of very large models. It provides system-level innovations including ZeRO memory optimization, ZeRO-Infinity, 3D parallelism, and mixture-of-experts support, and has powered training of models like BLOOM (176B) and Megatron-Turing NLG (530B). It is maintained by Microsoft and integrates with Hugging Face Transformers, PyTorch Lightning, and others.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 license body confirmed; full source public.", "sources": [ { "url": "https://github.com/deepspeedai/DeepSpeed/blob/master/LICENSE", "shows": "Apache License Version 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "~1.4M/mo", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads ~1.37M.", "sources": [ { "url": "https://pypistats.org/api/packages/deepspeed/recent", "shows": "last_month=1,374,390 downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Large-scale distributed training/inference optimization library (ZeRO, 3D parallelism, MoE).", "confidence": "high", "note": "Foundational scale-out training stack used for many of the largest published models.", "sources": [ { "url": "https://github.com/deepspeedai/DeepSpeed", "shows": "README: ZeRO, ZeRO-Infinity, 3D-Parallelism, MoE; powered BLOOM 176B", "accessed": "2026-06-22" } ] }, "maturity": 3.8, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 license body confirmed; full source public." }, { "product": "OpenVINO", "org": "Intel", "type": "software", "description": "OpenVINO is an open source toolkit for optimizing and deploying AI inference across computer vision, speech, NLP, and generative AI. It converts models from PyTorch, TensorFlow, and ONNX and deploys across CPUs (x86, ARM), Intel integrated/discrete GPUs, and Intel NPUs, with a GenAI API for LLM pipelines. It is developed by Intel and integrates with Hugging Face Optimum Intel and vLLM.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;no feature-gated core", "confidence": "high", "note": "Apache-2.0 license body confirmed; full source public.", "sources": [ { "url": "https://github.com/openvinotoolkit/openvino/blob/master/LICENSE", "shows": "Apache License Version 2.0 full text", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "~1.4M/mo", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI last_month downloads ~1.38M.", "sources": [ { "url": "https://pypistats.org/api/packages/openvino/recent", "shows": "last_month=1,379,327 downloads", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Inference optimization/deployment toolkit across Intel CPU/GPU/NPU with multi-framework model conversion.", "confidence": "high", "note": "Broad inference runtime with model conversion and GenAI API, primarily Intel-hardware oriented.", "sources": [ { "url": "https://github.com/openvinotoolkit/openvino", "shows": "README: optimize/deploy inference, convert from PyTorch/TF/ONNX, CPU/GPU/NPU targets", "accessed": "2026-06-22" } ] }, "maturity": 3.4, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 license body confirmed; full source public." }, { "product": "GGML", "org": "ggml-org (Georgi Gerganov)", "type": "software", "description": "GGML is a low-level C tensor library for machine learning with no third-party dependencies, zero runtime memory allocations, integer quantization support, automatic differentiation, and optimizers. It runs cross-platform and is the tensor backend behind llama.cpp and whisper.cpp. It is maintained by ggml.ai / Georgi Gerganov; much active development now occurs in the dependent llama.cpp and whisper.cpp repos.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;no feature-gated core", "confidence": "high", "note": "MIT license body confirmed (Copyright 2023-2026 The ggml authors); full source public.", "sources": [ { "url": "https://github.com/ggml-org/ggml/blob/master/LICENSE", "shows": "MIT License text, Copyright (c) 2023-2026 The ggml authors", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "~14.9k stars", "signal_type": "stars_fallback", "confidence": "medium", "note": "No PyPI package; GitHub shows ~14.9k stars (and underpins much higher-adoption llama.cpp).", "sources": [ { "url": "https://github.com/ggml-org/ggml", "shows": "GitHub repo ~14,900 stars", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Dependency-free C tensor library with quantization and autodiff; backbone of llama.cpp/whisper.cpp inference.", "confidence": "high", "note": "Foundational low-level runtime enabling efficient on-device LLM/ASR inference.", "sources": [ { "url": "https://github.com/ggml-org/ggml", "shows": "README: tensor library, quantization, autodiff, no deps, zero runtime allocs", "accessed": "2026-06-22" } ] }, "maturity": 3.8, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. MIT license body confirmed (Copyright 2023-2026 The ggml authors); full source public." }, { "product": "Syft / PySyft", "org": "OpenMined", "type": "software", "description": "PySyft is OpenMined's open source library for privacy-preserving and remote data science, letting data scientists run computations on private data they cannot see or copy via secure 'Datasite' servers. It supports techniques associated with federated learning and differential privacy and offers a NumPy-like remote analysis interface. It is one of the most established open frameworks for collaborative analysis on sensitive data.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI permissive);source:public", "confidence": "high", "note": "Apache-2.0 licensed with full public source code.", "sources": [ { "url": "https://github.com/OpenMined/PySyft", "shows": "Apache-2.0 license, public source, 9.9k stars", "accessed": "2026-06-22" }, { "url": "https://pypi.org/project/syft/", "shows": "PyPI metadata lists License Apache-2.0, source OpenMined/PySyft", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "broad", "signal_type": "stars_fallback", "confidence": "medium", "note": "9.9k GitHub stars (capped at 3 per stars-fallback rule); ~10.8k monthly PyPI downloads and 1M+ all-time.", "sources": [ { "url": "https://github.com/OpenMined/PySyft", "shows": "9.9k GitHub stars", "accessed": "2026-06-22" }, { "url": "https://pepy.tech/project/syft", "shows": "1,043,989 total downloads; 10,846 last 30 days", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Remote/privacy-preserving data science: submit computations to private datasets via Datasite servers, NumPy-like remote analysis, sandboxed job execution, cross-platform.", "confidence": "high", "note": "Mature, feature-rich platform for remote data science on private data.", "sources": [ { "url": "https://pypi.org/project/syft/", "shows": "remote data science, Datasite servers, NumPy-like analysis", "accessed": "2026-06-22" } ] }, "maturity": 3.4, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Apache-2.0 licensed with full public source code." }, { "product": "Feluda", "org": "Tattle", "type": "software", "description": "Feluda is a configurable engine by Tattle for analysing multilingual, multimodal content (images, text, video), built around a modular 'operators' plugin architecture. It supports vector embeddings, hash-based search, text extraction from images, and entity extraction. It is open infrastructure for analysing misinformation and social-media content, particularly in Indian-language contexts.", "openness": { "score": 5, "class": "open_source", "components": "license:GPL-3.0(OSI copyleft);source:public", "confidence": "high", "note": "GPL-3.0 licensed open source project with public source.", "sources": [ { "url": "https://github.com/tattle-made/feluda", "shows": "GPL-3.0 license, public source under tattle-made org", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "niche", "signal_type": "stars_fallback", "confidence": "high", "note": "96 GitHub stars; no PyPI package found.", "sources": [ { "url": "https://github.com/tattle-made/feluda", "shows": "96 GitHub stars", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "Configurable multilingual/multimodal content-analysis engine with modular operators: vector embeddings, hash-based search, image text extraction, entity extraction.", "confidence": "high", "note": "Flexible modular engine focused on multimodal content analysis.", "sources": [ { "url": "https://github.com/tattle-made/feluda", "shows": "README: operators, embeddings, hash search, text/entity extraction", "accessed": "2026-06-22" } ] }, "maturity": 2.4, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. GPL-3.0 licensed open source project with public source." } ] }, "deployment": { "label": "Deployment", "arc": "Infrastructure", "layer": "infrastructure", "stage": { "num": 4, "name": "Competitive Open Ecosystem" }, "gaps": [ "maturity" ], "products": [ { "product": "E2B Sandbox", "org": "E2B", "type": "software", "description": "Open-source secure cloud runtime that gives AI agents an ephemeral Linux microVM (Firecracker-based) controlled via a Python/TypeScript SDK. Picked over hyperscaler functions for sub-150ms cold starts, agent-native lifecycle (create/use/destroy), and the fact that the SDK + core are open and self-hostable via the e2b-dev/infra repo. ~12.3k stars on the SDK repo, used by major coding-agent products and growing fast in the agent-infra category.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI);source:public(SDKs+infra, Terraform self-host);managed-tier:e2b.dev hosted sandbox cloud (proprietary)", "confidence": "high", "note": "Apache-2.0 OSS core (SDKs + self-hostable infra) with a proprietary managed cloud; classic open-core per recipe (E2B named as open-core example).", "sources": [ { "url": "https://github.com/e2b-dev/E2B", "shows": "Apache-2.0 license; open source sandbox infra + Terraform self-host; managed cloud at e2b.dev", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI: ~4.58M downloads last month for the e2b package (primary). Vendor reports 1B+ sandboxes started, 94% of Fortune 100 as customers; named users incl. Perplexity, Hugging Face, Manus, Groq. Solidly 1M-10M monthly install band.", "sources": [ { "url": "https://pypistats.org/packages/e2b", "shows": "~4,583,328 downloads last month", "accessed": "2026-06-04" }, { "url": "https://e2b.dev/", "shows": "1B+ sandboxes started; named production customers (Perplexity, Hugging Face, Manus, Groq)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:VM (Firecracker microVM, strong); cold-start:fast (sub-second class); language/runtime breadth:Python+JS/TS code interpreter + custom templates; persistence:filesystem; scale:1B+ sandboxes started", "confidence": "high", "note": "VM-grade isolation (Firecracker) is the strongest tier in the feature matrix; broad SDK/runtime support and proven scale. Below the frontier-definer Firecracker itself (which it builds on), so 4 not 5.", "sources": [ { "url": "https://e2b.dev/", "shows": "each sandbox powered by Firecracker microVM; 1B+ sandboxes started", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "E2B open source sandbox infra for running AI-generated code; CLI @e2b/cli@2.10.3 (Jun 3 2026) confirmed live on GitHub June 2026. Isolation via Firecracker microVMs. OSS core + managed cloud (e2b.dev)." }, { "product": "Daytona Sandbox", "org": "Daytona", "type": "software", "description": "Open-source agent-native compute platform that provisions sub-200ms stateful sandboxes (start, pause, fork, snapshot, destroy) for AI agents. Picked over E2B for snapshot/fork semantics and an enterprise BYOC story. Raised $24M Series A in Feb 2026 led by FirstMark with strategic investment from Datadog and Figma Ventures, ~72k stars on GitHub.", "openness": { "score": 3, "class": "open_core", "components": "license:AGPL-3.0(OSI, copyleft);source:public;managed-tier:app.daytona.io hosted cloud + hybrid (proprietary)", "confidence": "high", "note": "AGPL-3.0 (OSI-approved but strong copyleft) core plus a proprietary managed cloud => open_core. Scored 3 rather than 4: AGPL copyleft constrains commercial reuse vs the permissive-Apache open-core peers, tempering effective openness.", "sources": [ { "url": "https://github.com/daytonaio/daytona", "shows": "AGPL-3.0 license; secure runtime for AI-generated code; managed cloud at app.daytona.io", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "PyPI: ~8.31M downloads last month for the daytona package (primary). Note part of this volume is inherited from the prior dev-environment product line, but the package now ships the sandbox SDK. Solidly 1M-10M band.", "sources": [ { "url": "https://pypistats.org/packages/daytona", "shows": "~8,313,410 downloads last month", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:dedicated kernel/filesystem/network stack per sandbox (VM-class, 'full composable computer'); cold-start:<90ms; language/runtime breadth:Python+TypeScript+JS; persistence:persistent state across sessions; scale:elastic", "confidence": "high", "note": "Strong isolation (dedicated kernel + isolated net/fs, allocated vCPU/RAM/disk) with sub-90ms cold-start and persistent sessions; among the strongest agent-runtime feature sets, just below the kernel/microVM frontier-definers.", "sources": [ { "url": "https://github.com/daytonaio/daytona", "shows": "dedicated kernel/filesystem/network stack per sandbox; <90ms spin-up; persistent state; Python/TS/JS", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Daytona secure/elastic runtime for AI-generated code execution; v0.184.0 (Jun 3 2026) live on GitHub June 2026. Repo now AGPL-3.0 (note: Daytona pivoted from dev-environments to agent code-execution sandboxes). Sub-90ms sandbox spin-up; managed cloud at app.daytona.io." }, { "product": "Firecracker", "org": "Amazon Web Services", "type": "software", "description": "Firecracker microVM virtualization; v1.16.0 (Jun 4 2026) confirmed live on GitHub. Rust microVM monitor (KVM) for secure multi-tenant function/container workloads; powers AWS Lambda + Fargate.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Rust 78.7%);governance:AWS-led open project;no-feature-gated-core", "confidence": "high", "note": "Fully Apache-2.0, source public, no gated core; AWS-developed but openly governed (recipe lists Firecracker as an open_source exemplar).", "sources": [ { "url": "https://github.com/firecracker-microvm/firecracker", "shows": "Apache-2.0 license; v1.16.0 (Jun 4 2026); powers AWS Lambda and Fargate", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Underpins AWS Lambda and AWS Fargate, among the highest-volume serverless platforms in the world (trillions of invocations), plus downstream use in Kata Containers, Flintlock, and E2B. Production reach far exceeds >10M end-equivalent. (34.8k stars corroborate only.)", "sources": [ { "url": "https://github.com/firecracker-microvm/firecracker", "shows": "developed at AWS; powers AWS Lambda and AWS Fargate; integrated into Kata Containers and Flintlock", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "isolation:hardware-virtualized microVM (KVM), strongest isolation tier; cold-start:~125ms boot, <5MiB memory overhead per microVM; language/runtime breadth:runs any OCI/container or function workload (kernel-level, language-agnostic); scale:powers Lambda/Fargate fleets", "confidence": "high", "note": "Frontier-definer for the deployment category: hardware-VM isolation at container-like speed/density, the reference microVM that the managed sandbox layer (E2B, others) builds on. Defines the strongest point on the isolation x cold-start x scale matrix.", "sources": [ { "url": "https://github.com/firecracker-microvm/firecracker", "shows": "lightweight microVMs combining hardware virtualization security with container speed; minimal memory footprint/attack surface", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": true, "version_note": "Firecracker microVM virtualization; v1.16.0 (Jun 4 2026) confirmed live on GitHub. Rust microVM monitor (KVM) for secure multi-tenant function/container workloads; powers AWS Lambda + Fargate." }, { "product": "gVisor", "org": "Google", "type": "software", "description": "gVisor application-kernel sandbox; repo live June 2026 (rolling date-tagged releases, no GitHub Releases page). Go userspace kernel intercepting syscalls (Sentry + Gofer); used as the sandbox runtime behind Google Cloud Run and GKE Sandbox.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Go);governance:Google-led open project;no-feature-gated-core", "confidence": "high", "note": "Fully Apache-2.0 application kernel, source public, no gated core (recipe lists gVisor as an open_source exemplar).", "sources": [ { "url": "https://github.com/google/gvisor", "shows": "Apache-2.0 license; application kernel implementing Linux-like interface in userspace (Go)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Underpins Google Cloud Run ('Powered by gVisor' on gvisor.dev) and is the sandbox layer for GKE Sandbox / App Engine / Cloud Functions, billions of container executions across Google Cloud. Reach far exceeds >10M end-equivalent. (18.5k stars corroborate only.)", "sources": [ { "url": "https://gvisor.dev/docs/", "shows": "'Powered by gVisor' linking to Google Cloud Run as a production user", "accessed": "2026-06-04" }, { "url": "https://github.com/google/gvisor", "shows": "Google-developed sandbox application kernel in production", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "isolation:application-kernel (userspace syscall interception, defense-in-depth), distinct strong-isolation third approach between seccomp and full VM; cold-start:fast userspace startup; language/runtime breadth:any Linux container workload (kernel-level); scale:powers Cloud Run/GKE Sandbox", "confidence": "high", "note": "Frontier-definer: a novel application-kernel isolation model giving VM-class security at userspace footprint/startup, the reference sandbox for Google Cloud's serverless containers. Co-frontier with Firecracker (different isolation philosophy).", "sources": [ { "url": "https://github.com/google/gvisor", "shows": "intercepts syscalls as a userspace application kernel; VM-class security benefits at userspace footprint and startup", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": true, "version_note": "gVisor application-kernel sandbox; repo live June 2026 (rolling date-tagged releases, no GitHub Releases page). Go userspace kernel intercepting syscalls (Sentry + Gofer); powers Google Cloud Run." }, { "product": "Kata Containers", "org": "OpenInfra Foundation", "type": "software", "description": "Kata Containers, lightweight VMs that feel like containers; v3.31.0 (May 19 2026) confirmed live on GitHub. OpenInfra Foundation governed (governance not explicit on repo landing page but established as an OpenInfra project). Can use Firecracker/QEMU as the VMM.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;governance:open community (OpenInfra Foundation);no-feature-gated-core", "confidence": "high", "note": "Fully Apache-2.0, source public, vendor-neutral open-foundation governance, no managed-tier gating from the project itself.", "sources": [ { "url": "https://github.com/kata-containers/kata-containers", "shows": "Apache-2.0 license; lightweight VMs with container UX + VM isolation; v3.31.0 (May 19 2026)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Established CNCF/OpenInfra-adjacent container-runtime project integrated into Kubernetes runtimeclass deployments; broad but operator-concentrated infrastructure adoption rather than a mass per-user count. No clean download/user figure verified; ~8k stars (corroborate only, not the basis). Level 3 reflects specialist infra reach.", "sources": [ { "url": "https://github.com/kata-containers/kata-containers", "shows": "standard implementation of lightweight VM containers; integrated runtime; 8k stars", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:hardware VM (pluggable VMM, QEMU/Firecracker/Cloud Hypervisor), strong VM-class isolation; cold-start:VM boot (slower than pure userspace); language/runtime breadth:any OCI container (language-agnostic, K8s-native); persistence:container lifecycle; scale:K8s clusters", "confidence": "high", "note": "VM-grade isolation with OCI/Kubernetes compatibility and pluggable hypervisors; very strong isolation but higher cold-start than microVM/userspace frontier-definers, so 4.", "sources": [ { "url": "https://github.com/kata-containers/kata-containers", "shows": "lightweight VMs performing like containers with VM workload-isolation/security; integrated with Firecracker", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Kata Containers, lightweight VMs that feel like containers; v3.31.0 (May 19 2026) confirmed live on GitHub. OpenInfra Foundation governed (governance not explicit on repo landing page but established as an OpenInfra project). Can use Firecracker/QEMU as the VMM." }, { "product": "Beta9", "org": "Beam Cloud", "type": "software", "description": "Open-source Docker-based sandbox runtime positioned as a self-hostable E2B alternative, with first-class GPU support, Python and Node.js runtimes, and no session time limits. Picked when teams need to keep agent execution on their own infra (HIPAA/SOC2/BYOC) and want GPU workloads alongside CPU sandboxes. ~1.6k stars, AGPL-3.0 licensed with a managed cloud offering.", "openness": { "score": 3, "class": "open_core", "components": "license:AGPL-3.0(OSI, copyleft);source:public;managed-tier:Beam (beam.cloud) fully-managed GPU cloud (proprietary)", "confidence": "high", "note": "AGPL-3.0 (OSI but strong copyleft) engine plus proprietary managed Beam cloud => open_core. Scored 3 (vs 4) for AGPL copyleft tempering commercial reuse, consistent with Daytona's treatment.", "sources": [ { "url": "https://github.com/beam-cloud/beta9", "shows": "AGPL-3.0 license; open source serverless AI runtime; engine powering managed Beam cloud (beam.cloud)", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "No PyPI/usage figure for the OSS runtime surfaced and no disclosed Beam customer/usage count; only ~1.7k GitHub stars available as a signal. stars_fallback caps level at 3; placed at 2 given the small star base and absence of any volume signal. Low confidence.", "sources": [ { "url": "https://github.com/beam-cloud/beta9", "shows": "1.7k stars; no published download/usage figures", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation:container (serverless containers, not VM/microVM per repo description); cold-start:fast container launch; language/runtime breadth:Python AI workloads + GPU (4090/H100); persistence:volume storage; scale:parallelize across hundreds of containers + autoscaling", "confidence": "medium", "note": "Capable serverless container runtime with GPU support, volumes, autoscaling and sandbox primitives, but container-level (not VM/microVM) isolation places it below the VM-grade peers on the isolation-strength dimension; mid-tier.", "sources": [ { "url": "https://github.com/beam-cloud/beta9", "shows": "serverless runtime: rapid container launches, parallelize across hundreds of containers, GPU support, volume storage, sandboxes", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "Beta9, open source serverless AI runtime (sandboxes, serverless inference endpoints, task queues); worker v0.1.597 (Jun 4 2026) confirmed live on GitHub. OSS engine powering managed Beam cloud (beam.cloud)." }, { "product": "OpenSandbox", "org": "Alibaba", "type": "software", "description": "Apache-2.0 sandbox runtime from Alibaba aimed squarely at AI agents, defines a 'Sandbox Protocol' with standard lifecycle/execution APIs and ships multi-language SDKs (Python, Java, JS, .NET) plus Docker and Kubernetes runtimes. Picked by teams wanting an open protocol-driven alternative to E2B at K8s scale; ~10.8k stars and 910 commits in the last 90d signal serious investment.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(Go execd + multi-language SDKs);runtimes:Docker/Kubernetes;no-managed-tier-observed;no-feature-gated-core", "confidence": "high", "note": "Apache-2.0, fully self-hostable, no managed/enterprise gating observed => open_source (not open_core). Alibaba says it open sourced the sandbox infra used internally.", "sources": [ { "url": "https://github.com/alibaba/OpenSandbox", "shows": "Apache-2.0 license; general-purpose AI-agent sandbox; Docker/K8s runtimes; multi-language SDKs; self-host", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Only ~2-3 months old; no PyPI/download or named-deployment figures surfaced. ~11.3k GitHub stars (3,845 within 72h of launch) is the only signal, stars_fallback, which caps level at 3. Placed at 2: rapid early stars but no usage volume yet. Revisit as usage data emerges.", "sources": [ { "url": "https://github.com/alibaba/OpenSandbox", "shows": "11.3k GitHub stars; released Mar 2026; no usage/download figures", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation:container (Docker/Kubernetes runtimes, not VM/microVM); cold-start:container launch; language/runtime breadth:Python/TS/Go/Java SDKs, unified sandbox API, MCP server (Claude Code/Cursor); persistence:file ops; scale:K8s", "confidence": "medium", "note": "Broad multi-language SDK surface and MCP integration with K8s scale, but container-level isolation (no VM/microVM tier observed) keeps it mid-tier on the isolation-strength dimension vs Firecracker/gVisor/Kata. Strong breadth, moderate isolation.", "sources": [ { "url": "https://github.com/alibaba/OpenSandbox", "shows": "Docker/Kubernetes runtimes; multi-language SDKs (Python/TS/Go/Java); unified API; MCP server", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "Alibaba OpenSandbox, general-purpose sandbox platform for AI agents (coding agents, GUI agents, agent eval, RL training); open sourced Mar 2026; components/execd 1.0.18 (May 25 2026) confirmed live on GitHub. Go execd daemon; Docker/Kubernetes runtimes; multi-language SDKs (Python/TS/Go/Java); MCP server. Self-host only (no managed tier observed)." }, { "product": "Agent Infra Sandbox", "org": "Agent Infra", "type": "software", "description": "Apache-2.0 all-in-one Docker sandbox bundling Browser, Shell, File, MCP server, and VSCode Server in a single container so an agent gets a complete computer-use environment from one image. Picked over E2B/Daytona by builders who want browser+shell+files in one box for computer-use agents rather than wiring up multiple services. ~4.8k stars, actively maintained.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);packaging:self-hostable Docker image + K8s deploy examples;no feature-gated core observed", "confidence": "high", "note": "Apache-2.0, fully self-hostable single-container runtime; no separate proprietary/managed tier gating the core.", "sources": [ { "url": "https://github.com/agent-infra/sandbox", "shows": "Apache-2.0 license, single-Docker-container all-in-one agent sandbox, self-hostable", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "stars_fallback", "confidence": "low", "note": "No download/install or named-production-user figures disclosed; only signal is ~4.9k GitHub stars + 426 forks + 18 releases. Stars-fallback caps level; placed at 1 (hobbyist/early) on the absence of any usage_volume evidence.", "sources": [ { "url": "https://github.com/agent-infra/sandbox", "shows": "~4.9k stars, 426 forks, 18 releases; no usage figures", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation:single Docker container (process/namespace, weaker than microVM/gVisor); runtime breadth:high (Python+Node sandboxes, browser/VNC, shell, Jupyter, VSCode, MCP servers); persistence:session-persistent within container; scale:K8s high-density deploy examples but no published concurrency limit", "confidence": "medium", "note": "Unusually broad multi-modal feature set (browser+shell+file+MCP+IDE in one) but isolation is container-level only, below the Firecracker/gVisor isolation strength of the frontier deployment runtimes; scored 3.", "sources": [ { "url": "https://github.com/agent-infra/sandbox", "shows": "feature list: Browser/VNC, Shell, File, MCP, Jupyter, VSCode in one Docker container", "accessed": "2026-06-04" } ] }, "maturity": 1.8, "mature": false, "version_note": "AIO Sandbox (agent-infra/sandbox), v1.9.3 released 29 May 2026; all-in-one Docker container bundling Browser, Shell, File, MCP, Jupyter and VSCode Server for AI agents. Maintained by agent-infra org (ByteDance OSS; image hosted on volces.com/ByteDance registry). Verified live on GitHub June 2026." }, { "product": "sandbox-runtime", "org": "Anthropic", "type": "software", "description": "Open-source lightweight sandboxing tool from Anthropic that enforces filesystem and network restrictions on arbitrary processes at the OS level (Linux bubblewrap, macOS seatbelt) without requiring a container. Picked because it powers the bash tool sandbox inside Claude Code and is the reference design for safe local agent execution. ~4.1k stars and active development (63 commits in 90d).", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(GitHub);distribution:npm package;no managed/proprietary tier(it is a local library, not a service)", "confidence": "high", "note": "Apache-2.0 OS-level sandboxing library, fully open and self-contained; no feature-gated core or paid tier.", "sources": [ { "url": "https://github.com/anthropic-experimental/sandbox-runtime", "shows": "Apache-2.0 license; OS-level (bubblewrap/seatbelt) sandbox library, no container required", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "npm @anthropic-ai/sandbox-runtime ~558.8k downloads in the last 30 days (npm registry API), plus 403 GitHub dependents and 4.3k stars. Package-manager pulls overstate distinct users (CI/mirror inflation) and it is a young research preview shipped alongside Claude Code, so reach placed conservatively at 10K-100K distinct users / level 2.", "sources": [ { "url": "https://api.npmjs.org/downloads/point/last-month/@anthropic-ai/sandbox-runtime", "shows": "~558,800 downloads May 4-Jun 2 2026", "accessed": "2026-06-04" }, { "url": "https://github.com/anthropic-experimental/sandbox-runtime", "shows": "403 dependents, 4.3k stars", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation:OS-level process sandbox (Linux bubblewrap namespaces+seccomp BPF, macOS Seatbelt) - weaker boundary than gVisor/microVM, no kernel isolation; cold-start:near-zero (no container/VM spin-up); runtime breadth:any local process (language-agnostic, host-defined); persistence:n/a (host fs, deny/allow rules); scale:per-host, no managed fleet", "confidence": "medium", "note": "Best-in-class for low-overhead, no-container process isolation with fine-grained fs+network allow/deny and pre-built seccomp filters; but a process-level sandbox is a weaker security boundary than the microVM/gVisor frontier and offers no managed scale, so 3.", "sources": [ { "url": "https://github.com/anthropic-experimental/sandbox-runtime", "shows": "bubblewrap+seccomp / sandbox-exec isolation, filesystem+network restriction model, no container", "accessed": "2026-06-04" } ] }, "maturity": 2.4, "mature": false, "version_note": "anthropic-experimental/sandbox-runtime (npm @anthropic-ai/sandbox-runtime), v0.0.53 released 4 Jun 2026. Lightweight OS-level sandbox (Linux bubblewrap + macOS sandbox-exec/Seatbelt) enforcing filesystem + network restrictions on arbitrary processes WITHOUT a container; used to sandbox the Claude Code bash tool. Research preview. Verified live on GitHub June 2026." }, { "product": "Coder OSS", "org": "Coder", "type": "software", "description": "Open-source platform for provisioning remote dev environments (Terraform-defined workspaces on K8s, VMs, or bare metal), increasingly used as the substrate for self-hosted agent execution environments. Picked over GitHub Codespaces when teams want full BYOC control and have aggressively repositioned in 2026 toward AI-agent workspaces. ~13.3k stars, very active (2,055 commits in 90d).", "openness": { "score": 4, "class": "open_core", "components": "license:AGPL-3.0(OSI, Community Edition);source:public(GitHub);commercial-tier:Premium/Enterprise paid features for large teams under separate Coder enterprise license", "confidence": "high", "note": "AGPL-3.0 OSI core (Community Edition) with a separate proprietary Premium/Enterprise tier => open_core, calibrated identically to the Ray anchor (O4 open_core).", "sources": [ { "url": "https://github.com/coder/coder", "shows": "AGPL-3.0 Community Edition + Premium paid tier (compare-plans), self-hosted workspaces", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 3, "signal_type": "stars_fallback", "reach": "100K-1M", "confidence": "low", "note": "No published install/active-user count for the OSS edition; ~13.4k GitHub stars is the only concrete signal. Stars-fallback caps at level 3; placed at 3 (not higher) on broad enterprise positioning + star base, but reach is unverified by usage_volume.", "sources": [ { "url": "https://github.com/coder/coder", "shows": "~13.4k stars; self-hosted CDE used by enterprises; no usage_volume figure", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation:workspace-level (Terraform-provisioned VMs/containers/k8s pods on user infra) - depends on the backing infra template, not a hardened ephemeral microVM sandbox; cold-start:workspace provisioning (seconds-minutes), not ms; runtime breadth:any (IaC-defined, full dev environments); persistence:persistent long-lived workspaces; scale:org-scale self-hosted", "confidence": "medium", "note": "Strong as a persistent governed dev-environment platform but a poor fit for the deployment litmus 'safely run arbitrary agent-generated code in an ephemeral sandbox' - isolation/cold-start depend on the infra template, not a purpose-built sandbox; scored 3. Belongs more in a dev-environment shelf than the agent-sandbox sub-segment.", "sources": [ { "url": "https://github.com/coder/coder", "shows": "Terraform-defined persistent workspaces, Wireguard tunnel, AI-agent workspace support", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "coder/coder, v2.33.6 released 23 May 2026. Self-hosted Cloud Development Environment platform (Terraform-defined workspaces, Wireguard tunnel); now marketed as 'Secure environments for developers and their agents' with native AI-agent workspace support. Verified live on GitHub June 2026. NOTE: primarily a CDE/workspace manager rather than an ephemeral agent code-execution sandbox." }, { "product": "AWS Lambda", "org": "Amazon Web Services", "type": "software", "description": "AWS Lambda, GA serverless function service; runs on AWS Firecracker microVMs (open sourced 2018, powers Lambda + Fargate). The Lambda service itself is proprietary/SaaS - scored as the closed counterpart per the deployment recipe (Lambda/Cloud Run = closed). Verified live (AWS docs/blog) June 2026.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary AWS-only SaaS (no self-host);underlying-microVM:Firecracker is Apache-2.0 OSS but the Lambda control/data plane is closed;weights:n/a;source:closed", "confidence": "high", "note": "Lambda is a proprietary managed serverless service (closed), per recipe's explicit 'Lambda as closed counterpart'. Firecracker (the microVM substrate) is separately open source but is a different artifact; Lambda itself is not self-hostable.", "sources": [ { "url": "https://aws.amazon.com/blogs/aws/firecracker-lightweight-virtualization-for-serverless-computing/", "shows": "Firecracker microVM powers Lambda; Lambda is AWS-managed proprietary service", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "AWS states Lambda 'processes trillions of executions for hundreds of thousands of active customers every month' - mass-market serverless deployment substrate, unambiguously >10M reach. (Adoption attributed to general serverless usage, not specifically agent-code-execution.)", "sources": [ { "url": "https://aws.amazon.com/blogs/aws/firecracker-lightweight-virtualization-for-serverless-computing/", "shows": "trillions of executions / hundreds of thousands of active customers monthly", "accessed": "2026-06-04" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "isolation:Firecracker microVM (per-invocation hardware-virtualized, KVM) - strongest isolation tier, ~125ms VM start, ~5MiB/VM; cold-start:ms-to-sub-second; runtime breadth:very high (managed runtimes Node/Python/Java/Go/Ruby/.NET + custom runtimes/container images, arbitrary code); persistence:stateless+attachable storage (EFS, /tmp); scale:trillions of executions/mo, effectively unbounded autoscale", "confidence": "high", "note": "Frontier-definer for this category: microVM-grade isolation, the widest runtime breadth, and the largest proven scale of any deployment runtime here. Scored 5.", "sources": [ { "url": "https://aws.amazon.com/blogs/aws/firecracker-lightweight-virtualization-for-serverless-computing/", "shows": "Firecracker microVM isolation, ~125ms start, ~5MiB/VM, trillions of executions", "accessed": "2026-06-04" } ] }, "maturity": 5.0, "mature": false, "version_note": "AWS Lambda, GA serverless function service; runs on AWS Firecracker microVMs (open sourced 2018, powers Lambda + Fargate). The Lambda service itself is proprietary/SaaS - scored as the closed counterpart per the deployment recipe (Lambda/Cloud Run = closed). Verified live (AWS docs/blog) June 2026." }, { "product": "Modal Sandboxes", "org": "Modal Labs", "type": "software", "description": "Closed-source serverless platform that lets you run Python and shell code (including GPU workloads) in seconds-startup containers, with a dedicated Sandbox primitive for executing untrusted agent-generated code. Picked when AI teams need GPU-capable sandboxes with Pythonic ergonomics. Raised $355M Series C in May 2026 at a $4.65B valuation; $300M annualized revenue driven largely by coding-agent demand.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary Modal managed cloud (no self-host of the gVisor execution plane);open-part:client SDK only (modal Python/JS client libraries);pricing:pay-as-you-go usage, free starter credits", "confidence": "high", "note": "Closed managed service. The client SDK is open, but the execution substrate is a proprietary managed cloud with no self-host; an open client over a closed runtime is not open_core. Corrects a prior mis-calibration that grouped Modal with the genuinely self-hostable E2B (E2B ships self-hostable infra; Modal does not).", "sources": [ { "url": "https://modal.com/pricing", "shows": "Modal is a managed proprietary cloud platform, pay-per-CPU-cycle, $30/mo free credits", "accessed": "2026-06-04" }, { "url": "https://modal.com/docs/guide/sandbox", "shows": "Sandboxes = secure containers for untrusted agent code, driven via Modal SDK", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "No disclosed Modal Sandboxes user/usage count on primary pages; Modal is a well-known managed AI-compute platform with broad developer use and a documented free tier. Placed at 3 on reported multi-customer traction, not a measured usage_volume figure (kept conservative absent a hard number).", "sources": [ { "url": "https://modal.com/docs/guide/sandbox", "shows": "documented production sandbox feature on a widely-used managed platform", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:gVisor user-space kernel (Sentry intercepts syscalls) - strong container isolation, below microVM but above plain containers; cold-start:optimized container stack, fast startup; runtime breadth:arbitrary images/dependencies (custom images, any deps, repo checkout); persistence:named/ID sandbox lookup, custom images; scale:reported 50,000+ concurrent sessions; timeouts default 5min up to 24h", "confidence": "medium", "note": "Strong, purpose-built agent sandbox: gVisor isolation + arbitrary images + high concurrency. Sits just below the Firecracker-microVM frontier (Lambda/Vercel) on isolation strength; scored 4, level with the Ray/Ollama capability anchors.", "sources": [ { "url": "https://modal.com/docs/guide/sandbox", "shows": "sandbox lifecycle, custom images, timeouts up to 24h, untrusted-code execution", "accessed": "2026-06-04" }, { "url": "https://modal.com/resources/best-code-execution-sandboxes-ai-agents", "shows": "gVisor user-space kernel isolation, 50,000+ concurrent sessions (corroborating)", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Modal Sandboxes - 'secure containers for executing untrusted user or agent code' on the Modal platform; gVisor-based isolation. Open Python/JS SDK + proprietary managed cloud (pay-per-CPU-cycle, $30/mo free credits on Starter). Docs verified live June 2026." }, { "product": "Vercel Sandbox", "org": "Vercel", "type": "software", "description": "Closed-source compute primitive (GA in 2026) that runs untrusted/agent-generated code in isolated Firecracker microVMs with millisecond startup, default Node 24 + Python 3.13 runtimes, filesystem and shell access. Picked by teams already on Vercel building AI agents that need a managed sandbox tightly integrated with the AI SDK. The execution layer behind Vercel Open Agents.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary Vercel Fluid-compute managed cloud (no self-host of the Firecracker microVM plane);open-part:client SDK/CLI only (@vercel/sandbox JS + Python, vercel/sandbox on GitHub);license:proprietary service", "confidence": "high", "note": "Closed managed service. The SDK/CLI are open, but they are only a client for Vercel's proprietary microVM cloud, which cannot be self-hosted; an open client over a closed runtime is not open_core (the core is proprietary). Reclassified from open_core; sits with the other proprietary sandbox clouds (Google Cloud Run, Fly Sprites, Northflank).", "sources": [ { "url": "https://vercel.com/docs/sandbox", "shows": "Firecracker microVM isolation; SDK/CLI on GitHub vercel/sandbox; managed Vercel cloud, pay-for-active-CPU", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "npm @vercel/sandbox ~9.06M downloads in the last 30 days (npm registry API). Package-manager pulls include CI/mirror inflation and overstate distinct human users, so reach is placed at the 1M-10M band (level 4) rather than >10M; GA status + large Vercel install base corroborate genuine production use.", "sources": [ { "url": "https://api.npmjs.org/downloads/point/last-month/@vercel/sandbox", "shows": "~9,062,145 downloads May 4-Jun 2 2026", "accessed": "2026-06-04" }, { "url": "https://vercel.com/docs/sandbox", "shows": "GA ephemeral compute primitive, JS+Python SDK", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:Firecracker microVM with dedicated kernel (stronger than container/gVisor); cold-start:starts in milliseconds; runtime breadth:moderate (Amazon Linux 2023; node22/24/26 + python3.13 runtimes, sudo+root, install any package, can run Docker/VPN/FUSE); persistence:persistent-by-default (auto-save state on stop) + explicit snapshotting; scale:managed autoscale, up to 45min execution", "confidence": "high", "note": "Firecracker microVM isolation (top tier) + ms cold-start + persistence/snapshots make it a strong agent sandbox; held at 4 rather than 5 because the managed-runtime breadth is narrower (Node/Python first-class) than Lambda's any-runtime/any-image frontier and scale is not at Lambda's proven level.", "sources": [ { "url": "https://vercel.com/docs/sandbox", "shows": "Firecracker microVM per sandbox, ms startup, persistent+snapshot, node/python runtimes, up to 45m", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Vercel Sandbox - GA ephemeral compute primitive for running untrusted/AI-generated code in isolated Firecracker microVMs; docs last updated 29 May 2026. Open SDK/CLI (vercel/sandbox on GitHub, @vercel/sandbox npm + vercel.sandbox Python) over Vercel's proprietary managed cloud. Verified live June 2026." }, { "product": "Cloudflare Sandboxes", "org": "Cloudflare", "type": "software", "description": "Cloudflare's two-tier sandboxing for AI agents, shipped during Agents Week 2026: Dynamic Workers (V8 isolates, ms-scale cold start, ~100x faster and more memory-efficient than containers, open beta April 2026) and Sandboxes (persistent Linux environments, GA April 2026 for full-OS workloads). Runs on Cloudflare's global edge network and is available to paid Workers users. Execution is tied to Cloudflare's proprietary platform; only the Sandbox SDK is open source.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary Cloudflare edge (Workers/Durable Objects/Dynamic Worker Loader - no self-host of the isolate/container plane);open-part:Sandbox SDK only (@cloudflare/sandbox Apache-2.0, cloudflare/sandbox-sdk on GitHub);pricing:Workers paid plan, Dynamic Workers $0.002/unique-worker/day", "confidence": "high", "note": "Closed managed service. The Sandbox SDK is Apache-2.0, but execution requires Cloudflare's proprietary managed edge network (no self-host of the isolate/container plane); an open client SDK over a closed runtime is not open_core. Reclassified from open_core.", "sources": [ { "url": "https://github.com/cloudflare/sandbox-sdk", "shows": "Apache-2.0 SDK; runs on Cloudflare Containers/Workers/Durable Objects (managed edge)", "accessed": "2026-06-04" }, { "url": "https://blog.cloudflare.com/dynamic-workers/", "shows": "Dynamic Worker Loader open beta, isolate sandboxing, post-beta $0.002/worker/day pricing", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "medium", "note": "npm @cloudflare/sandbox ~533k downloads in the last 30 days (npm registry API); Dynamic Workers in open beta to all paid Workers users with a named customer (Zite) reporting 'millions of execution requests daily.' Package pulls + young beta status => reach placed at 100K-1M (level 3); execution-request volume is per-customer, not distinct-user adoption.", "sources": [ { "url": "https://api.npmjs.org/downloads/point/last-month/@cloudflare/sandbox", "shows": "~533,043 downloads May 4-Jun 2 2026", "accessed": "2026-06-04" }, { "url": "https://blog.cloudflare.com/dynamic-workers/", "shows": "open beta to all paid Workers users; Zite millions of execution requests/day", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation:dual - V8 isolates (Dynamic Workers, ms start, MB memory, hardened second-layer sandbox) for JS/TS + container-backed Sandbox SDK for arbitrary commands/processes; cold-start:isolates start in a few ms (~100x faster than containers); runtime breadth:JS/TS native via isolates, broader via container sandbox; persistence:Durable Objects-backed state; scale:edge-global, unlimited dynamic workers", "confidence": "medium", "note": "Distinctive isolate-based approach gives best-in-class cold-start and density, plus a container path for arbitrary code; held at 4 because isolate sandboxing is JS/TS-centric (narrower than microVM any-image), Dynamic Workers is still open beta, and overall scale evidence is per-customer rather than category-leading. Scored as a combined product per the registry name.", "sources": [ { "url": "https://blog.cloudflare.com/dynamic-workers/", "shows": "V8 isolate sandboxing, ~ms start, 100x faster/100x more memory-efficient than containers", "accessed": "2026-06-04" }, { "url": "https://github.com/cloudflare/sandbox-sdk", "shows": "container-backed sandbox: run commands, manage files, background processes, expose services", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Two related 2026 offerings: (a) Cloudflare Sandbox SDK (cloudflare/sandbox-sdk, @cloudflare/sandbox v0.11.0, 1 Jun 2026) - container-backed sandboxes on Workers/Durable Objects; (b) Dynamic Workers / Worker Loader (open beta announced 24 Mar 2026) - V8-isolate-based per-runtime sandboxing, ~100x faster than containers. Open SDK over Cloudflare's proprietary edge network. Verified live June 2026." }, { "product": "Google Cloud Run", "org": "Google", "type": "software", "description": "Fully managed serverless container platform with gVisor-based sandboxing, pay-per-request pricing, and scale-to-zero. Picked over Lambda for any-OCI-image flexibility and longer max execution. In 2026 Cloud Run added 'Cloud Run Instances' for long-running background agents and deep Gemini Enterprise Agent Platform integration, making it Google's go-to closed-source sandbox for agent code.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary(Google SaaS);source:closed;isolation-runtime:gVisor(open source, github.com/google/gvisor) but the Cloud Run product/control-plane is proprietary;managed-only(no self-host)", "confidence": "high", "note": "Proprietary managed serverless platform; only the underlying gVisor sandbox runtime is OSS, not Cloud Run itself.", "sources": [ { "url": "https://cloud.google.com/run", "shows": "Cloud Run described as fully-managed Google serverless container product (proprietary SaaS)", "accessed": "2026-06-04" }, { "url": "https://github.com/google/gvisor", "shows": "gVisor (Cloud Run's isolation runtime) is the open source component, distinct from the closed product", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Google Next '26 blog (primary, verified verbatim): external active developers and apps on Cloud Run doubled YoY, more new customers/apps in 2025 than its first 6 years combined; Replit VP quote: >1M live projects hosted on Replit with Cloud Run as primary target. Hyperscaler serverless reach is well into the >10M band.", "sources": [ { "url": "https://cloud.google.com/blog/products/serverless/whats-new-for-cloud-run-at-next26", "shows": "active developers/apps doubled YoY; more new customers in 2025 than first 6 years combined; Replit >1M live projects on Cloud Run", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation: container + gVisor syscall sandbox (now a dedicated `sandbox` agent tool for AI-generated code); cold-start: scale-to-zero with sub-second to low-second starts; language/runtime breadth: any OCI container (very broad); persistence: stateless request model + volume mounts; scale: 0 to thousands of instances autoscaling on a hyperscaler", "confidence": "medium", "note": "Very broad runtime/language coverage and hyperscaler scale; new agent `sandbox` tool (announced Next '26, 'coming soon') adds first-class AI-code-execution via a `sandbox do --` command. gVisor isolation is process/syscall-level rather than full microVM (Sprites/Northflank), so not the top isolation tier. Strong but not the single frontier-definer.", "sources": [ { "url": "https://cloud.google.com/blog/products/serverless/whats-new-for-cloud-run-at-next26", "shows": "built-in `sandbox` tool for agents to run AI-generated code in ephemeral isolated envs; container model, autoscale 0->thousands", "accessed": "2026-06-04" }, { "url": "https://cloud.google.com/run", "shows": "OCI container deploy, scale-to-zero, broad runtime support", "accessed": "2026-06-04" } ] }, "maturity": 4.6, "mature": false, "version_note": "Google Cloud Run, fully-managed serverless container platform. Confirmed live June 2026 via cloud.google.com/run. At Next '26 Google announced a built-in `sandbox` tool for agents to run AI-generated code in ephemeral, strictly-isolated environments; container isolation historically backed by gVisor. Scored as the proprietary/SaaS deployment counterpart per the recipe (Cloud Run = closed counterpart)." }, { "product": "Fly Sprites", "org": "Fly.io", "type": "software", "description": "Closed-source stateful sandbox product (launched Jan 2026), Firecracker microVMs with 100GB persistent NVMe storage that boot in 1–12s cold, ~300ms warm checkpoint/restore, and auto-idle when inactive. Picked by teams who reject ephemeral sandboxes for coding agents and want a persistent computer the agent can return to. Pitched explicitly against E2B/Modal/Daytona for long-running Claude Code-style sessions.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary(Fly.io managed);source:closed;clients:CLI/REST/JS/Go SDKs (client libs only, no open runtime);isolation:Firecracker(Firecracker itself is OSS/AWS, but the Sprites product/orchestration is proprietary)", "confidence": "high", "note": "Proprietary managed sandbox SaaS; no open source runtime published (only client SDKs). Underlying Firecracker is OSS but the product is not.", "sources": [ { "url": "https://sprites.dev/", "shows": "Sprites = hardware-isolated Firecracker execution env for arbitrary code; managed service, CLI/REST/JS/Go clients; no self-hostable open runtime", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 1, "reach": "<10K", "signal_type": "reported_traction", "confidence": "low", "note": "Launched Jan 2026 (~5 months at scoring time) as Fly.io's answer to E2B. No disclosed user/customer/usage counts on the product page or launch coverage; positioned for AI coding-agent sessions (Claude Code/Codex). Early-stage; placed at 1 on reported traction with low confidence pending real numbers.", "sources": [ { "url": "https://sprites.dev/", "shows": "product live, usage-based pricing ($0.07/CPU-hr); no user/customer figures disclosed", "accessed": "2026-06-04" }, { "url": "https://simonwillison.net/2026/Jan/9/sprites-dev/", "shows": "Sprites launched early Jan 2026 as Fly.io's E2B competitor (corroborating recency, not a usage figure)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation: Firecracker microVM (hardware-isolated, strongest tier); cold-start: 1-12s cold, warm restore <1s, checkpoint/restore <1s; persistence: persistent ext4 / 100GB NVMe surviving across sessions (a key differentiator); language/runtime breadth: arbitrary Linux ('any code blob'); scale: Fly.io global infra", "confidence": "medium", "note": "Top-tier microVM isolation plus persistent stateful sandboxes (explicitly anti-ephemeral) is a strong, differentiated feature set. Not a 5 only because it is brand-new with unproven scale vs incumbents; isolation + persistence are best-in-class.", "sources": [ { "url": "https://sprites.dev/", "shows": "Firecracker isolation, persistent ext4 filesystem, sub-1s restore, arbitrary Linux code", "accessed": "2026-06-04" } ] }, "maturity": 2.2, "mature": false, "version_note": "Sprites (sprites.dev) by Fly.io, launched Jan 2026. Persistent, hardware-isolated Firecracker microVMs ('a persistent Linux computer') for running arbitrary code / AI agents, with 100GB NVMe persistence and sub-1s checkpoint/restore. Confirmed live June 2026 via sprites.dev. Proprietary managed service (Fly.io's E2B competitor)." }, { "product": "Replit Agent / Code Execution API", "org": "Replit", "type": "software", "description": "Closed-source cloud dev platform with Nix-based environments supporting 30,000+ OS packages, exposing a Code Execution API that runs untrusted/AI-generated code in sandboxed VMs. Picked when teams want a tightly integrated build+run+deploy story (Replit Agent 4 leans heavily on this) rather than an unbundled SDK. Raised $400M Series D in March 2026 at a $9B valuation; 50M+ users.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary(Replit managed Autoscale Deployments, no self-host);open-part:third-party isolation primitive omegajail(github.com/omegaup/omegajail) + an ARCHIVED thin client (replit-code-exec, ISC, read-only since 2025-02-11);no self-hostable execution service", "confidence": "medium", "note": "Closed managed service. The sandbox primitive (omegajail) is open source but it is a third-party dependency, not Replit's own open core, and the thin client is archived; there is no self-hostable open product, so an open dependency under a closed service is not open_core. Reclassified from open_core (borderline).", "sources": [ { "url": "https://replit.com/blog/ai-agents-code-execution", "shows": "Code Execution API uses omegajail (open source unprivileged container sandbox) on Replit's proprietary Autoscale Deployments; replit-code-exec PyPI client; Python-focused", "accessed": "2026-06-04" }, { "url": "https://github.com/replit/replit-code-exec", "shows": "Replit code-exec client: stateless API for AI agents, runs untrusted Python in ephemeral unprivileged containers using omegajail (github.com/omegaup/omegajail); ISC license; repo ARCHIVED read-only 2025-02-11", "accessed": "2026-06-04" }, { "url": "https://blog.replit.com/ai-agents-code-execution", "shows": "canonical Replit blog: Code Execution API uses omegajail on Autoscale Deployments, ~100ms, Python-focused (Feb 2024)", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Replit platform serves >50M users (Replit docs/coverage) and runs >1M live projects on Cloud Run (Google Next '26, primary). The code-execution API specifically discloses no standalone usage figures, so adoption is attributed to the Replit Agent/platform surface it powers, not the raw API. Placed at 4 (platform-surface reach); the API alone would be lower.", "sources": [ { "url": "https://cloud.google.com/blog/products/serverless/whats-new-for-cloud-run-at-next26", "shows": "Replit runs >1M live projects (on Cloud Run), primary-source scale signal for the Replit surface", "accessed": "2026-06-04" }, { "url": "https://replit.com/products/agent", "shows": "Replit Agent product page (the agent surface the code-exec API backs)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation: omegajail unprivileged container (process/container-level, weaker than microVM); cold-start: ~100ms (very fast); language/runtime breadth: primarily Python, customizable Docker image; persistence: stateless (prior stateful-Repl experiment discontinued); scale: Replit Autoscale Deployments", "confidence": "medium", "note": "Fast container-level execution tuned for agent code-eval, but container (not microVM) isolation, narrow language focus, and no persistence put it mid-tier vs microVM-backed peers. Note: the public self-serve client (replit-code-exec) is archived (2025-02-11); the capability now lives inside the Replit Agent surface.", "sources": [ { "url": "https://replit.com/blog/ai-agents-code-execution", "shows": "omegajail container sandbox, ~100ms start, Python-focused, stateless (stateful-Repl experiment discontinued)", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "Replit Code Execution API for AI agents (replit.com/blog/ai-agents-code-execution), a stateless sandboxed API for running untrusted (primarily Python) code, isolated via omegajail (Replit's unprivileged container sandbox) on Autoscale Deployments, ~100ms start. Confirmed live June 2026; `replit-code-exec` PyPI client exists. Proprietary service. NOTE: 'Replit Agent' is an autonomous coding agent (orchestration), but the deployment-relevant artifact here is the code-execution sandbox, scored as deployment per the recipe litmus." }, { "product": "WebContainers", "org": "StackBlitz", "type": "software", "description": "WebAssembly Node.js runtime that boots a full dev environment (filesystem, package manager, terminal, dev server) inside the browser with no remote VM. Picked when an agent and user need to share an in-browser sandbox with millisecond startup and near-zero server cost. `webcontainer-core` is MIT (~4.6K stars) but stale (last push April 2025); the production WebContainer API and StackBlitz/Bolt.new platform are proprietary. Powers Bolt.new, which hit 7M+ users and $40M ARR by Dec 2025; StackBlitz raised Series B in Jan 2025 at $700M valuation.", "openness": { "score": 2, "class": "source_available", "components": "public-repo:webcontainer-core is MIT but issue-tracker only (no runtime source);actual-runtime:proprietary(StackBlitz);license-terms:free for OSS/prototypes, commercial license required for for-profit production (charge beyond 10k API req/mo);distribution:@webcontainer/api npm (proprietary binary)", "confidence": "high", "note": "The MIT 'open' repo holds no implementation; the runtime is closed and gated by a commercial license for production use. Best classed source_available leaning closed; flag open_washed for the MIT-issue-tracker-as-open source framing.", "sources": [ { "url": "https://github.com/stackblitz/webcontainer-core", "shows": "MIT repo is a central hub for issues/bug reports, not the runtime implementation", "accessed": "2026-06-04" }, { "url": "https://www.npmjs.com/package/@webcontainer/api", "shows": "WebContainer API distributed as npm package; commercial license required for for-profit production use (free for OSS/POC, charge beyond 10k req/mo)", "accessed": "2026-06-04" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "low", "note": "Powers StackBlitz's own IDE and is embedded by named projects (SvelteKit, Astro, Scrimba, egghead.io, Xata) for in-browser code experiences. No disclosed download/MAU figure on the primary pages; placed at 3 on named-integration reported traction. No hard usage_volume number found, so confidence low.", "sources": [ { "url": "https://webcontainers.io/", "shows": "WebContainers used by SvelteKit, Astro, Scrimba, egghead.io, Xata (named integrations); no quantitative usage figure", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation: browser/WASM sandbox (client-side, inherently isolated from host but not a server execution env); cold-start: instant (no server provisioning); language/runtime breadth: Node.js + npm ecosystem, JS/TS focused, plus arbitrary WASM (narrow vs general Linux sandboxes); persistence: per-session in-browser; scale: runs on the end-user's device (no server cost, no server scale)", "confidence": "medium", "note": "Unique in-browser execution with instant start and zero server cost, but Node/JS-only and not a general-purpose server-side code sandbox for arbitrary languages, a different (narrower) capability profile than microVM/container peers. Mid-tier within deployment.", "sources": [ { "url": "https://webcontainers.io/", "shows": "runs Node.js/npm and WASM fully in-browser, 'up to 10x faster than local', any modern framework", "accessed": "2026-06-04" } ] }, "maturity": 3.0, "mature": false, "version_note": "WebContainers by StackBlitz, a browser-based (WASM) runtime that executes Node.js / npm projects fully client-side in the browser tab. Distributed as @webcontainer/api on npm. Confirmed live June 2026 via webcontainers.io. The public github.com/stackblitz/webcontainer-core repo is MIT but is only an issue tracker / docs hub; the actual runtime is a proprietary StackBlitz service requiring a commercial license for for-profit production use." }, { "product": "OpenAI Code Interpreter", "org": "OpenAI", "type": "software", "description": "Closed-source code-execution tool exposed via the Responses API and Assistants API, runs Python (and bash in newer versions) in a fully sandboxed VM with attached files, so the model can compute, analyze data, and iterate on solutions. Picked when teams want a turnkey 'agent can run code' primitive bundled directly with the OpenAI model API rather than wiring up E2B/Modal themselves.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary(OpenAI-managed sandbox);source:closed;isolation:undisclosed;access:API/tool only(no self-host, no open runtime)", "confidence": "high", "note": "Fully proprietary managed code-execution sandbox; OpenAI does not document the isolation mechanism or release any runtime.", "sources": [ { "url": "https://developers.openai.com/api/docs/assistants/tools/code-interpreter", "shows": "Code Interpreter runs Python in a proprietary OpenAI-managed sandbox; $0.03/session; isolation details not disclosed", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "active_users", "confidence": "low", "note": "Code Interpreter ships inside ChatGPT (advanced data analysis) and the Assistants/Responses API; the ChatGPT surface it rides has ~1B weekly users (flagship anchor record, GPT-5). No standalone Code-Interpreter usage figure is disclosed, so adoption is attributed to the surface, placed conservatively at 4 (a fraction of ChatGPT users invoke code execution) with low confidence given no primary per-tool number.", "sources": [ { "url": "https://developers.openai.com/api/docs/assistants/tools/code-interpreter", "shows": "Code Interpreter is a first-class tool in the Assistants/Responses API and underlies ChatGPT data analysis (the surface that drives usage)", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation: undisclosed managed sandbox; cold-start: per-session (1hr persistence within session); language/runtime breadth: Python only; persistence: per-session file workspace, 512MB file handling; scale: OpenAI infra", "confidence": "low", "note": "Reliable Python-only data-analysis sandbox with iterative retry, but single-language, undisclosed isolation, and session-scoped, narrower than general-purpose deployment sandboxes. Mid-tier; capability detail limited because OpenAI does not document the isolation/scale internals.", "sources": [ { "url": "https://developers.openai.com/api/docs/assistants/tools/code-interpreter", "shows": "Python sandbox, iterative code retry, 512MB files, 1-hour session", "accessed": "2026-06-04" } ] }, "maturity": 3.6, "mature": false, "version_note": "OpenAI Code Interpreter, a tool (Assistants API, migrating to Responses API) that writes and runs Python in a proprietary OpenAI-managed sandbox; processes files up to 512MB, sessions billed at $0.03/session (1-hour default). Confirmed live June 2026 via OpenAI developer docs. Proprietary; isolation mechanism undisclosed." }, { "product": "Northflank Sandboxes", "org": "Northflank", "type": "software", "description": "Closed-source managed sandbox platform offering microVM isolation via Kata Containers and gVisor, any OCI image, unlimited session duration, and BYOC deployment. Picked when teams need production-grade isolation plus a complete platform (services, jobs, databases) beyond ephemeral sandboxes; processes 2M+ isolated workloads/month and is SOC 2 Type 2 compliant.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary(Northflank managed SaaS);source:closed;isolation:microVM;no self-hostable open runtime", "confidence": "high", "note": "Proprietary managed deployment/sandbox platform; no open source runtime published.", "sources": [ { "url": "https://northflank.com/", "shows": "Northflank Sandboxes = secure microVM sandbox for untrusted/code-gen execution; proprietary commercial SaaS", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "reported_traction", "confidence": "medium", "note": "Northflank (platform-wide, primary): 2,000+ organizations, 80,000+ developers in production, 130B+ requests, millions of containers, 330+ availability zones. These are platform-level figures (not Sandboxes-specific); the Sandboxes product is newer. Placed at 3 on the developer/org base; the Sandboxes feature alone is likely lower.", "sources": [ { "url": "https://northflank.com/", "shows": "2,000+ orgs, 80,000+ developers in production, 130B+ requests, millions of containers (platform-level)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "isolation: microVM (hardware-isolated, top tier) plus secure containers; cold-start: ~200ms microVM spin-up; language/runtime breadth: arbitrary containers/runtimes (broad); persistence: stateful workloads / long-running agents supported; scale: multi-cloud, 330+ AZs, millions of containers", "confidence": "medium", "note": "Strong: microVM isolation, ~200ms cold-start, broad runtime support, multi-cloud scale, and explicit support for untrusted code-gen at scale. Just below frontier on proven sandbox-specific scale vs the dedicated leaders; isolation+latency are top-tier.", "sources": [ { "url": "https://northflank.com/", "shows": "microVM 200ms isolation, untrusted code at scale, millions of containers, 330+ AZs", "accessed": "2026-06-04" } ] }, "maturity": 3.4, "mature": false, "version_note": "Northflank Sandboxes, secure microVM/sandbox product for running untrusted / AI-generated code at scale (~200ms microVM spin-up), positioned for code-gen agents. Confirmed live June 2026 via northflank.com. Proprietary SaaS (Northflank Ltd)." }, { "product": "Spaces", "org": "Hugging Face", "type": "software", "description": "Managed sandbox platform for hosting interactive ML apps, supports Gradio, Streamlit, Docker, and static SDKs with a free CPU tier, paid CPU/GPU upgrades, and a ZeroGPU shared-pool tier. Differentiates through tight integration with the Hugging Face Hub (one-click deploy from any model card) and community discoverability, making it the default demo surface for nearly every open-weight model release. ~400K+ apps as of early 2026 with reportedly ~40K new Spaces created per month and 100M+ monthly visits; the canonical 'where you go to try an open model in your browser'.", "openness": { "score": 1, "class": "closed", "components": "platform:proprietary(HF-managed hosting/orchestration, no self-hostable Spaces);open-part:app SDKs only (Gradio Apache-2.0, Streamlit) which are separate, portable OSS products;hardware-tiers:CPU free + paid GPU upgrades", "confidence": "medium", "note": "Closed managed hosting service. Apps are built with OSS SDKs (Gradio etc.) that run anywhere, but Spaces itself - the managed host/orchestration - is proprietary and not self-hostable; those SDKs are separate products, not a Spaces open core. Reclassified from open_core (borderline).", "sources": [ { "url": "https://huggingface.co/docs/hub/spaces", "shows": "Spaces hosts apps via Gradio (OSS SDK), Streamlit, arbitrary Dockerfile, or static HTML on HF-managed infra with paid GPU upgrades", "accessed": "2026-06-04" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Hugging Face homepage (primary): Spaces hosts 1M+ AI applications, on a platform used by 50,000+ organizations. 1M+ hosted apps plus the broad end-user traffic to those demos puts served reach into the >10M band. Headline (1M+ Spaces) cites HF's own homepage; per-Space user totals not individually disclosed so confidence medium.", "sources": [ { "url": "https://huggingface.co", "shows": "Spaces = 1M+ AI applications hosted; 50,000+ organizations on Hugging Face", "accessed": "2026-06-04" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "isolation: managed container per Space (Docker), not microVM-grade for untrusted multi-tenant code; cold-start: warm/sleep model (free tier sleeps), seconds; language/runtime breadth: Gradio/Streamlit/Docker/static (broad for app hosting); persistence: persistent storage tiers + dev mode; scale: CPU free tier + paid GPU upgrades on HF infra", "confidence": "medium", "note": "Excellent for hosting/demoing ML apps with broad SDK and GPU support, but it is an app-hosting platform rather than a hardened arbitrary-untrusted-code sandbox (container, not microVM isolation), and free Spaces sleep. Mid-tier within the deployment sandbox lens.", "sources": [ { "url": "https://huggingface.co/docs/hub/spaces", "shows": "Gradio/Streamlit/Docker/static hosting, GPU hardware upgrades, persistent storage and dev mode", "accessed": "2026-06-04" } ] }, "maturity": 4.2, "mature": false, "version_note": "Hugging Face Spaces, managed hosting/runtime for ML demo apps via Gradio, Streamlit, Docker (arbitrary Dockerfile), or static HTML, with CPU/GPU hardware tiers. Confirmed live June 2026 via huggingface.co and HF Hub docs. Proprietary managed platform built on OSS SDKs (Gradio is OSS); the Spaces hosting platform itself is closed." }, { "product": "Ollama", "org": "Ollama", "type": "software", "description": "Local LLM runtime that wraps llama.cpp in a user-friendly CLI and REST API with a Docker-like model management experience (ollama pull, ollama run). Handles model downloading, quantization selection, and GPU/CPU routing automatically. 172K GitHub stars, the most popular way for developers to run models locally. Became the default local inference backend for dozens of AI applications and MCP integrations.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT;source:public;no-feature-gated-core", "confidence": "high", "note": "license:MIT;source:public;no-feature-gated-core", "sources": [ { "url": "https://github.com/ollama/ollama/blob/main/LICENSE", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://github.com/ollama/ollama", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://ollama.com/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "Reported 52M+ monthly model pulls in early 2026; 170K+ GitHub stars. Pull/usage volume places it in the 1-10M+ active-user range; rated 4 conservatively on monthly pull volume (not stars).", "sources": [ { "url": "https://github.com/ollama/ollama", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.ollama.com/", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "Local model serving with one-command run, OpenAI-compatible REST API, broad model library (Llama, Qwen, Gemma, gpt-oss, DeepSeek). Serving-layer tool; no MLPerf result cited, rated on feature/coverage matrix.", "sources": [ { "url": "https://github.com/ollama/ollama", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://ollama.com/library", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.0, "mature": false, "version_note": "Active 2026; ~170K+ GitHub stars, library of thousands of model builds (GGUF)" }, { "product": "Ray", "org": "Anyscale", "type": "software", "description": "Ray-2.55.1, released April 22, 2026 (GitHub). Ray joined the PyTorch Foundation (per Anyscale 2026 announcement). Confirmed to exist as of June 2026.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI);source:public(all Ray libraries OSS);commercial-tier:Anyscale managed SaaS on top", "confidence": "high", "note": "license:Apache-2.0(OSI);source:public(all Ray libraries OSS);commercial-tier:Anyscale managed SaaS on top", "sources": [ { "url": "https://github.com/ray-project/ray", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.anyscale.com/product/open-source/ray", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://www.anyscale.com/blog/ray-by-anyscale-joins-pytorch-foundation", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ], "bucket": "open" }, "adoption": { "level": 5, "reach": ">10M", "signal_type": "usage_volume", "confidence": "high", "note": "Re-scored 4->5. The prior record had no verified download count (reported_traction). Direct registry data now shows ray at 61.31M PyPI downloads in the last 30 days, comfortably above the map's prevailing usage_volume level-5 floor of >10M/mo (and above even the stricter 50M floor the HF-stack files use, cf. pytorch ~88.6M, triton ~64.7M at level 5). Ray is the standard open distributed-compute / model-serving substrate (Ray Serve, Tune, Train, Data) under the PyTorch Foundation.", "sources": [ { "url": "https://pepy.tech/projects/ray", "shows": "61,306,813 downloads in last 30 days", "accessed": "2026-06-25" }, { "url": "https://www.anyscale.com/blog/ray-by-anyscale-joins-pytorch-foundation", "shows": "Ray under the PyTorch Foundation (governance / ecosystem standing)", "accessed": "2026-06-04" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": null, "confidence": "high", "note": "General distributed-compute + model-serving framework (Ray Core, Serve, Tune, Train, Data). No standardized inference/training benchmark (MLPerf) applies to Ray as a whole; scored on breadth of distributed-serving feature coverage. Not directly comparable to single-engine inference benchmarks.", "sources": [ { "url": "https://www.anyscale.com/product/open-source/ray", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" }, { "url": "https://docs.anyscale.com/get-started/what-is-ray", "shows": "flagship phase-C verification source", "accessed": "2026-06-04" } ] }, "maturity": 4.6, "mature": true, "version_note": "Ray-2.55.1, released April 22, 2026 (GitHub). Ray joined the PyTorch Foundation (per Anyscale 2026 announcement). Confirmed to exist as of June 2026." }, { "product": "OpenPCC", "org": "Confident Security", "type": "software", "description": "Open-source framework and standard for verifiably-private AI inference, explicitly modeled on Apple's Private Cloud Compute but fully open, vendor-neutral, auditable, and self-deployable. It wraps every inference interaction (prompt, tokens, logs, telemetry) in encryption and policy so no vendor, operator, or insider can read it, combining confidential compute on commodity CPU/GPU TEEs, integrated remote attestation, tamper-proof transparency logs, an HPKE-based encrypted client-server stream, and an Oblivious-HTTP anonymization layer to defeat metadata/timing side-channels. Model- and provider-agnostic; ships Go and C client SDKs.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(github.com/openpcc/openpcc + spec/whitepaper);reference-impl(Go)+client-SDKs;commercial-managed-service(CONFSEC)separate", "confidence": "high", "note": "Fully OSI-licensed (Apache-2.0) across the reference implementation and related repos, with the spec and whitepaper public. Confident Security runs a commercial managed service (CONFSEC) on top, but that is an open-core *business* around a fully open standard, not open-core licensing.", "sources": [ { "url": "https://github.com/openpcc/openpcc", "shows": "Apache-2.0; \"open source framework for verifiably private AI inference\"; OHTTP gateway, attestation, transparency log, client SDKs", "accessed": "2026-06-17" }, { "url": "https://cloudsecurityalliance.org/blog/2025/11/13/introducing-openpcc", "shows": "defines OpenPCC as an Apache-2.0 standard for provably-confidential cloud compute; launch Nov 2025; Confident Security", "accessed": "2026-06-17" } ], "bucket": "open" }, "adoption": { "level": 2, "signal_type": "reported_traction", "confidence": "low", "note": "Launched Nov 5 2025 with a $5M seed (Decibel, Ex/Ante, South Park Commons, Halcyon, SAIF); ~940 GitHub stars by mid-2026 (notable for a young infra project) but no production adopters documented in primary sources. Early-stage.", "sources": [ { "url": "https://github.com/openpcc/openpcc", "shows": "~940 stars; reference implementation", "accessed": "2026-06-17" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "confidential inference on commodity CPU + GPU (NVIDIA) TEEs; integrated CPU/GPU remote attestation; HPKE-based encrypted client-server streaming (Two-Way); tamper-proof verifiable transparency logs; unlinkable/anonymized requests via Oblivious HTTP; client SDKs (Go native + C lib for Python/JS); model/provider-agnostic (prototyped on Intel TDX + H100 serving Llama-3 8B on vLLM)", "confidence": "medium", "note": "Broad, well-architected confidential-serving stack with attestation plus anonymization that goes beyond plain confidential compute. Not top because it is young with limited proven scale and no documented production adopters.", "sources": [ { "url": "https://cloudsecurityalliance.org/blog/2025/11/13/introducing-openpcc", "shows": "technical scope: TEE inference, attestation, transparency logs, Oblivious-HTTP anonymization, client SDKs", "accessed": "2026-06-17" } ] }, "maturity": 2.8, "mature": false, "version_note": "Apache-2.0; maintained by Confident Security (San Francisco; $5M seed). Launched Nov 5 2025; ~940 stars by mid-2026. Reference implementation in Go plus a written spec/whitepaper; related repos (twoway, ohttp, bhttp) are also Apache-2.0. Confident Security operates a commercial managed service (CONFSEC) on top of the open standard, but the standard and implementation themselves are fully open." }, { "product": "nono", "org": "nolabs", "type": "software", "description": "An open-source, zero-setup sandbox that runs AI agents (Claude Code, Copilot, and others) with least-privilege isolation on macOS, Linux, and Windows. Written in Rust by nolabs-ai (built by members of the Sigstore team), it wraps agent processes in a least-privilege sandbox with near-zero startup latency, ships FFI bindings for Rust, Python, TypeScript, and Go, and hosts an agent registry. Fully open source (Apache-2.0) and self-hostable as a standalone CLI or library, with no proprietary backend, distributed via curl, Homebrew, and Linux package managers.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(full Rust implementation, self-hostable);distribution:curl / Homebrew / Debian-Ubuntu-Fedora-Arch-RHEL-openSUSE-Nix + FFI bindings (nono-py, nono-ts, nono-go);service:none(standalone CLI/library, no proprietary backend)", "confidence": "high", "note": "Fully open source: an Apache-2.0 Rust implementation that is self-hostable as a standalone CLI or library with no proprietary service tier. A clean contrast with the managed sandbox clouds (Vercel, Cloudflare, Modal), which publish only a client SDK over a closed runtime.", "sources": [ { "url": "https://github.com/nolabs-ai/nono", "shows": "Apache-2.0 license; full Rust source; self-hostable; install via curl / Homebrew / Linux package managers; FFI bindings nono-py / nono-ts / nono-go", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "1K-10K", "signal_type": "stars_fallback", "confidence": "low", "note": "~2,900 GitHub stars (June 2026) for a 2026 release, with multi-platform packages and FFI bindings published. Stars are the only public proxy here, so this is capped per the methodology and read as early but real traction rather than scaled adoption.", "sources": [ { "url": "https://github.com/nolabs-ai/nono", "shows": "~2,900 stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 3, "basis": "feature coverage (cross-platform least-privilege agent sandboxing)", "value": "Least-privilege agent sandboxing across macOS / Linux / Windows with near-zero startup, FFI bindings for Rust / Python / TypeScript / Go, and an agent registry. Lighter-weight than the microVM-based sandboxes (E2B, Firecracker) but with the widest local-platform coverage and lowest setup friction.", "confidence": "medium", "note": "Solid, cross-platform sandbox with very low setup friction; scored mid-range for the category -- below the heavier microVM / OS-isolation options on hard isolation depth, but broad and easy to adopt locally.", "sources": [ { "url": "https://nono.sh", "shows": "zero-setup, cross-platform least-privilege agent sandboxing; agent registry", "accessed": "2026-06-29" } ] }, "maturity": 2.4, "mature": false, "version_note": "nono (nono.sh) — least-privilege agent sandbox in Rust by nolabs-ai (built by members of the Sigstore team). Apache-2.0, fully self-hostable, no proprietary service. ~2,900 GitHub stars (June 2026). Installable via curl, Homebrew, and Linux package managers (Debian/Ubuntu, Fedora, Arch, RHEL, openSUSE, Nix, WSL2); FFI bindings nono-py / nono-ts / nono-go; agent registry at registry.nono.sh. Verified live June 2026." } ] }, "edge_hardware": { "label": "Edge AI hardware", "arc": "Infrastructure", "layer": "infrastructure", "stage": { "num": 3, "name": "Viable Alternatives" }, "gaps": [ "maturity", "openness" ], "products": [ { "product": "Raspberry Pi 5", "org": "Raspberry Pi", "type": "hardware", "description": "A credit-card-sized single-board computer (SBC) built on the Broadcom BCM2712 SoC with a 2.4GHz quad-core Arm Cortex-A76 CPU and VideoCore VII GPU, offered with 1/2/4/8/16GB LPDDR4X RAM. It has no dedicated NPU, so on-device AI runs on the CPU/GPU or via add-on accelerators (e.g. the AI HAT+). Made by Raspberry Pi, it is the most ubiquitous hobbyist/edge SBC and the de facto reference platform for community edge-AI projects.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:reduced published;toolchain:open kernel tree + Raspberry Pi OS;datasheets:public;blobs:minimal;retail:mass-market", "confidence": "high", "note": "Open Linux kernel tree and published datasheets/reduced schematics, globally purchasable, though full board design files and some firmware remain closed.", "sources": [ { "url": "https://www.raspberrypi.com/products/raspberry-pi-5/", "shows": "Official product page: BCM2712, Cortex-A76, RAM options, production until 2036", "accessed": "2026-06-22" }, { "url": "https://github.com/raspberrypi/linux", "shows": "Official Raspberry Pi kernel source tree (open BSP)", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "Flagship Raspberry Pi SBC sold globally through countless distributors with a huge community; in production until at least January 2036.", "sources": [ { "url": "https://www.raspberrypi.com/products/raspberry-pi-5/", "shows": "Official availability commitment and distribution", "accessed": "2026-06-22" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "No NPU; up to 16GB RAM; Cortex-A76 quad-core + VideoCore VII; runs small quantized LLMs slowly on CPU", "confidence": "high", "note": "General-purpose SBC with no dedicated AI accelerator; on-device AI is modest unless paired with an accelerator HAT.", "sources": [ { "url": "https://www.raspberrypi.com/products/raspberry-pi-5/", "shows": "CPU/GPU/RAM spec with no NPU/TOPS figure listed", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Runs an open Linux kernel tree (github.com/raspberrypi/linux) and ships published datasheets/reduced schematics, globally purchasable, though full board design files and some firmware remain closed. No representative GitHub artifact is declared; the product is hardware, and the kernel fork is a general-purpose OS mirror whose commit and contributor signal is unrelated to the board as an AI platform (tracked in issue 54)." }, { "product": "Raspberry Pi AI HAT+", "org": "Raspberry Pi", "type": "hardware", "description": "An AI accelerator HAT+ add-on board for the Raspberry Pi 5 that integrates a Hailo neural-network accelerator: the 13 TOPS Hailo-8L or the 26 TOPS Hailo-8 variant. It conforms to the Raspberry Pi HAT+ specification, connects over PCIe, and integrates with the Pi camera stack for tasks like object detection and segmentation. Made by Raspberry Pi (silicon by Hailo); it is the official first-party way to add meaningful NPU compute to a Pi 5.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:HAT+ mechanical spec public;toolchain:open Pi driver integration, Hailo compiler gated;datasheets:public brief;blobs:Hailo firmware required;retail:mass-market", "confidence": "medium", "note": "Officially supported open Pi software stack and public product page, globally purchasable, but the underlying Hailo model-compiler toolchain is registration-gated.", "sources": [ { "url": "https://www.raspberrypi.com/products/ai-hat/", "shows": "Official product page: Hailo-8 (26 TOPS) / Hailo-8L (13 TOPS), HAT+ form factor, from $70", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "First-party Raspberry Pi accessory sold through the global Pi distributor network with strong community/tutorial coverage.", "sources": [ { "url": "https://www.raspberrypi.com/products/ai-hat/", "shows": "Official availability and pricing from $70", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "13 TOPS (Hailo-8L) or 26 TOPS (Hailo-8); vision inference; no onboard RAM (uses host Pi)", "confidence": "high", "note": "Mid-tier vision-focused NPU; strong for CNN inference but not large LLMs, and dependent on the host Pi for memory.", "sources": [ { "url": "https://www.raspberrypi.com/products/ai-hat/", "shows": "26/13 TOPS Hailo accelerator spec and supported tasks", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Officially supported open Pi software stack and public product page, globally purchasable, but the underlying Hailo model-compiler toolchain is registration-gated." }, { "product": "NVIDIA Jetson Orin Nano Super Developer Kit", "org": "NVIDIA", "type": "hardware", "description": "An entry-level edge-AI developer kit pairing the Jetson Orin Nano 8GB module with a carrier board; the 'Super' mode boosts it to 67 INT8 (sparse) TOPS. It uses an Ampere GPU with 1024 CUDA cores and 32 Tensor Cores, a 6-core Arm Cortex-A78AE CPU, and 8GB LPDDR5. Made by NVIDIA, it was repriced to $249 and is the cheapest on-ramp into the CUDA/Jetson generative-AI ecosystem, capable of running small LLMs and VLMs locally.", "openness": { "class": "documented", "score": 3, "components": "schematics:carrier design guide public;toolchain:JetPack/Jetson Linux free but NVIDIA-SLA proprietary;datasheets:public;blobs:proprietary GPU drivers/bootloader required;retail:mass-market", "confidence": "high", "note": "Public datasheets and globally purchasable, but the SDK requires proprietary NVIDIA drivers/blobs under license, so it sits in the documented tier despite the community OE4T BSP.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/nano-super-developer-kit/", "shows": "Official spec: 67 INT8 TOPS, 1024-core Ampere GPU, 6-core A78AE, 8GB LPDDR5, $249", "accessed": "2026-06-22" }, { "url": "https://developer.nvidia.com/embedded/jetson-linux", "shows": "Jetson Linux (L4T) BSP under NVIDIA Software License Agreement", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "Widely available developer kit at a low $249 price with a large NVIDIA Jetson developer community and global distribution.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/nano-super-developer-kit/", "shows": "Official $249 pricing and developer-kit positioning", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "67 INT8 (sparse) TOPS; 8GB LPDDR5; runs small LLMs (~7B quantized) and VLMs locally", "confidence": "high", "note": "Solid entry-tier generative-AI compute, but 8GB RAM caps model size versus higher Orin tiers.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/nano-super-developer-kit/", "shows": "67 INT8 TOPS and 8GB LPDDR5 spec", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheets and globally purchasable, but the SDK requires proprietary NVIDIA drivers/blobs under license, so it sits in the documented tier despite the community OE4T BSP." }, { "product": "NVIDIA Jetson AGX Orin", "org": "NVIDIA", "type": "hardware", "description": "NVIDIA's high-end edge-AI system-on-module (SoM), delivering up to 275 TOPS via a 2048-core Ampere GPU with 64 Tensor Cores and a 12-core Arm Cortex-A78AE CPU. It ships in 32GB and 64GB LPDDR5 variants on a 699-pin module. Made by NVIDIA, it is the flagship Jetson module for robotics and autonomous machines, able to run large LLMs/VLMs at the edge.", "openness": { "class": "documented", "score": 3, "components": "schematics:module datasheet + carrier design guide public;toolchain:JetPack/Jetson Linux proprietary (NVIDIA SLA);datasheets:public;blobs:proprietary drivers/bootloader;retail:via distributors", "confidence": "high", "note": "Public datasheets and commercially purchasable, but SDK relies on proprietary NVIDIA blobs under license, placing it in the documented tier.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "Official spec: up to 275 TOPS, 2048-core Ampere GPU, 32/64GB LPDDR5, 699-pin module", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Available as a dev kit and production module through NVIDIA's distributor network with a large Jetson community, but the high price targets professional robotics.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "Module family page indicating production module and dev kit availability", "accessed": "2026-06-22" } ] }, "capability": { "score": 5, "basis": "feature_matrix", "value": "Up to 275 TOPS; 32GB or 64GB LPDDR5; runs large LLMs/VLMs and multi-model robotics pipelines at the edge", "confidence": "high", "note": "Top-tier edge-AI compute; the 64GB/275 TOPS variant runs large models that exceed smaller Orin modules.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "275 TOPS and up to 64GB LPDDR5 spec", "accessed": "2026-06-22" } ] }, "maturity": 4.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheets and commercially purchasable, but SDK relies on proprietary NVIDIA blobs under license, placing it in the documented tier." }, { "product": "NVIDIA Jetson Orin NX", "org": "NVIDIA", "type": "hardware", "description": "A compact mid-range edge-AI system-on-module (SoM) in a 260-pin SO-DIMM form factor, delivering up to 157 TOPS via a 1792-core Ampere GPU with 56 Tensor Cores and an 8-core Arm Cortex-A78AE CPU. It comes in 8GB and 16GB LPDDR5 variants. Made by NVIDIA, it slots between the Orin Nano and AGX Orin and is widely used in drones, cameras, and compact robotics for local LLM/vision workloads.", "openness": { "class": "documented", "score": 3, "components": "schematics:module datasheet + carrier design guide public;toolchain:JetPack/Jetson Linux proprietary (NVIDIA SLA);datasheets:public;blobs:proprietary drivers/bootloader;retail:via distributors", "confidence": "high", "note": "Public datasheets and purchasable through distributors, but the SDK depends on proprietary NVIDIA blobs under license.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "Official spec: up to 157 TOPS, 1792-core Ampere GPU, 8/16GB LPDDR5, 260-pin SO-DIMM", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Production SO-DIMM module sold through NVIDIA's distributor network with strong Jetson community; popular in commercial drones/robotics but not mass-market.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "Module family page indicating production module availability", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "Up to 157 TOPS; 8GB or 16GB LPDDR5; runs mid-to-large LLMs/VLMs and multi-stream vision", "confidence": "high", "note": "Strong mid-range edge compute; the 16GB/157 TOPS variant handles sizeable models, below AGX Orin but above Orin Nano.", "sources": [ { "url": "https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin", "shows": "157 TOPS and up to 16GB LPDDR5 spec", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheets and purchasable through distributors, but the SDK depends on proprietary NVIDIA blobs under license." }, { "product": "Orange Pi 5", "org": "Orange Pi (Shenzhen Xunlong)", "type": "hardware", "description": "A single-board computer (SBC) built around the Rockchip RK3588S SoC, an 8-core 64-bit ARM processor (4x Cortex-A76 + 4x Cortex-A55) with a Mali-G610 GPU and a built-in NPU rated at 6 TOPS. It ships in 4/8/16GB LPDDR4/4X configurations and is made by Orange Pi (Shenzhen Xunlong). Widely sold at retail for roughly $69-$115, it supports Android 12, Debian, Ubuntu, and Orange Pi OS, and is one of the most popular and affordable RK3588S edge-AI boards.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:published PDF;toolchain:open GPL-2.0 BSP/build system;datasheets:public RK3588S;blobs:Rockchip firmware required;retail:global", "confidence": "high", "note": "Proprietary Rockchip silicon with an open GPL-2.0 BSP, public datasheet and schematics, and global retail availability, but bootable images depend on Rockchip firmware blobs.", "sources": [ { "url": "https://github.com/orangepi-xunlong/orangepi-build", "shows": "official GPL-2.0 build system supporting Orange Pi 5 / RK3588S", "accessed": "2026-06-22" }, { "url": "http://www.orangepi.org/html/hardWare/computerAndMicrocontrollers/service-and-support/Orange-pi-5.html", "shows": "official support page with schematic, manual, and source downloads", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "Sold directly on AliExpress and Amazon at ~$69-$115 across multiple SKUs, with a large third-party software/community ecosystem.", "sources": [ { "url": "https://www.amazon.com/Orange-Pi-Frequency-Development-Android12/dp/B0BN16ZLXB", "shows": "Orange Pi 5 retail listing on Amazon", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "6 TOPS NPU; up to 16GB LPDDR4/4X; runs quantized LLMs up to ~8B (W4A16) and CV models via RKNN/RKLLM", "confidence": "high", "note": "The RK3588S NPU does 6 TOPS with mixed-precision support and runs small on-device LLMs, but well below discrete GPUs and dedicated NPUs.", "sources": [ { "url": "http://www.orangepi.org/html/hardWare/computerAndMicrocontrollers/details/Orange-Pi-5.html", "shows": "official spec: RK3588S, 6 TOPS NPU, 4/8/16GB", "accessed": "2026-06-22" } ] }, "maturity": 4.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary Rockchip silicon with an open GPL-2.0 BSP, public datasheet and schematics, and global retail availability, but bootable images depend on Rockchip firmware blobs." }, { "product": "Radxa ROCK 5B", "org": "Radxa", "type": "hardware", "description": "A single-board computer (SBC) built on the Rockchip RK3588 SoC, pairing a quad-core Cortex-A76 (up to 2.4 GHz) with a quad-core Cortex-A55 and a built-in 6 TOPS NPU, in a 100 x 75 mm form factor. It is offered in 4/8/16GB LPDDR4X (ROCK 5B) and up to 32GB LPDDR5 (ROCK 5B+). Made by Radxa, which publishes complete hardware design schematics and software source, it is one of the most popular high-end RK3588 maker boards.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:PDF + SMD maps + DXF + 3D STP + CE/FCC public;toolchain:open BSP/kernel/U-Boot;blobs:Rockchip rkbin/DDR/TF-A required;retail:global", "confidence": "high", "note": "Schematics and mechanical files are publicly downloadable and the BSP is open, but the RK3588 needs proprietary Rockchip blobs and editable PCB source is not released.", "sources": [ { "url": "https://docs.radxa.com/en/rock5/rock5b/download", "shows": "downloadable schematic PDFs, SMD maps, DXF, 3D STP, CE/FCC reports", "accessed": "2026-06-22" }, { "url": "https://github.com/radxa-repo/bsp", "shows": "official open Radxa BSP build tool with ROCK 5B profile", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 5, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "One of the most popular RK3588 SBCs, sold globally through authorized distributors with 15+ third-party OS ports and an active community.", "sources": [ { "url": "https://ameridroid.com/products/rock5-model-b", "shows": "live retail listing at an authorized US distributor", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "6 TOPS NPU; up to 16GB LPDDR4X (5B) / 32GB LPDDR5 (5B+); RK3588 8-core CPU", "confidence": "high", "note": "6 TOPS INT8 plus up to 16-32GB RAM runs quantized small/mid LLMs and real-time vision via RKNN, strong for an edge SBC.", "sources": [ { "url": "https://docs.radxa.com/en/rock5/rock5b/getting-started/introduction", "shows": "6 TOPS NPU; RAM up to 16GB (5B) / 32GB LPDDR5 (5B+)", "accessed": "2026-06-22" } ] }, "maturity": 4.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Schematics and mechanical files are publicly downloadable and the BSP is open, but the RK3588 needs proprietary Rockchip blobs and editable PCB source is not released." }, { "product": "Khadas Edge2", "org": "Khadas (Shenzhen Wesion)", "type": "hardware", "description": "A credit-card-sized single-board computer (SBC) built around the Rockchip RK3588S2, an 8nm 8-core 64-bit ARM SoC with a Mali-G610 MP4 GPU and a built-in NPU rated at 6 TOPS. It ships in Basic (8GB + 32GB eMMC) and Pro (16GB + 64GB eMMC) configurations in a compact fanless design. Made by Khadas (Shenzhen Wesion), it runs Ubuntu and Android with mainline Armbian support.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:versioned PDFs published;toolchain:open GPL-2.0 Fenix + mainline Armbian;datasheets:public SoC TRM;blobs:Rockchip NPU/boot blobs required;retail:global", "confidence": "high", "note": "Khadas publishes versioned board schematics and an open toolchain with mainline Armbian, but proprietary Rockchip silicon and required blobs keep it at open_toolchain.", "sources": [ { "url": "https://dl.khadas.com/products/edge2/schematics/", "shows": "versioned schematic PDFs (edge2-sch-v11/12/13)", "accessed": "2026-06-22" }, { "url": "https://github.com/khadas/fenix", "shows": "Fenix build system (GPL-2.0) with Edge2 as a target", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "niche", "signal_type": "reported_traction", "confidence": "high", "note": "Sold on Amazon US and Newegg (Basic and Pro), with an active community forum and mainline Armbian support, but smaller than Raspberry Pi-class platforms.", "sources": [ { "url": "https://www.amazon.com/khadas-Computer-RK3588S-Portable-Compact/dp/B0BFVSC2D5", "shows": "Edge2 Pro 16GB retail listing on Amazon US", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "6 TOPS NPU; 8/16GB LPDDR4X; runs quantized vision models in real time and small LLMs (~1-3B INT4) via RKNN", "confidence": "high", "note": "The RK3588S2 NPU delivers 6 TOPS, handling real-time CV and small quantized LLMs, mid-tier versus Jetson Orin.", "sources": [ { "url": "https://www.khadas.com/edge2", "shows": "6 TOPS NPU; 8GB/16GB LPDDR4X options", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Khadas publishes versioned board schematics and an open toolchain with mainline Armbian, but proprietary Rockchip silicon and required blobs keep it at open_toolchain." }, { "product": "BeagleY-AI", "org": "BeagleBoard.org Foundation", "type": "hardware", "description": "An open source single-board computer (SBC) from the BeagleBoard.org Foundation, built around the Texas Instruments AM67A vision processor. It pairs a quad-core Cortex-A53 CPU at 1.4 GHz with a TI C7x deep-learning accelerator subsystem (dual C7x DSP + MMA) rated at up to 4 TOPS, an IMG BXS GPU, and Cortex-R5 real-time cores, with 4GB LPDDR4 and a 40-pin Raspberry Pi-compatible header (~$72). It is OSHWA-certified and targets vision, robotics, and smart cameras.", "openness": { "class": "open_hardware", "score": 5, "components": "schematics + 3D/mechanical files (CC-BY/CC-BY-SA-4.0);OSHWA-certified US002616;open Linux/U-Boot BSP;public TI AM67A datasheet;TI DSP/WiFi firmware blobs a caveat", "confidence": "high", "note": "Full board design files published under Creative Commons and OSHWA-certified with an open toolchain and public datasheet, though proprietary AM67A silicon and firmware blobs are a caveat.", "sources": [ { "url": "https://github.com/beagleboard/beagley-ai", "shows": "repo with design files, 3D models, OSHWA cert, schematic PDF, CC-BY-4.0 license", "accessed": "2026-06-22" }, { "url": "https://docs.beagleboard.org/boards/beagley/ai/03-design.html", "shows": "CC-BY-SA-4.0 license, OSHWA cert US002616, open design-file hosting", "accessed": "2026-06-22" } ], "bucket": "open" }, "adoption": { "level": 4, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "In stock at $71.99 with volume pricing at Seeed and orderable via TI, backed by the established BeagleBoard.org community, though recent (2024) so installed base trails Raspberry Pi.", "sources": [ { "url": "https://www.seeedstudio.com/BeagleYr-AI-beagleboard-orgr-4-TOPS-AI-Acceleration-powered-by-TI-AM67A.html", "shows": "$71.99, in stock, with volume pricing", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "4 TOPS (TI C7x DL accelerator); 4GB LPDDR4; runs quantized CNNs / small object-detection and classification models", "confidence": "high", "note": "4 TOPS plus an IMG BXS GPU suits edge vision and quantized CNNs, but fixed 4GB RAM and modest TOPS rule out large LLMs.", "sources": [ { "url": "https://www.beagleboard.org/boards/beagley-ai", "shows": "dual C7x DSP + MMA at 4 TOPS, 4GB LPDDR4, quad Cortex-A53", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Full board design files published under Creative Commons and OSHWA-certified with an open toolchain and public datasheet, though proprietary AM67A silicon and firmware blobs are a caveat." }, { "product": "Rockchip RK3588", "org": "Rockchip", "type": "hardware", "description": "An application-processor SoC/chipset (not a board) made by Fuzhou Rockchip. It is an octa-core ARM SoC pairing 4x Cortex-A76 with 4x Cortex-A55, a Mali-G610 MC4 GPU, and a 6 TOPS triple-core NPU supporting INT4/INT8/INT16/FP16/BF16/TF32, with LPDDR4x/LPDDR5 up to 32GB and PCIe 3.0. It is the chipset behind a broad range of edge/SBC products (Orange Pi 5, Radxa Rock 5B, Khadas Edge2), making it the de facto high-end ARM SBC SoC of its generation.", "openness": { "class": "documented", "score": 3, "components": "full TRM + datasheet public (no NDA);NPU SDK on GitHub but proprietary RKNN license + prebuilt blobs;rknpu kernel driver open;proprietary GPU/NPU firmware", "confidence": "high", "note": "Documentation is unusually open (full TRM and datasheet public) and the RKNN toolkit is openly distributed, but the SDK ships as prebuilt binaries under a proprietary license with closed firmware.", "sources": [ { "url": "https://www.rock-chips.com/a/en/products/RK35_Series/2022/0926/1660.html", "shows": "official RK3588 page: 8-core A76+A55, 6 TOPS triple-core NPU, Mali-G610 MC4", "accessed": "2026-06-22" }, { "url": "https://github.com/airockchip/rknn-toolkit2/blob/master/LICENSE", "shows": "SDK under a proprietary RKNN SDK License", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 5, "reach": "mass-market", "signal_type": "reported_traction", "confidence": "high", "note": "Powers a very large multi-vendor set of SBCs and modules (Orange Pi, Radxa, Khadas, FriendlyELEC, Firefly), widely available at retail and broadly supported in community Linux.", "sources": [ { "url": "https://forum.armbian.com/topic/26398-video-comparing-rk3588-sbcs-nanopi-r6s-khadas-edge2-radxa-rock5b-mekotronics-r58-mini-r58x-4g-orange-pi-5/", "shows": "independent comparison of many RK3588 SBCs from multiple vendors", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "6 TOPS triple-core NPU (INT4-TF32); up to 32GB LPDDR4x/LPDDR5 (board-dependent); runs quantized on-device LLMs via RKNN-LLM", "confidence": "high", "note": "The 6 TOPS NPU is genuine edge-AI capability and the RKLLM SDK runs quantized LLMs on-chip, but modest versus dedicated accelerators.", "sources": [ { "url": "https://github.com/airockchip/rknn-llm", "shows": "official RKLLM SDK running LLMs on RK3588-series NPU", "accessed": "2026-06-22" } ] }, "maturity": 4.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Documentation is unusually open (full TRM and datasheet public) and the RKNN toolkit is openly distributed, but the SDK ships as prebuilt binaries under a proprietary license with closed firmware." }, { "product": "Hailo-8", "org": "Hailo Technologies", "type": "hardware", "description": "An edge AI inference accelerator delivering up to 26 TOPS at roughly 2.5W typical power, built by Hailo Technologies (Israel). It is sold as a bare chip and in M.2, mPCIe, and PCIe card form factors with on-die memory, supporting TensorFlow, TFLite, Keras, PyTorch, and ONNX via the Hailo Dataflow Compiler and HailoRT runtime. It is widely deployed in robotics, ADAS, security cameras, and industrial vision.", "openness": { "class": "documented", "score": 3, "components": "schematics:no;toolchain:open runtime (HailoRT) + closed/registration Dataflow Compiler;datasheets:public briefs;blobs:proprietary silicon + firmware;retail:global modules", "confidence": "high", "note": "Proprietary silicon with an open source runtime and public product briefs, but the model-compiler toolchain requires registration and silicon is closed.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-8-ai-accelerator/", "shows": "official product page: form factors, frameworks, software suite", "accessed": "2026-06-22" }, { "url": "https://github.com/hailo-ai/hailort", "shows": "open source HailoRT inference runtime", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "broad", "signal_type": "reported_traction", "confidence": "high", "note": "Retail-available modules and a large developer base; vendor reports 100K+ users and hundreds of deployed customer products, with an active GitHub org.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-8-ai-accelerator/", "shows": "vendor claim of >100K users and hundreds of customer products", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "26 TOPS; CNN/vision inference; supports TF/TFLite/PyTorch/ONNX; on-die memory", "confidence": "high", "note": "Strong dedicated CNN inference throughput for the edge, but vision-oriented and limited to models that fit on-die memory.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-8-ai-accelerator/", "shows": "26 TOPS, 2.5W typical, on-die memory", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary silicon with an open source runtime and public product briefs, but the model-compiler toolchain requires registration and silicon is closed." }, { "product": "Hailo-10H", "org": "Hailo Technologies", "type": "hardware", "description": "Hailo's generative-AI edge accelerator, offered as an M.2 module and as a chip, delivering 40 TOPS at INT4 and 20 TOPS at INT8 with ~2.5W typical power. Unlike the Hailo-8, it adds a direct DDR interface so it can run LLMs and vision-language models (VLMs) on-device. It targets PCs, automotive, and edge devices needing local generative AI, supported by Hailo's software suite and a GenAI model zoo.", "openness": { "class": "documented", "score": 3, "components": "schematics:no;toolchain:open runtime + MIT GenAI model zoo;datasheets:public (June 2025);blobs:proprietary silicon/firmware;retail:M.2 modules purchasable", "confidence": "high", "note": "Proprietary silicon with published datasheets and an open GenAI model zoo, but the silicon and full toolchain are closed.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-10h-m-2-ai-acceleration-module/", "shows": "official module product page for Hailo-10H", "accessed": "2026-06-22" }, { "url": "https://github.com/hailo-ai/hailo_model_zoo_genai", "shows": "MIT-licensed GenAI model zoo for Hailo-10H", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Newer generative-AI part; purchasable M.2 modules and a dedicated GenAI repo, but smaller install base than the Hailo-8.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-10h-m-2-ai-acceleration-module/", "shows": "productized, purchasable M.2 generative-AI module", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "40 TOPS INT4 / 20 TOPS INT8; runs LLMs & VLMs via direct DDR interface", "confidence": "high", "note": "Edge-first part that runs generative models locally via a DDR interface, a step beyond CNN-only edge accelerators.", "sources": [ { "url": "https://hailo.ai/products/ai-accelerators/hailo-10h-m-2-ai-acceleration-module/", "shows": "40 TOPS INT4 / 20 TOPS INT8, DDR interface, LLM/VLM support", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary silicon with published datasheets and an open GenAI model zoo, but the silicon and full toolchain are closed." }, { "product": "Google Coral Dev Board", "org": "Google", "type": "hardware", "description": "A single-board computer from Google's Coral platform that pairs a removable System-on-Module (NXP i.MX 8M SoC) with a baseboard and integrates Google's Edge TPU ASIC. The Edge TPU delivers 4 TOPS at about 2 TOPS/watt, accelerating int8-quantized TensorFlow Lite models for local inference. It is aimed at prototyping edge AI that can scale to production by reusing the SoM. The Edge TPU is designed and supplied by Google.", "openness": { "class": "documented", "score": 3, "components": "schematics:no full board files;toolchain:open runtime (libedgetpu) + TFLite, closed Edge TPU compiler;datasheets:public;blobs:proprietary Edge TPU ASIC + closed compiler;retail:broadly purchasable", "confidence": "high", "note": "Public datasheet and open runtime/TFLite path, but the Edge TPU silicon and the model compiler are proprietary closed binaries.", "sources": [ { "url": "https://coral.ai/docs/dev-board/datasheet/", "shows": "datasheet: 4 TOPS, 2 TOPS/W, SoM form factor, TFLite, Google-designed Edge TPU", "accessed": "2026-06-22" }, { "url": "https://github.com/google-coral/libedgetpu", "shows": "open source Edge TPU userspace runtime driver", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "broad", "signal_type": "reported_traction", "confidence": "medium", "note": "Long-established, widely distributed edge-AI dev platform with an active google-coral GitHub org; the older edgetpu repo was archived April 2026 with libedgetpu now maintained.", "sources": [ { "url": "https://github.com/google-coral/libedgetpu", "shows": "maintained official runtime repo; broad ecosystem", "accessed": "2026-06-22" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "4 TOPS (2 TOPS/W); int8-quantized models; TensorFlow Lite only", "confidence": "high", "note": "Modest 4 TOPS limited to int8 TFLite models; suited to small CNN inference rather than large or generative models.", "sources": [ { "url": "https://coral.ai/docs/dev-board/datasheet/", "shows": "4 TOPS, 2 TOPS/W, TFLite-focused Edge TPU", "accessed": "2026-06-22" } ] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheet and open runtime/TFLite path, but the Edge TPU silicon and the model compiler are proprietary closed binaries." }, { "product": "Axelera Metis AIPU", "org": "Axelera AI", "type": "hardware", "description": "An edge AI inference accelerator from Axelera AI (Netherlands), built on a quad-core in-memory-compute architecture and sold as M.2 modules, PCIe cards, and an integrated compute board. A single quad-core Metis delivers on the order of 214 TOPS, with PCIe configurations scaling to multiple units. It is paired with the Voyager SDK (100+ models) and targets computer-vision workloads such as multi-channel video analytics and quality inspection.", "openness": { "class": "documented", "score": 3, "components": "schematics:no;toolchain:Voyager SDK on public GitHub;datasheets:product pages/briefs public;blobs:proprietary AIPU silicon + firmware;retail:webstore + partners", "confidence": "medium", "note": "Proprietary in-memory-compute silicon with a publicly available Voyager SDK on GitHub and retail purchasing, but board design files are not open.", "sources": [ { "url": "https://www.axelera.ai/", "shows": "Metis AIPU form factors, ~214 TOPS, Voyager SDK, webstore availability", "accessed": "2026-06-22" }, { "url": "https://github.com/axelera-ai-hub/voyager-sdk", "shows": "publicly available Voyager SDK repo under Axelera's official org", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Commercially available via webstore and partners with a public, versioned SDK, but a younger ecosystem than Hailo/Coral.", "sources": [ { "url": "https://github.com/axelera-ai-hub/voyager-sdk", "shows": "actively released SDK branches indicating developer traction", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "~214 TOPS per quad-core Metis; PCIe x4 scales across 4 units; Voyager SDK with 100+ CV models", "confidence": "medium", "note": "High raw inference throughput via in-memory compute, currently oriented toward CNN/computer-vision pipelines.", "sources": [ { "url": "https://www.axelera.ai/", "shows": "~214 TOPS Metis, PCIe scaling, Voyager SDK model coverage", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary in-memory-compute silicon with a publicly available Voyager SDK on GitHub and retail purchasing, but board design files are not open." }, { "product": "MemryX MX3", "org": "MemryX", "type": "hardware", "description": "An edge AI accelerator from MemryX (USA) built on a dataflow architecture, where each MX3 chip provides ~6 TOPS and an M.2 module packs four chips for up to 24 TOPS at roughly 6-8W with passive cooling. The M.2 (2280, M-Key) module uses PCIe Gen3 and supports 4/8/16-bit weights plus BFloat16, with no model-zoo lock-in. Sold at retail (~$149) via Amazon, Mouser, and distributors, it won a 2025 Edge AI & Vision Alliance Product of the Year award and is supported by the MemryX SDK and open source tooling.", "openness": { "class": "documented", "score": 3, "components": "schematics:no;toolchain:open runtime (MxAccl) + utils + driver mirror, closed NeuralCompiler;datasheets:public;blobs:proprietary silicon;retail:Amazon/Mouser ~$149", "confidence": "high", "note": "Proprietary dataflow silicon with public datasheets, retail availability, and several open source runtime/driver repos, though the compiler core is closed.", "sources": [ { "url": "https://developer.memryx.com/specs/M.2_datasheet.html", "shows": "MX3 M.2 datasheet: chip count, TOPS, data formats, power", "accessed": "2026-06-22" }, { "url": "https://github.com/memryx/MxAccl", "shows": "open source MemryX C++ runtime library", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Low-cost retail availability (Amazon, Mouser), independent reviews, a 2025 industry award, and an active GitHub org point to growing but emerging traction.", "sources": [ { "url": "https://www.tomshardware.com/tech-industry/artificial-intelligence/memryx-launches-usd149-mx3-m-2-ai-accelerator-module-capable-of-24-tops-compute-power", "shows": "retail launch at $149 with broad distribution", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "6 TOPS/chip, up to 24 TOPS (4-chip M.2); 4/8/16-bit weights + BFloat16; ~42M params per module", "confidence": "high", "note": "Moderate throughput with flexible precision, but on-chip parameter capacity caps deployable model size.", "sources": [ { "url": "https://developer.memryx.com/specs/M.2_datasheet.html", "shows": "per-chip/module TOPS, supported data formats, parameter capacity", "accessed": "2026-06-22" } ] }, "maturity": 3.0, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary dataflow silicon with public datasheets, retail availability, and several open source runtime/driver repos, though the compiler core is closed." }, { "product": "Qualcomm Dragonwing RB3 Gen 2 Development Kit", "org": "Qualcomm", "type": "hardware", "description": "A Qualcomm edge AI and IoT development kit built around the QCS6490 SoC (Kryo 670 CPU, Adreno 643L GPU, Hexagon NPU) delivering up to 12 dense TOPS, with a QCS5430 variant also offered. It follows the modular 96Boards form factor and ships with a Qualcomm Linux software stack plus Qualcomm AI Hub, Edge Impulse, and Foundries.io support. It is a reference platform for prototyping commercial edge-AI products on Qualcomm silicon.", "openness": { "class": "documented", "score": 3, "components": "schematics:partial (96Boards mezzanine standard);toolchain:open Qualcomm Linux + AI Hub;datasheets:public brief;blobs:proprietary silicon + firmware;retail:via Thundercomm/distributors", "confidence": "high", "note": "Proprietary Qualcomm silicon with public docs and a Linux SDK, sold through distributors, but the board itself is not fully open hardware.", "sources": [ { "url": "https://www.qualcomm.com/developer/hardware/rb3-gen-2-development-kit", "shows": "official page: QCS6490, up to 12 TOPS, 96Boards design, Qualcomm Linux stack", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Sold via Thundercomm and other distributors; first-class support in Edge Impulse, indicating an established but specialist developer base.", "sources": [ { "url": "https://docs.edgeimpulse.com/hardware/boards/qualcomm-rb3-gen-2-dev-kit", "shows": "platform support in the Edge Impulse ecosystem", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "12 TOPS (Hexagon NPU); QCS6490 octa-core; multi-camera vision", "confidence": "high", "note": "12 dense TOPS with octa-core CPU and dual-ISP vision puts it among the more capable edge-AI dev kits here.", "sources": [ { "url": "https://www.qualcomm.com/developer/hardware/rb3-gen-2-development-kit", "shows": "up to 12 dense TOPS, QCS6490 SoC specs", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Proprietary Qualcomm silicon with public docs and a Linux SDK, sold through distributors, but the board itself is not fully open hardware." }, { "product": "Arduino UNO Q", "org": "Arduino", "type": "hardware", "description": "A hybrid single-board computer in the UNO form factor that pairs a Qualcomm Dragonwing QRB2210 quad-core applications processor (Adreno GPU, dual ISP, running Debian Linux) with an STM32U585 microcontroller for real-time control. It targets AI-accelerated vision and sound applications, programmed via the open source Arduino App Lab. Made by Arduino (now a Qualcomm company), it ships in 2GB/16GB and 4GB/32GB variants around EUR 47-65, a low-cost on-ramp to Qualcomm edge-AI silicon for the maker community.", "openness": { "class": "documented", "score": 3, "components": "schematics:open (gerbers + schematics CC-BY-SA 4.0);toolchain:open App Lab;datasheets:public;blobs:proprietary Qualcomm SoC firmware;retail:global", "confidence": "high", "note": "Board schematics and gerbers are CC-BY-SA 4.0 and tooling is open, but the QRB2210 SoC remains proprietary silicon needing firmware blobs.", "sources": [ { "url": "https://www.arduino.cc/product-uno-q", "shows": "QRB2210 + STM32U585, schematics/gerbers under CC-BY-SA 4.0, App Lab open source", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "broad", "signal_type": "reported_traction", "confidence": "high", "note": "Sold globally through Arduino Store plus Digikey, Mouser, and RS Components, backed by Arduino's large maker community.", "sources": [ { "url": "https://store.arduino.cc/pages/uno-q", "shows": "availability across Arduino Store and authorized distributors", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "QRB2210 quad-core 2.0 GHz + Adreno GPU; up to 4GB RAM; AI acceleration", "confidence": "medium", "note": "Capable Linux+MCU hybrid with GPU/ISP-based AI acceleration, but no published TOPS figure so throughput is modest versus dedicated-NPU kits.", "sources": [ { "url": "https://store.arduino.cc/pages/uno-q", "shows": "QRB2210 quad-core, 2GB/4GB RAM, AI-accelerated tasks", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Board schematics and gerbers are CC-BY-SA 4.0 and tooling is open, but the QRB2210 SoC remains proprietary silicon needing firmware blobs." }, { "product": "Texas Instruments AM67A", "org": "Texas Instruments", "type": "hardware", "description": "A Texas Instruments edge-AI vision SoC built on a quad Arm Cortex-A53 CPU at 1.4 GHz plus dual C7x DSP cores with Matrix Multiply Accelerators delivering 4 TOPS of deep-learning performance. On 16nm FinFET, it integrates a multi-camera ISP (up to 4 CSI-2 inputs), PCIe Gen3, and Gigabit Ethernet with TSN, supporting up to 8GB LPDDR4. It is supported by TI's Processor SDK and Edge AI Studio and is the chip behind the BeagleY-AI SBC.", "openness": { "class": "documented", "score": 3, "components": "schematics:reference designs (BeagleY-AI board files open);toolchain:open SDK + TIDL tools + Edge AI Studio;datasheets:public (Rev B);blobs:proprietary silicon + TIDL firmware;retail:active via TI and partners", "confidence": "high", "note": "Public datasheet and open SDK/TIDL tools on GitHub, commercially active, but proprietary silicon and firmware keep it short of fully open hardware.", "sources": [ { "url": "https://www.ti.com/product/AM67A", "shows": "official page: 4 TOPS C7x+MMA, quad A53, public datasheet, ACTIVE status, SDKs", "accessed": "2026-06-22" }, { "url": "https://github.com/TexasInstruments/edgeai-tidl-tools", "shows": "open source TIDL deep-learning toolchain for AM67A-class processors", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "broad", "signal_type": "reported_traction", "confidence": "high", "note": "ACTIVE production part available from TI and distributors, and the basis of the widely sold BeagleY-AI board, giving solid commercial and community reach.", "sources": [ { "url": "https://www.beagleboard.org/boards/beagley-ai", "shows": "AM67A powers the BeagleY-AI SBC", "accessed": "2026-06-22" } ] }, "capability": { "score": 4, "basis": "feature_matrix", "value": "4 TOPS (dual C7x+MMA); quad A53 1.4 GHz; up to 8GB LPDDR4", "confidence": "high", "note": "4 TOPS plus a strong vision/ISP pipeline and 8GB memory make it a capable dedicated edge-AI vision SoC.", "sources": [ { "url": "https://www.ti.com/product/AM67A", "shows": "4 TOPS deep-learning, quad A53, 8GB LPDDR4, 600MP/s ISP", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheet and open SDK/TIDL tools on GitHub, commercially active, but proprietary silicon and firmware keep it short of fully open hardware." }, { "product": "NXP i.MX 8M Plus", "org": "NXP Semiconductors", "type": "hardware", "description": "An NXP applications processor family with up to a quad Arm Cortex-A53 CPU, a Cortex-M7 real-time core, and an integrated NPU delivering up to 2.3 TOPS for on-device ML and vision. It includes dual ISPs and two camera inputs and targets smart home, building, city, and Industry 4.0 use cases. NXP provides a Linux/Yocto BSP and public datasheets, and the part is commercially available with SoM options from ADLINK, Toradex, and SolidRun.", "openness": { "class": "documented", "score": 3, "components": "schematics:no (EVK reference designs only);toolchain:Linux/Yocto BSP;datasheets:public (IMX8MPCEC);blobs:proprietary silicon + NPU firmware;retail:commercial via NXP and SoM partners", "confidence": "high", "note": "Public datasheets and a Linux BSP with broad commercial availability, but proprietary silicon with no open board files.", "sources": [ { "url": "https://www.nxp.com/products/i.MX8MPLUS", "shows": "official page: Cortex-A53/M7, up to 2.3 TOPS NPU, ML/vision focus", "accessed": "2026-06-22" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "broad", "signal_type": "reported_traction", "confidence": "high", "note": "Widely deployed industrial processor with SoMs from multiple vendors (ADLINK, Toradex Verdin, SolidRun) and broad distributor availability.", "sources": [ { "url": "https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-plus", "shows": "Toradex Verdin SoM productizes the i.MX 8M Plus", "accessed": "2026-06-22" } ] }, "capability": { "score": 3, "basis": "feature_matrix", "value": "2.3 TOPS NPU; quad Cortex-A53 + Cortex-M7; LPDDR4 support", "confidence": "high", "note": "2.3 TOPS integrated NPU with dual-ISP vision is solid for industrial ML but lower throughput than the 4-12 TOPS dedicated kits here.", "sources": [ { "url": "https://www.nxp.com/docs/en/fact-sheet/IMX8MPLUSFS.pdf", "shows": "fact sheet: 2.3 TOPS NPU, CPU/ISP specs", "accessed": "2026-06-22" } ] }, "maturity": 3.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Public datasheets and a Linux BSP with broad commercial availability, but proprietary silicon with no open board files." }, { "product": "Sipeed MaixCAM", "org": "Sipeed", "type": "hardware", "description": "A compact AI vision and audio dev board built around the SOPHGO SG2002 SoC (a 1 GHz RISC-V C906 big core plus a 700 MHz small core) with a 1 TOPS INT8 NPU that also supports BF16 models, 256MB DDR3, a 2.3-inch IPS touchscreen, and dual MIPI-CSI camera input. It is programmed via the open source MaixPy (Python) and MaixCDK (C/C++) SDKs and sold through Taobao/AliExpress. It matters as an ultra-low-cost RISC-V edge-AI vision module with strong open tooling.", "openness": { "class": "open_toolchain", "score": 4, "components": "schematics:available via wiki/GitHub;toolchain:fully open (MaixPy/MaixCDK, Apache-2.0);datasheets:public on wiki;blobs:SG2002 silicon proprietary;retail:global via AliExpress/Taobao", "confidence": "high", "note": "Open Apache-2.0 SDKs, public schematics/datasheets, and global retail availability, though the SG2002 silicon itself is proprietary.", "sources": [ { "url": "https://wiki.sipeed.com/hardware/en/maixcam/maixcam.html", "shows": "spec page: SG2002 RISC-V, 1 TOPS NPU, 256MB DDR3, open schematics/datasheets, MaixPy/MaixCDK", "accessed": "2026-06-22" }, { "url": "https://github.com/sipeed/MaixPy", "shows": "open source (Apache-2.0) Python SDK for MaixCAM", "accessed": "2026-06-22" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "niche", "signal_type": "reported_traction", "confidence": "medium", "note": "Sold globally via AliExpress/Taobao with an active MaixPy/MaixHub community, but reach is concentrated among hobbyist and RISC-V edge-AI developers.", "sources": [ { "url": "https://github.com/sipeed/MaixPy", "shows": "active open source repo and release cadence", "accessed": "2026-06-22" } ] }, "capability": { "score": 2, "basis": "feature_matrix", "value": "1 TOPS INT8 NPU; RISC-V C906; 256MB DDR3", "confidence": "high", "note": "1 TOPS and only 256MB RAM suit lightweight vision/audio models, the lowest compute tier among these devices.", "sources": [ { "url": "https://wiki.sipeed.com/hardware/en/maixcam/maixcam.html", "shows": "1 TOPS@INT8 NPU, RISC-V C906 1GHz, 256MB DDR3", "accessed": "2026-06-22" } ] }, "maturity": 2.5, "mature": false, "version_note": "Verified live 2026-06-22 via primary sources. Open Apache-2.0 SDKs, public schematics/datasheets, and global retail availability, though the SG2002 silicon itself is proprietary." } ] }, "safeguards": { "label": "Safety & Guardrails", "arc": "Infrastructure", "layer": "infrastructure", "stage": { "num": 3, "name": "Viable Alternatives" }, "gaps": [ "maturity", "adoption" ], "products": [ { "product": "Aegis AI Content Safety", "org": "NVIDIA", "type": "model", "description": "NVIDIA's content-safety classifier (now also called Llama Nemotron Safety Guard Defensive), a LlamaGuard/Llama2-7B-based model parameter-efficiently tuned on NVIDIA's Aegis safety dataset to classify prompts and responses across 13 critical risk categories.", "openness": { "score": 3, "class": "open_weights", "components": "weights:adapter-open(HF) but base LlamaGuard gated via Meta access;data:Aegis safety dataset documented;license:Llama-2-Community(NOT OSI)", "confidence": "medium", "note": "Open-weights at the adapter level but the LlamaGuard base is access-gated through Meta and the license is the non-OSI Llama 2 Community License, so it lands at open_weights (3) with a gating caveat.", "sources": [ { "url": "https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0", "shows": "Llama 2 Community License; adapter weights public, base gated via Meta; 13 risk categories; built on LlamaGuard/Llama2-7B", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "low", "note": "NVIDIA safety classifier with a documented dataset and NeMo integration; modest standalone HF adoption, distributed largely via NVIDIA's stack.", "sources": [ { "url": "https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0", "shows": "NVIDIA Aegis content-safety classifier; 13 categories", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "risk coverage", "value": "13 critical safety risk categories across prompts and responses, with an open safety dataset and a defensive/permissive variant pair.", "confidence": "low", "note": "Broad risk taxonomy with a documented training dataset; comparable coverage to Llama Guard, its base.", "sources": [ { "url": "https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0", "shows": "13 risk categories; defensive variant", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "NVIDIA Aegis AI Content Safety (Defensive 1.0), aka Llama Nemotron Safety Guard Defensive V1. Llama 2 Community License (not OSI). Adapter weights public on HF, but the LlamaGuard base must be obtained via Meta's access request, so not fully open-weight. Aegis safety dataset documented. Verified June 2026." }, { "product": "OpenGuardrails", "org": "OpenGuardrails", "type": "software", "description": "An open, context-aware AI guardrails platform that pairs a safety/manipulation-detection model (OpenGuardrails-Text-2510, a 3.3B quantized LLM) with a deployable guardrails platform: APIs, deployment scripts, and modular components for private or on-prem use. Covers content safety, manipulation attacks (prompt injection, jailbreak, code-interpreter abuse), and data leakage across 119 languages.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI) for both platform and model;source:public;self-host:yes(on-prem/private);model:OpenGuardrails-Text-2510 open weights Apache-2.0", "confidence": "high", "note": "Fully open source: both the guardrails platform and the safety model are Apache-2.0 and self-hostable, including on-prem deployment.", "sources": [ { "url": "https://github.com/openguardrails/openguardrails", "shows": "Apache-2.0 platform; ~80 stars; on-prem deployment", "accessed": "2026-06-29" }, { "url": "https://huggingface.co/openguardrails/OpenGuardrails-Text-2510", "shows": "Apache-2.0; 3.3B quantized safety model; 119 languages", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "1K-10K", "signal_type": "stars_fallback", "confidence": "low", "note": "New (late 2025) project, ~80 GitHub stars (June 2026); stars-only proxy, capped per methodology; early-stage adoption.", "sources": [ { "url": "https://github.com/openguardrails/openguardrails", "shows": "~80 stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "platform + model + multilingual", "value": "A safety/manipulation-detection model plus a deployable platform (APIs, deployment scripts, modular components) covering content safety, prompt injection/jailbreak, code-interpreter abuse, and data leakage across 119 languages.", "confidence": "low", "note": "Unusually complete for its age: ships both an open safety model and an on-prem platform, with broad multilingual coverage.", "sources": [ { "url": "https://arxiv.org/abs/2510.19169", "shows": "context-aware guardrails platform + safety model; 119 languages; content safety, manipulation, data leakage", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "OpenGuardrails: Apache-2.0 platform AND model (OpenGuardrails-Text-2510, 3.3B quantized); 119 languages; on-prem/private deployment. ~80 GitHub stars (June 2026); arXiv 2510.19169. The first project to open-source both a large-scale safety LLM and a production guardrails platform. Verified June 2026." }, { "product": "Zentropi CoPE", "org": "Zentropi", "type": "model", "description": "Zentropi's Content Policy Evaluator (CoPE-A, 9B, built on Gemma-2-9B with LoRA): a steerable, bring-your-own-policy content-labeling model. Rather than a fixed taxonomy, it evaluates content against natural-language policies you define, scoring across harm areas like hate speech, sexual content, violence, harassment, self-harm, and toxicity.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(zentropi-ai/cope-a-9b on HF);base:Gemma-2-9B(LoRA);license:zentropi-openrail-m(OpenRAIL-M, use-restricted, NOT OSI)", "confidence": "medium", "note": "Open weights and self-hostable, but under an OpenRAIL-M license with use restrictions (e.g. surveillance), which is not OSI-approved, so open_weights (3).", "sources": [ { "url": "https://huggingface.co/zentropi-ai/cope-a-9b", "shows": "zentropi-openrail-m (OpenRAIL-M, use-restricted); open weights; Gemma-2-9B LoRA; policy-adaptive labeling; 23 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "1K-10K", "signal_type": "reported_traction", "confidence": "low", "note": "Niche but notable: a ROOST Model Community partner model (distributed via RMC); 23 HF likes (June 2026), downloads not tracked.", "sources": [ { "url": "https://huggingface.co/zentropi-ai/cope-a-9b", "shows": "23 likes (June 2026); RMC partner model", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "steerable bring-your-own-policy", "value": "Policy-adaptive labeling: evaluates content against arbitrary natural-language policies rather than a fixed taxonomy, with strong F1 across hate, sexual, violence, harassment, self-harm, and toxicity.", "confidence": "low", "note": "Like gpt-oss-safeguard, a bring-your-own-policy model that generalizes to custom policies; strong reported F1 across harm areas.", "sources": [ { "url": "https://huggingface.co/zentropi-ai/cope-a-9b", "shows": "policy-adaptive labeling; hate/sexual/violence/harassment/self-harm/toxicity", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "Zentropi CoPE-A (9B) on Gemma-2-9B (LoRA); license zentropi-openrail-m (OpenRAIL-M, use-restricted, NOT OSI). Open weights, self-hostable. RMC partner model distributed via the ROOST Model Community. 23 likes (June 2026); arXiv 2512.18027. Verified June 2026." }, { "product": "Qwen3Guard", "org": "Alibaba Cloud", "type": "model", "description": "Alibaba's Qwen3-based safety moderation model family (0.6B / 4B / 8B) that classifies prompts and responses as safe, controversial, or unsafe across 119 languages, with a token-level streaming variant for real-time moderation. Deployable via vLLM, SGLang, and HF inference.", "openness": { "score": 4, "class": "open_weights", "components": "weights:open(Qwen3Guard 0.6B/4B/8B on HF);data:not-released;license:Apache-2.0(OSI)", "confidence": "medium", "note": "Open weights under Apache-2.0 (OSI); training data not released, so open_weights (4) rather than open_source. Unusually broad multilingual coverage (119 languages) plus a streaming variant.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "shows": "Apache-2.0; open weights; 0.6B/4B/8B; 119 languages; safe/controversial/unsafe; ~74,669 downloads/month, 119 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "8B variant ~74.7K HF downloads/month (June 2026); cumulative across the three sizes is higher. Strong, fast-growing adoption.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "shows": "~74,669 downloads/month, 119 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "coverage + multilingual + streaming", "value": "Prompt and response classification across 119 languages with a three-level verdict and a token-level streaming variant; size ladder from 0.6B to 8B.", "confidence": "medium", "note": "Among the broadest open guardrails on language coverage, with a streaming mode that few others offer.", "sources": [ { "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "shows": "119 languages; prompt+response; streaming variant", "accessed": "2026-06-29" } ] }, "maturity": 4.0, "mature": false, "version_note": "Qwen3Guard (0.6B/4B/8B), Apache-2.0 open weights; 119 languages; generative + streaming variants. 8B ~74.7K HF downloads/month, 119 likes (June 2026). Verified live June 2026." }, { "product": "WildGuard", "org": "Ai2", "type": "model", "description": "Ai2's open safety moderation model (7B, fine-tuned from Mistral-7B) that jointly detects harmful prompts, harmful responses, and model refusals across 13 risk subcategories. Trained on the open WildGuardMix corpus and benchmarked above GPT-4 on several moderation tasks.", "openness": { "score": 4, "class": "open_weights", "components": "weights:open(allenai/wildguard on HF);data:open(WildGuardMix corpus public);base:Mistral-7B;license:Apache-2.0(OSI)", "confidence": "medium", "note": "Open weights under Apache-2.0 with the WildGuardMix training corpus published; from Ai2, which sits at the open end of the spectrum. Scored open_weights (4); a strong open release, just short of the full open_source pipeline tier.", "sources": [ { "url": "https://huggingface.co/allenai/wildguard", "shows": "Apache-2.0; 7B Mistral fine-tune; prompt/response/refusal detection; open WildGuardMix corpus; ~265K downloads/month", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "~265K HF downloads/month (June 2026); one of the most-downloaded open guardrail models.", "sources": [ { "url": "https://huggingface.co/allenai/wildguard", "shows": "~265,259 downloads/month, 52 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "joint detection + benchmarks", "value": "Joint prompt-harm, response-harm, and refusal detection across 13 subcategories; reported above GPT-4 and prior open baselines, with low over-blocking on benign traffic.", "confidence": "medium", "note": "Strong precision and low over-refusal; the joint refusal-detection head is a useful differentiator.", "sources": [ { "url": "https://huggingface.co/allenai/wildguard", "shows": "prompt/response/refusal detection; 13 subcategories; beats GPT-4 on benchmarks", "accessed": "2026-06-29" } ] }, "maturity": 4.0, "mature": false, "version_note": "WildGuard (7B, Mistral-based), Apache-2.0 open weights; trained on the open WildGuardMix dataset. ~265K HF downloads/month, 52 likes (June 2026). Verified live June 2026." }, { "product": "Llama Prompt Guard 2", "org": "Meta", "type": "model", "description": "Meta's compact prompt-injection and jailbreak classifier (86M and 22M), labeling prompts benign or malicious. Part of LlamaFirewall, it is tuned for very low latency (sub-100ms) and high precision, with multilingual coverage across eight languages.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Llama-Prompt-Guard-2-86M/22M on HF);data:not-released;license:Llama-4-Community(NOT OSI)", "confidence": "medium", "note": "Open weights under Meta's Llama 4 Community License, which is use-restricted and not OSI-approved, so open_weights (3) like the rest of the Llama family.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "shows": "Llama 4 Community License; open weights; 86M/22M; prompt-injection/jailbreak classifier; ~90,441 downloads/month, 149 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "86M variant ~90.4K HF downloads/month (June 2026); widely used as a low-latency injection filter, including inside LlamaFirewall.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "shows": "~90,441 downloads/month, 149 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "injection detection + latency", "value": "Dedicated prompt-injection/jailbreak detection at 99.8% AUC and 97.5% recall at 1% FPR (English), 8 languages, sub-100ms latency; narrow but best-in-class at its job.", "confidence": "medium", "note": "Specialized injection/jailbreak classifier with strong accuracy and very low latency; complements full-taxonomy guards.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M", "shows": "99.8% AUC, 97.5% recall @1% FPR; 8 languages; sub-100ms", "accessed": "2026-06-29" } ] }, "maturity": 4.0, "mature": false, "version_note": "Llama Prompt Guard 2 (86M / 22M); Llama 4 Community License (not OSI). 99.8% AUC, 97.5% recall at 1% FPR on English jailbreaks; 8 languages. 86M ~90.4K HF downloads/month, 149 likes (June 2026). Verified live June 2026." }, { "product": "LlamaFirewall", "org": "Meta", "type": "software", "description": "Meta's open, system-level guardrail framework for AI agents (part of PurpleLlama). It composes layered defenses: Prompt Guard 2 (jailbreak/injection detection), Agent Alignment Checks (detecting goal hijacking), and CodeShield (filtering insecure generated code), into a real-time firewall around agent execution.", "openness": { "score": 5, "class": "open_source", "components": "framework:open(LlamaFirewall + CodeShield code, MIT-licensed in PurpleLlama);self-host:yes;note:bundled guard models (Prompt Guard 2, Llama Guard) carry the Llama Community License", "confidence": "high", "note": "The firewall framework and CodeShield are open-source (MIT) and self-hostable; it orchestrates Meta's Llama-licensed guard models. Scored on the framework, which is genuinely open source (5).", "sources": [ { "url": "https://github.com/meta-llama/PurpleLlama", "shows": "PurpleLlama: MIT evals + Llama-licensed safeguard models; ~4.2k stars; LlamaFirewall, Prompt Guard, CodeShield components", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "PurpleLlama ~4.2K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/meta-llama/PurpleLlama", "shows": "~4.2k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "layered agent defenses", "value": "Layered, agent-focused defenses: jailbreak/injection detection (Prompt Guard 2), agent-alignment/goal-hijack checks, and insecure-code filtering (CodeShield) in one real-time firewall.", "confidence": "medium", "note": "One of the few guardrail systems aimed specifically at agent security, with alignment checks and code filtering beyond I/O moderation.", "sources": [ { "url": "https://github.com/meta-llama/PurpleLlama", "shows": "Prompt Guard 2 + Agent Alignment Checks + CodeShield", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "LlamaFirewall, in meta-llama/PurpleLlama (~4.2K stars). PurpleLlama licensing is mixed: evals/benchmarks MIT, safeguard models under the Llama Community License. Components: Prompt Guard 2, Agent Alignment Checks, CodeShield. Verified live June 2026." }, { "product": "HarmBench", "org": "Center for AI Safety", "type": "software", "description": "A standardized evaluation framework for automated red-teaming from the Center for AI Safety. It runs a common pipeline across many attack methods and target models so red-team attacks and defenses can be compared rigorously and co-developed.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public;self-host:yes;service:none", "confidence": "high", "note": "Fully open source: MIT-licensed standardized red-teaming evaluation framework, self-hostable.", "sources": [ { "url": "https://github.com/centerforaisafety/HarmBench", "shows": "MIT; standardized automated red-teaming eval; 18 attacks x 33 target models; ~993 stars", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "1K-10K", "signal_type": "stars_fallback", "confidence": "low", "note": "~1K GitHub stars (June 2026); stars-only proxy. Widely cited as the standardized red-teaming benchmark in research.", "sources": [ { "url": "https://github.com/centerforaisafety/HarmBench", "shows": "~993 stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "standardized eval coverage", "value": "A standardized pipeline spanning 18 automated attack methods and 33 target models, enabling rigorous attack/defense comparison and co-development.", "confidence": "medium", "note": "The reference standardized harness for comparing red-team attacks and defenses; research-grade breadth.", "sources": [ { "url": "https://github.com/centerforaisafety/HarmBench", "shows": "18 attack methods, 33 target models, standardized pipeline", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "HarmBench, MIT, Center for AI Safety; ~1K GitHub stars (June 2026). Standardized red-teaming eval across 18 attacks and 33 target models. Verified live June 2026." }, { "product": "Giskard", "org": "Giskard", "type": "software", "description": "An open-source Python library for testing and red-teaming LLM and agentic systems. Giskard Scan automatically probes applications for vulnerabilities (prompt injection, harmful output, hallucination), and Giskard Checks runs scenario-based evaluations, including multi-turn agent validation.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;self-host:yes;service:none(per the repo)", "confidence": "high", "note": "Fully open source: Apache-2.0 Python library, self-hostable; no commercial tier mentioned on the repo.", "sources": [ { "url": "https://github.com/Giskard-AI/giskard", "shows": "Apache-2.0; ~5.5k stars; automated scan + scenario checks; adversarial/injection testing", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~5.5K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/Giskard-AI/giskard", "shows": "~5.5k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "scan + checks coverage", "value": "Automated vulnerability scanning (injection, harmful output, hallucination) plus scenario-based and multi-turn agent evaluations.", "confidence": "medium", "note": "Combines automated red-team scanning with structured evaluation; broad coverage for both safety and reliability.", "sources": [ { "url": "https://github.com/Giskard-AI/giskard", "shows": "Scan + Checks; adversarial, injection, multi-turn agent validation", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "Giskard, Apache-2.0, Python; ~5.5K GitHub stars (June 2026). Scan (automated red-teaming) + Checks (scenario evals). Verified live June 2026." }, { "product": "DeepTeam", "org": "Confident AI", "type": "software", "description": "An open-source red-teaming framework for LLM systems from Confident AI. It simulates adversarial attacks (jailbreaking, prompt injection, multi-turn exploitation) with 50+ built-in vulnerability checks and 20+ attack methods aligned to OWASP Top 10 for LLMs, NIST AI RMF, and MITRE ATLAS, and ships production guardrails.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;self-host:yes;service:none", "confidence": "high", "note": "Fully open source: Apache-2.0 red-teaming framework, self-hostable.", "sources": [ { "url": "https://github.com/confident-ai/deepteam", "shows": "Apache-2.0; ~1.9k stars; 50+ vuln checks, 20+ attacks; OWASP/NIST/MITRE aligned", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "1K-10K", "signal_type": "stars_fallback", "confidence": "low", "note": "~1.9K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/confident-ai/deepteam", "shows": "~1.9k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "vuln + standards coverage", "value": "50+ vulnerability checks and 20+ attack methods (jailbreak, injection, multi-turn) mapped to OWASP Top 10 for LLMs, NIST AI RMF, and MITRE ATLAS, plus production guardrails.", "confidence": "medium", "note": "Strong standards alignment (OWASP/NIST/MITRE) and broad attack coverage; also ships runtime guardrails.", "sources": [ { "url": "https://github.com/confident-ai/deepteam", "shows": "50+ vulnerabilities, 20+ attacks, OWASP/NIST/MITRE", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "DeepTeam, Apache-2.0, Confident AI; ~1.9K GitHub stars (June 2026). 50+ vulnerability checks, 20+ attack methods; OWASP/NIST/MITRE aligned. Verified live June 2026." }, { "product": "Perspective API", "org": "Jigsaw", "type": "software", "description": "Jigsaw's (Google) free hosted API for scoring the toxicity of text (and related attributes like insult, threat, and identity attack), widely used by platforms and researchers for comment moderation since 2017. Closed, API-only; no weights or self-hosting.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary hosted API by Jigsaw/Google(no weights, no self-host);access:free with quota;scores:toxicity + related attributes", "confidence": "high", "note": "Closed, API-only hosted classifier; no weights or source. Long-standing and influential, but proprietary.", "sources": [ { "url": "https://perspectiveapi.com/", "shows": "Jigsaw (Google) free hosted toxicity-scoring API; no weights or self-host", "accessed": "2026-06-29" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "One of the most widely-adopted moderation APIs since 2017, used across major platforms and research; free with quota.", "sources": [ { "url": "https://perspectiveapi.com/", "shows": "widely-used free hosted moderation API since 2017", "accessed": "2026-06-29" } ] }, "capability": { "score": 3, "basis": "attribute coverage", "value": "Scores toxicity and related attributes (insult, threat, profanity, identity attack) for text, multilingual; established but a fixed attribute set.", "confidence": "low", "note": "Mature toxicity scoring with broad real-world use; narrower than full safety taxonomies and policy-conditioned models.", "sources": [ { "url": "https://perspectiveapi.com/", "shows": "toxicity + insult/threat/identity-attack scoring, multilingual", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "Perspective API by Jigsaw (Google); free hosted toxicity-scoring API, proprietary (no weights, no self-host). Long-standing comment-moderation API. Verified June 2026." }, { "product": "Llama Guard", "org": "Meta", "type": "model", "description": "Meta's input/output safety classifier built on Llama 3.1-8B, the de-facto open guardrail model. It labels both prompts and responses across 14 hazard categories (violent crime, child exploitation, hate, self-harm, code-interpreter abuse, and more) and supports eight languages. Widely used as the moderation layer in front of open chat and agent stacks.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(Llama-Guard-3-8B safetensors on HF);data:not-released;code:inference recipe public;license:Llama-3.1-Community(NOT OSI, >700M-MAU restriction)", "confidence": "medium", "note": "Open-weights safety classifier under Meta's Llama Community License, which carries use restrictions and is not OSI-approved, so it lands at the open_weights tier (3) like the rest of the Llama family rather than open_source.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Guard-3-8B", "shows": "Llama 3.1 Community License; open weights; 14 hazard categories; I/O classifier; ~237K downloads/month, 307 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "high", "note": "~237K Hugging Face downloads/month on the 8B variant (June 2026), the most-adopted open guardrail model; widely embedded as a moderation layer.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Guard-3-8B", "shows": "~237,092 downloads/month, 307 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "hazard coverage + benchmarks", "value": "14 hazard categories across input and output, 8 languages, MLCommons-aligned taxonomy; strong and widely-benchmarked safety classifier, the reference open baseline.", "confidence": "medium", "note": "Broad hazard taxonomy and multilingual coverage; the reference open guardrail others are measured against.", "sources": [ { "url": "https://huggingface.co/meta-llama/Llama-Guard-3-8B", "shows": "14 hazard categories, 8 languages, I/O moderation", "accessed": "2026-06-29" } ] }, "maturity": 4.0, "mature": false, "version_note": "Llama Guard 3 (8B), built on Llama 3.1; Llama 3.1 Community License (not OSI; >700M MAU restriction). ~237K HF downloads/month, 307 likes (June 2026). Verified live June 2026." }, { "product": "ShieldGemma", "org": "Google DeepMind", "type": "model", "description": "Google DeepMind's content-safety classifier built on Gemma 2, shipped in 2B/9B/27B sizes. It predicts policy violations across four harm categories (sexually explicit, dangerous content, hate, harassment) for both prompts and responses, as a yes/no classifier.", "openness": { "score": 3, "class": "open_weights", "components": "weights:open(shieldgemma 2B/9B/27B on HF);data:not-released;license:Gemma(NOT OSI, use-restricted)", "confidence": "medium", "note": "Open weights under the Gemma license, which is use-restricted and not OSI-approved (per the openness rubric, Gemma = open_weights, not open_source), so it sits at 3.", "sources": [ { "url": "https://huggingface.co/google/shieldgemma-2b", "shows": "Gemma license (not OSI); open weights; four harm categories; ~8,555 downloads/month, 124 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "2B variant ~8.5K HF downloads/month, 124 likes (June 2026); solid adoption across the three sizes.", "sources": [ { "url": "https://huggingface.co/google/shieldgemma-2b", "shows": "~8,555 downloads/month, 124 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 3, "basis": "hazard coverage", "value": "Four harm categories (sexual, dangerous, hate, harassment), yes/no policy classifier in 2B/9B/27B; narrower taxonomy than Llama Guard but multiple sizes.", "confidence": "medium", "note": "Competent four-category classifier with a size ladder; narrower hazard taxonomy than Llama Guard / Granite Guardian.", "sources": [ { "url": "https://huggingface.co/google/shieldgemma-2b", "shows": "four harm categories; 2B/9B/27B sizes", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "ShieldGemma (2B/9B/27B) on Gemma 2; Gemma license (NOT OSI). 2B variant ~8.5K HF downloads/month, 124 likes (June 2026). Verified live June 2026." }, { "product": "Granite Guardian", "org": "IBM", "type": "model", "description": "IBM's open guardrail model family for detecting risks in prompts and responses, plus RAG-specific checks. It covers harm, social bias, jailbreaking, violence, profanity and sexual content, and adds hallucination checks for retrieval pipelines (context relevance, groundedness, answer relevance).", "openness": { "score": 4, "class": "open_weights", "components": "weights:open(granite-guardian-3.2-5b on HF);data:not-released;license:Apache-2.0(OSI)", "confidence": "medium", "note": "Open weights under Apache-2.0 (permissive, OSI), but the training data is not released, so it stops short of the open_source (5) tier; a strong open_weights release at 4.", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-5b", "shows": "Apache-2.0; open weights; harm/bias/jailbreak + RAG groundedness checks; ~1,377 downloads/month, 14 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "usage_volume", "confidence": "medium", "note": "~1.4K HF downloads/month on the 3.2-5B variant (June 2026); cumulative across the Guardian family is higher. Enterprise-leaning distribution via watsonx.", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-5b", "shows": "~1,377 downloads/month, 14 likes (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "risk coverage + RAG checks", "value": "Harm, bias, jailbreak, violence, profanity, sexual content plus RAG hallucination checks (context relevance, groundedness, answer relevance); broader than pure I/O moderation.", "confidence": "medium", "note": "One of the broadest open guardrails: standard harm taxonomy plus RAG-specific groundedness/hallucination checks.", "sources": [ { "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-5b", "shows": "harm/bias/jailbreak + RAG groundedness/answer-relevance checks", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "Granite Guardian 3.2 (5B, distilled from the 8B); Apache-2.0 open weights. ~1.4K HF downloads/month, 14 likes (June 2026). Verified live June 2026." }, { "product": "gpt-oss-safeguard", "org": "OpenAI", "type": "model", "description": "OpenAI's open safety reasoning model, built on gpt-oss and released with Hugging Face and ROOST. Rather than emitting a fixed label, it reasons over a safety policy you supply and returns a reasoned decision, with configurable reasoning effort. Shipped in 20b and 120b variants under Apache 2.0.", "openness": { "score": 4, "class": "open_weights", "components": "weights:open(gpt-oss-safeguard-20b/120b on HF);data:not-released;license:Apache-2.0(OSI)", "confidence": "medium", "note": "Open weights under Apache-2.0 (OSI), training data not released, so open_weights (4) rather than open_source. Notable as a policy-conditioned safety reasoning model rather than a fixed classifier.", "sources": [ { "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b", "shows": "Apache-2.0; open weights; policy-conditioned safety reasoning; 20b/120b; ~131K downloads/month, 235 likes", "accessed": "2026-06-29" } ], "bucket": "open-ish" }, "adoption": { "level": 4, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "high", "note": "20b variant ~131K HF downloads/month, 235 likes, 100+ Spaces (June 2026); strong early adoption for a recent release.", "sources": [ { "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b", "shows": "~131,038 downloads/month, 235 likes, 100+ Spaces (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "reasoning + configurable policy", "value": "Policy-conditioned safety reasoning (bring-your-own-policy), reasoned decisions rather than just scores, configurable reasoning effort; 20b and 120b.", "confidence": "medium", "note": "A different shape from fixed classifiers: reasons over an arbitrary supplied policy, which generalizes to novel categories at some latency cost.", "sources": [ { "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b", "shows": "reasoned decisions, configurable reasoning effort, bring-your-own-policy", "accessed": "2026-06-29" } ] }, "maturity": 4.0, "mature": false, "version_note": "gpt-oss-safeguard (20b / 120b), Apache-2.0 open weights; released by OpenAI with Hugging Face and ROOST, distributed via the ROOST Model Community (RMC). 20b ~131K HF downloads/month, 235 likes (June 2026). Verified live June 2026." }, { "product": "NeMo Guardrails", "org": "NVIDIA", "type": "software", "description": "NVIDIA's open-source toolkit for adding programmable guardrails around LLM applications. It wraps input/output moderation, dialog control, retrieval filtering, and tool-use control through a small policy language (Colang), and works across multiple LLM providers. Deployable as a Python library, CLI, Docker, or standalone server.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public(full Python implementation);self-host:yes(library/CLI/Docker/server);service:none", "confidence": "high", "note": "Fully open source: Apache-2.0, complete source, self-hostable with no proprietary tier.", "sources": [ { "url": "https://github.com/NVIDIA/NeMo-Guardrails", "shows": "Apache-2.0; full Python source; self-hostable as library/CLI/Docker/server; ~6.6k stars", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~6.6K GitHub stars (June 2026); stars-only proxy, capped per methodology; broadly referenced guardrail framework.", "sources": [ { "url": "https://github.com/NVIDIA/NeMo-Guardrails", "shows": "~6.6k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "feature coverage", "value": "Programmable rails for I/O moderation, dialog/topic control, retrieval filtering, and tool-use control via Colang; multi-provider.", "confidence": "medium", "note": "One of the most feature-complete open guardrail orchestration layers; the Colang policy language adds flexibility beyond simple scanners.", "sources": [ { "url": "https://github.com/NVIDIA/NeMo-Guardrails", "shows": "I/O moderation, dialog control, retrieval filtering, tool-use control", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "NeMo Guardrails, Apache-2.0, Python; ~6.6K GitHub stars (June 2026). Fully open source and self-hostable. Verified live June 2026." }, { "product": "Guardrails AI", "org": "Guardrails AI", "type": "software", "description": "An open framework for input/output guards on LLM applications, with a Hub of pre-built validators (PII, toxicity, prompt injection, format/structure). Wraps model calls in guards that detect risks and enforce structured output, self-hostable as a Python library.", "openness": { "score": 4, "class": "open_core", "components": "license:Apache-2.0(OSI) for the framework;source:public;hub:Guardrails Hub validator registry;commercial:hosted/enterprise tier", "confidence": "medium", "note": "Apache-2.0 framework that is fully self-hostable, with a Hub of validators and a commercial tier on top; classed open_core alongside other OSS-core-plus-commercial tools.", "sources": [ { "url": "https://github.com/guardrails-ai/guardrails", "shows": "Apache-2.0 framework; ~7.1k stars; Guardrails Hub validators; commercial offerings", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~7.1K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/guardrails-ai/guardrails", "shows": "~7.1k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "validator coverage", "value": "Input/output guards plus a Hub of pre-built validators (PII, toxicity, injection, structure) and structured-output enforcement.", "confidence": "medium", "note": "Broad validator ecosystem via the Hub; strong on structured-output enforcement in addition to safety checks.", "sources": [ { "url": "https://github.com/guardrails-ai/guardrails", "shows": "I/O guards, validator Hub, structured-output enforcement", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "Guardrails AI framework, Apache-2.0, ~7.1K GitHub stars (June 2026); plus Guardrails Hub (validator registry) and commercial offerings. Verified live June 2026." }, { "product": "LLM Guard", "org": "Protect AI", "type": "software", "description": "An open security toolkit for LLM interactions from Protect AI, providing input and output scanners for prompt-injection, PII/data-leakage, toxicity, and harmful-content detection and sanitization. Self-hostable as a Python library.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(full Python implementation);self-host:yes;service:none(the OSS toolkit; Protect AI's platform is separate)", "confidence": "high", "note": "Fully open source: MIT-licensed, complete source, self-hostable. (Protect AI's commercial platform is a separate product; this is the standalone OSS toolkit.)", "sources": [ { "url": "https://github.com/protectai/llm-guard", "shows": "MIT; full Python source; input/output scanners; ~3.1k stars", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~3.1K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/protectai/llm-guard", "shows": "~3.1k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 3, "basis": "scanner coverage", "value": "Input/output scanners for prompt injection, PII/data leakage, toxicity, and harmful content, with sanitization.", "confidence": "medium", "note": "Solid set of I/O scanners; lighter-weight orchestration than NeMo Guardrails but easy to drop in.", "sources": [ { "url": "https://github.com/protectai/llm-guard", "shows": "input/output scanners: injection, PII, toxicity, harmful content", "accessed": "2026-06-29" } ] }, "maturity": 2.5, "mature": false, "version_note": "LLM Guard, MIT, Python; ~3.1K GitHub stars (June 2026). Maintained by Protect AI (acquired by Palo Alto Networks). Verified live June 2026." }, { "product": "PyRIT", "org": "Microsoft", "type": "software", "description": "The Python Risk Identification Tool from the Microsoft AI Red Team: an open framework that automates red-teaming of generative AI systems, orchestrating adversarial prompts, attack strategies, and scoring to surface harms and vulnerabilities at scale.", "openness": { "score": 5, "class": "open_source", "components": "license:MIT(OSI);source:public(full Python framework);self-host:yes;service:none", "confidence": "high", "note": "Fully open source: MIT-licensed automation framework, self-hostable, no proprietary tier.", "sources": [ { "url": "https://github.com/microsoft/PyRIT", "shows": "MIT; open red-teaming framework from the Microsoft AI Red Team", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "Widely-used AI red-teaming framework from the Microsoft AI Red Team; stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/microsoft/PyRIT", "shows": "open framework, Microsoft AI Red Team; relocated to microsoft/PyRIT", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "red-team coverage", "value": "Automated adversarial prompting, attack-strategy orchestration, and scoring to identify harms across providers.", "confidence": "medium", "note": "A leading open red-teaming harness; strong automation and scoring, multi-provider.", "sources": [ { "url": "https://github.com/microsoft/PyRIT", "shows": "automated adversarial prompting, attack orchestration, scoring", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "PyRIT, MIT, maintained by the Microsoft AI Red Team; repo now at github.com/microsoft/PyRIT. Verified live June 2026." }, { "product": "garak", "org": "NVIDIA", "type": "software", "description": "An open LLM vulnerability scanner maintained by NVIDIA. It probes models for hallucination, data leakage, prompt injection, misinformation, toxicity, and jailbreaks across many providers (Hugging Face, OpenAI, Bedrock, and others), the nmap of LLM security testing.", "openness": { "score": 5, "class": "open_source", "components": "license:Apache-2.0(OSI);source:public;self-host:yes;service:none", "confidence": "high", "note": "Fully open source: Apache-2.0 scanner, self-hostable, no proprietary tier.", "sources": [ { "url": "https://github.com/NVIDIA/garak", "shows": "Apache-2.0; ~8.3k stars; probes for injection, leakage, jailbreaks, toxicity across providers", "accessed": "2026-06-29" } ], "bucket": "open" }, "adoption": { "level": 3, "reach": "10K-100K", "signal_type": "stars_fallback", "confidence": "low", "note": "~8.3K GitHub stars (June 2026); stars-only proxy, capped per methodology.", "sources": [ { "url": "https://github.com/NVIDIA/garak", "shows": "~8.3k stars (June 2026)", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "probe coverage", "value": "Large library of vulnerability probes (hallucination, leakage, injection, misinformation, toxicity, jailbreaks) across many model providers.", "confidence": "medium", "note": "Broadest open library of LLM vulnerability probes; multi-provider coverage.", "sources": [ { "url": "https://github.com/NVIDIA/garak", "shows": "probes: hallucination, leakage, injection, misinformation, toxicity, jailbreaks", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "garak LLM vulnerability scanner, Apache-2.0, maintained by NVIDIA; ~8.3K GitHub stars (June 2026). Verified live June 2026." }, { "product": "OpenAI Moderation API", "org": "OpenAI", "type": "software", "description": "OpenAI's hosted moderation endpoint, a free API that classifies text (and images) against OpenAI's content-policy categories (hate, harassment, self-harm, sexual, violence, and more). Closed, API-only; no weights or self-hosting.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary hosted API(no weights, no self-host);access:free moderation endpoint", "confidence": "high", "note": "Closed, API-only hosted classifier; no weights or source.", "sources": [ { "url": "https://platform.openai.com/docs/guides/moderation", "shows": "free hosted moderation API; OpenAI content-policy categories; no weights", "accessed": "2026-06-29" } ], "bucket": "closed" }, "adoption": { "level": 4, "reach": "1M-10M", "signal_type": "usage_volume", "confidence": "medium", "note": "Free, default moderation endpoint for the large OpenAI developer base; widely used as a first-line filter.", "sources": [ { "url": "https://platform.openai.com/docs/guides/moderation", "shows": "free hosted moderation endpoint", "accessed": "2026-06-29" } ] }, "capability": { "score": 3, "basis": "category coverage", "value": "Multi-category text (and image) moderation aligned to OpenAI's content policy; convenient but fixed taxonomy.", "confidence": "low", "note": "Convenient broad-category moderation; fixed policy, no customization vs policy-conditioned models.", "sources": [ { "url": "https://platform.openai.com/docs/guides/moderation", "shows": "multi-category text/image moderation", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "OpenAI Moderation API; free hosted endpoint, proprietary (no weights, no self-host). Verified live June 2026." }, { "product": "Azure AI Content Safety", "org": "Microsoft Azure", "type": "software", "description": "Microsoft's managed content-safety service on Azure: hosted APIs for text and image moderation across hate, sexual, violence and self-harm categories, plus prompt-shield (jailbreak/injection) and groundedness detection. Closed, managed service.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary managed Azure service(no self-host);features:text/image moderation, prompt shields, groundedness", "confidence": "high", "note": "Closed managed cloud service; no weights or source.", "sources": [ { "url": "https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety", "shows": "managed content-safety APIs: moderation, prompt shields, groundedness; proprietary", "accessed": "2026-06-29" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "low", "note": "Default content-safety layer for Azure OpenAI and Azure AI customers; broad enterprise reach.", "sources": [ { "url": "https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety", "shows": "managed service for the Azure AI customer base", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "feature coverage", "value": "Text and image moderation, prompt-shield jailbreak/injection detection, and groundedness detection in one managed service.", "confidence": "low", "note": "Broad managed feature set (moderation + prompt shields + groundedness); strong coverage, closed.", "sources": [ { "url": "https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety", "shows": "moderation + prompt shields + groundedness", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "Azure AI Content Safety; managed Azure service (text/image moderation, prompt shields, groundedness). Proprietary, no self-host. Verified live June 2026." }, { "product": "Amazon Bedrock Guardrails", "org": "Amazon Web Services", "type": "software", "description": "Managed guardrails for Amazon Bedrock: configurable policies for denied topics, content filters, PII redaction, word filters, and contextual-grounding checks, applied to any Bedrock model. Closed, managed service.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary managed AWS service(no self-host);features:denied topics, content filters, PII redaction, contextual grounding", "confidence": "high", "note": "Closed managed cloud service; no weights or source.", "sources": [ { "url": "https://aws.amazon.com/bedrock/guardrails/", "shows": "managed guardrails: denied topics, content filters, PII redaction, grounding; proprietary", "accessed": "2026-06-29" } ], "bucket": "closed" }, "adoption": { "level": 3, "reach": "100K-1M", "signal_type": "usage_volume", "confidence": "low", "note": "Default guardrail layer for the Amazon Bedrock customer base; broad enterprise reach.", "sources": [ { "url": "https://aws.amazon.com/bedrock/guardrails/", "shows": "managed guardrails across Bedrock models", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "feature coverage", "value": "Denied topics, content filters, PII redaction, word filters, and contextual-grounding checks across any Bedrock model.", "confidence": "low", "note": "Broad configurable managed guardrail set; strong coverage, closed.", "sources": [ { "url": "https://aws.amazon.com/bedrock/guardrails/", "shows": "denied topics, filters, PII redaction, contextual grounding", "accessed": "2026-06-29" } ] }, "maturity": 3.5, "mature": false, "version_note": "Amazon Bedrock Guardrails; managed AWS service (denied topics, content filters, PII redaction, grounding checks). Proprietary, no self-host. Verified live June 2026." }, { "product": "Lakera Guard", "org": "Lakera", "type": "software", "description": "Lakera's hosted AI security product: a real-time API defending LLM applications against prompt injection, jailbreaks, data leakage, and content risks. Closed/SaaS; Lakera was acquired by Check Point in 2025.", "openness": { "score": 1, "class": "closed", "components": "service:proprietary SaaS API(no self-host);note:vendor Lakera acquired by Check Point Nov 2025", "confidence": "high", "note": "Closed SaaS security product; no weights or source. (Lakera also runs the public Gandalf prompt-injection game, but Guard itself is proprietary.)", "sources": [ { "url": "https://www.lakera.ai/", "shows": "proprietary AI security / prompt-injection defense SaaS; Lakera acquired by Check Point (2025)", "accessed": "2026-06-29" } ], "bucket": "closed" }, "adoption": { "level": 2, "reach": "10K-100K", "signal_type": "reported_traction", "confidence": "low", "note": "Established prompt-injection-defense vendor with notable enterprise customers; acquired by Check Point in 2025. No public usage counts.", "sources": [ { "url": "https://www.lakera.ai/", "shows": "commercial AI security product; Check Point acquisition 2025", "accessed": "2026-06-29" } ] }, "capability": { "score": 4, "basis": "defense coverage", "value": "Real-time prompt-injection, jailbreak, data-leakage, and content-risk defense for LLM apps.", "confidence": "low", "note": "Category-defining prompt-injection defense; strong coverage, now part of Check Point's platform.", "sources": [ { "url": "https://www.lakera.ai/", "shows": "real-time prompt-injection / jailbreak / data-leakage defense", "accessed": "2026-06-29" } ] }, "maturity": 3.0, "mature": false, "version_note": "Lakera Guard; proprietary SaaS API (prompt-injection/jailbreak defense). Lakera acquired by Check Point (completed Nov 2025). Verified live June 2026." } ] } }, "order": [ "base_pretrained", "finetuned_chat", "inference_code", "finetuning_code", "evaluation_code", "benchmark_eval_data", "training_synthetic_datasets", "orchestration_agents", "ui_api", "telemetry_observability", "agent_tools_protocols", "ml_frameworks", "deployment", "edge_hardware", "safeguards" ], "n_total": 452, "generated": "2026-06-29", "long_tail": { "counts": { "repos": 15375, "models": 6428, "packages": 2823, "total": 24626, "scored": 452, "overlap": 226, "scored_outside": 226, "uncategorized": 24400, "universe": 24852 }, "top": [ { "name": "ultraworkers/claw-code", "type": "repo", "usage_label": "188,924 stars", "description": "The repo is finally unlocked. enjoy the party! The fastest repo in history to surpass 100K stars ⭐. Join Discord: https:" }, { "name": "obra/superpowers", "type": "repo", "usage_label": "170,701 stars", "description": "An agentic skills framework & software development methodology that works." }, { "name": "affaan-m/everything-claude-code", "type": "repo", "usage_label": "168,837 stars", "description": "The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development f" }, { "name": "AUTOMATIC1111/stable-diffusion-webui", "type": "repo", "usage_label": "162,653 stars", "description": "Stable Diffusion web UI" }, { "name": "f/prompts.chat", "type": "repo", "usage_label": "160,986 stars", "description": "f.k.a. Awesome ChatGPT Prompts. Share, discover, and collect prompts from the community. Free and open source — self-hos" }, { "name": "Snailclimb/JavaGuide", "type": "repo", "usage_label": "155,279 stars", "description": "Java 面试 & 后端通用面试指南，覆盖计算机基础、数据库、分布式、高并发、系统设计与 AI 应用开发" }, { "name": "cross-encoder/ms-marco-MiniLM-L6-v2", "type": "model", "usage_label": "31.2M downloads", "description": "cross-encoder · text-ranking" }, { "name": "BAAI/bge-small-en-v1.5", "type": "model", "usage_label": "27.9M downloads", "description": "BAAI · feature-extraction" }, { "name": "Falconsai/nsfw_image_detection", "type": "model", "usage_label": "27.1M downloads", "description": "Falconsai · image-classification" }, { "name": "amazon/chronos-2", "type": "model", "usage_label": "20.9M downloads", "description": "amazon · time-series-forecasting" }, { "name": "BAAI/bge-m3", "type": "model", "usage_label": "18.7M downloads", "description": "BAAI · sentence-similarity" }, { "name": "FacebookAI/roberta-base", "type": "model", "usage_label": "17.9M downloads", "description": "FacebookAI · fill-mask" }, { "name": "FacebookAI/xlm-roberta-base", "type": "model", "usage_label": "17.3M downloads", "description": "FacebookAI · fill-mask" }, { "name": "FacebookAI/roberta-large", "type": "model", "usage_label": "16.3M downloads", "description": "FacebookAI · fill-mask" }, { "name": "Bingsu/adetailer", "type": "model", "usage_label": "14.8M downloads", "description": "Bingsu · model" }, { "name": "colbert-ir/colbertv2.0", "type": "model", "usage_label": "14.1M downloads", "description": "colbert-ir · model" }, { "name": "BAAI/bge-large-en-v1.5", "type": "model", "usage_label": "12.9M downloads", "description": "BAAI · feature-extraction" }, { "name": "apple/DFN5B-CLIP-ViT-H-14-378", "type": "model", "usage_label": "12.0M downloads", "description": "apple · model" }, { "name": "huggingface/documentation-images", "type": "dataset", "usage_label": "2.6M downloads", "description": "huggingface dataset" }, { "name": "banned-historical-archives/banned-historical-archives", "type": "dataset", "usage_label": "1.8M downloads", "description": "banned-historical-archives dataset" }, { "name": "ayuo/hd_tmp", "type": "dataset", "usage_label": "1.4M downloads", "description": "ayuo dataset" }, { "name": "mteb/results", "type": "dataset", "usage_label": "1.3M downloads", "description": "mteb dataset" }, { "name": "Salesforce/wikitext", "type": "dataset", "usage_label": "1.2M downloads", "description": "Salesforce dataset" }, { "name": "mrmrx/CADS-dataset", "type": "dataset", "usage_label": "1.2M downloads", "description": "mrmrx dataset" }, { "name": "xlangai/ubuntu_osworld_file_cache", "type": "dataset", "usage_label": "1.1M downloads", "description": "xlangai dataset" }, { "name": "LLM360/TxT360", "type": "dataset", "usage_label": "804k downloads", "description": "LLM360 dataset" }, { "name": "fineinstructions/fineinstructions_nemotron", "type": "dataset", "usage_label": "799k downloads", "description": "fineinstructions dataset" }, { "name": "mlfoundations/MINT-1T-PDF-CC-2024-18", "type": "dataset", "usage_label": "691k downloads", "description": "mlfoundations dataset" }, { "name": "NTU-NLP-sg/xCodeEval", "type": "dataset", "usage_label": "672k downloads", "description": "NTU-NLP-sg dataset" }, { "name": "scipy", "type": "package", "usage_label": "3.0B downloads", "description": "PyPI package" }, { "name": "uvicorn", "type": "package", "usage_label": "2.6B downloads", "description": "PyPI package" }, { "name": "fastapi", "type": "package", "usage_label": "2.5B downloads", "description": "PyPI package" }, { "name": "scikit-learn", "type": "package", "usage_label": "1.8B downloads", "description": "PyPI package" }, { "name": "openai", "type": "package", "usage_label": "1.7B downloads", "description": "PyPI package" }, { "name": "nltk", "type": "package", "usage_label": "518.2M downloads", "description": "PyPI package" }, { "name": "anthropic", "type": "package", "usage_label": "485.0M downloads", "description": "PyPI package" }, { "name": "xgboost", "type": "package", "usage_label": "412.1M downloads", "description": "PyPI package" }, { "name": "opencv-python", "type": "package", "usage_label": "395.2M downloads", "description": "PyPI package" } ] } }

Explore the full dataset