mcp-core.php
2 days ago
mcp-oauth.php
23 hours ago
mcp-rest.php
1 month ago
mcp.conf
1 year ago
mcp.js
9 months ago
mcp.md
9 months ago
mcp.php
23 hours ago
model-audit.php
2 days ago
workspace-mock.html
2 days ago
wpai-connectors.php
2 months ago
wpai-gateway-availability.php
3 months ago
wpai-gateway-directory.php
3 months ago
wpai-gateway-image-model.php
3 months ago
wpai-gateway-model.php
3 months ago
wpai-gateway-providers.php
3 months ago
wpai-gateway.php
3 months ago
model-audit.php
144 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Model coverage audit for the dynamic-model engines (Google, OpenRouter, …). |
| 4 | * |
| 5 | * Those engines build their model list at "Refresh Models" time from the provider's own |
| 6 | * API, then guess each model's nice name, features, tags and tools with a pile of |
| 7 | * regexes. When a provider ships a new naming shape those regexes quietly miss, and the |
| 8 | * model either vanishes from the dropdown or shows up without the capabilities it has. |
| 9 | * That is invisible until a user complains, so this audits it. |
| 10 | * |
| 11 | * Run it from the WordPress root: |
| 12 | * wp eval-file labs/model-audit.php # every dynamic engine |
| 13 | * wp eval-file labs/model-audit.php google # one engine type |
| 14 | * |
| 15 | * Findings are heuristics, not verdicts. Read them, then check the provider's docs |
| 16 | * before touching the regexes in classes/engines/<engine>.php. |
| 17 | */ |
| 18 | |
| 19 | global $mwai; |
| 20 | $core = $mwai->core; |
| 21 | $only = $args[0] ?? null; |
| 22 | |
| 23 | // Engines whose models come from the provider API rather than constants/models.php. |
| 24 | $dynamic_types = [ 'google', 'openrouter' ]; |
| 25 | $issues_total = 0; |
| 26 | |
| 27 | foreach ( (array) $core->get_option( 'ai_envs' ) as $env ) { |
| 28 | $type = $env['type'] ?? ''; |
| 29 | if ( !in_array( $type, $dynamic_types, true ) ) { |
| 30 | continue; |
| 31 | } |
| 32 | if ( $only && $type !== $only ) { |
| 33 | continue; |
| 34 | } |
| 35 | if ( empty( $env['apikey'] ) ) { |
| 36 | echo "SKIP {$type} ({$env['name']}): no API key\n"; |
| 37 | continue; |
| 38 | } |
| 39 | |
| 40 | echo "\n=== {$type} / {$env['name']} ===\n"; |
| 41 | |
| 42 | try { |
| 43 | $engine = Meow_MWAI_Engines_Factory::get( $core, $env['id'] ); |
| 44 | $models = $engine->retrieve_models(); |
| 45 | } |
| 46 | catch ( Exception $e ) { |
| 47 | echo " ERROR: " . $e->getMessage() . "\n"; |
| 48 | continue; |
| 49 | } |
| 50 | |
| 51 | echo " models classified: " . count( $models ) . "\n"; |
| 52 | $issues = []; |
| 53 | |
| 54 | // 1. Duplicate display names. Google ships preview and GA variants of the same model |
| 55 | // and the name formatter strips the suffix from both, which makes the dropdown |
| 56 | // ambiguous. Anything still colliding here needs disambiguating. |
| 57 | $by_name = []; |
| 58 | foreach ( $models as $m ) { |
| 59 | $by_name[$m['name']][] = $m['model']; |
| 60 | } |
| 61 | foreach ( $by_name as $name => $ids ) { |
| 62 | if ( count( $ids ) > 1 ) { |
| 63 | $issues[] = "duplicate name \"{$name}\" for: " . implode( ', ', $ids ); |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // 2. Names that clearly fell through the formatter: still carrying a raw id, or a |
| 68 | // lowercased fragment that should have become a proper word. |
| 69 | foreach ( $models as $m ) { |
| 70 | if ( $m['name'] === $m['model'] ) { |
| 71 | $issues[] = "unformatted name for {$m['model']} (name === id)"; |
| 72 | } |
| 73 | else if ( preg_match( '/(preview|latest|exp|[0-9]{4}-[0-9]{2}|customtools|nano-banana)/i', $m['name'] ) |
| 74 | && !preg_match( '/\(/', $m['name'] ) ) { |
| 75 | $issues[] = "raw id fragment left in name \"{$m['name']}\" ({$m['model']})"; |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | // 3. Capability mismatches: the id advertises something the features do not. |
| 80 | // These are the exact shapes that were silently wrong before 2026-07-26. |
| 81 | $expectations = [ |
| 82 | // id pattern => feature that must be present |
| 83 | '/-image(-preview)?$|nano-banana/' => 'image-generation', |
| 84 | '/embedding/' => 'embedding', |
| 85 | '/(native-audio|-live-)/' => 'realtime', |
| 86 | '/(veo|video)/' => 'video-generation', |
| 87 | ]; |
| 88 | foreach ( $models as $m ) { |
| 89 | $features = (array) ( $m['features'] ?? [] ); |
| 90 | foreach ( $expectations as $pattern => $needed ) { |
| 91 | if ( preg_match( $pattern, $m['model'] ) && !in_array( $needed, $features, true ) ) { |
| 92 | $issues[] = "{$m['model']} looks like '{$needed}' but features are: " |
| 93 | . ( implode( ',', $features ) ?: '(none)' ); |
| 94 | } |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | // 4. Models with no usable feature at all: they will render in the dropdown and then |
| 99 | // fail at query time. |
| 100 | foreach ( $models as $m ) { |
| 101 | if ( empty( $m['features'] ) ) { |
| 102 | $issues[] = "{$m['model']} has no features"; |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // 5. Anything the provider exposes that never made it into the list. Only the engine |
| 107 | // knows the raw payload, so this compares against the provider call where possible. |
| 108 | if ( $type === 'google' ) { |
| 109 | $res = wp_remote_get( |
| 110 | 'https://generativelanguage.googleapis.com/v1beta/models?pageSize=200&key=' . $env['apikey'], |
| 111 | [ 'timeout' => 30 ] |
| 112 | ); |
| 113 | if ( !is_wp_error( $res ) ) { |
| 114 | $body = json_decode( wp_remote_retrieve_body( $res ), true ); |
| 115 | $classified = array_column( $models, 'model' ); |
| 116 | foreach ( (array) ( $body['models'] ?? [] ) as $raw ) { |
| 117 | $id = str_replace( 'models/', '', $raw['name'] ); |
| 118 | $methods = (array) ( $raw['supportedGenerationMethods'] ?? [] ); |
| 119 | // Only care about ids we would plausibly want to offer. |
| 120 | $wanted = preg_match( '/^(gemini|nano-banana|imagen|veo)/', $id ) |
| 121 | && ( in_array( 'generateContent', $methods, true ) || in_array( 'embedContent', $methods, true ) ); |
| 122 | // Deliberate exclusions: dated snapshots, TTS, robotics. |
| 123 | $excluded = preg_match( '/-(tts|robotics)|(preview|exp)-\d{2}-\d{2,4}$|-\d{8}$/', $id ); |
| 124 | if ( $wanted && !$excluded && !in_array( $id, $classified, true ) ) { |
| 125 | $issues[] = "provider exposes {$id} (" . implode( ',', $methods ) . ") but it was dropped"; |
| 126 | } |
| 127 | } |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | if ( empty( $issues ) ) { |
| 132 | echo " no issues found\n"; |
| 133 | } |
| 134 | else { |
| 135 | $issues = array_values( array_unique( $issues ) ); |
| 136 | $issues_total += count( $issues ); |
| 137 | foreach ( $issues as $i ) { |
| 138 | echo " ISSUE: {$i}\n"; |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | echo "\nTOTAL ISSUES: {$issues_total}\n"; |
| 144 |