mcp-core.php
2 weeks ago
mcp-oauth.php
2 weeks ago
mcp-rest.php
2 months ago
mcp.conf
1 year ago
mcp.php
3 weeks ago
model-audit.php
3 days ago
workspace-mock.html
1 month ago
wpai-connectors.php
3 months ago
wpai-gateway-availability.php
4 months ago
wpai-gateway-directory.php
4 months ago
wpai-gateway-image-model.php
4 months ago
wpai-gateway-model.php
4 months ago
wpai-gateway-providers.php
4 months ago
wpai-gateway.php
4 months ago
model-audit.php
155 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Model coverage audit for the dynamic-model engines (Google, OpenRouter, …). |
| 4 | * |
| 5 | * Those engines build their model list at "Refresh Models" time from the provider's own |
| 6 | * API, then guess each model's nice name, features, tags and tools with a pile of |
| 7 | * regexes. When a provider ships a new naming shape those regexes quietly miss, and the |
| 8 | * model either vanishes from the dropdown or shows up without the capabilities it has. |
| 9 | * That is invisible until a user complains, so this audits it. |
| 10 | * |
| 11 | * Run it from the WordPress root: |
| 12 | * wp eval-file labs/model-audit.php # every dynamic engine |
| 13 | * wp eval-file labs/model-audit.php google # one engine type |
| 14 | * |
| 15 | * Findings are heuristics, not verdicts. Read them, then check the provider's docs |
| 16 | * before touching the regexes in classes/engines/<engine>.php. |
| 17 | */ |
| 18 | |
| 19 | global $mwai; |
| 20 | $core = $mwai->core; |
| 21 | $only = $args[0] ?? null; |
| 22 | |
| 23 | // Engines whose models come from the provider API rather than constants/models.php. |
| 24 | $dynamic_types = [ 'google', 'openrouter' ]; |
| 25 | |
| 26 | // Engines where WE build the display name from the raw id. OpenRouter hands us a curated |
| 27 | // name ("Google: Gemini 3 Flash Preview") and we pass it straight through, so the name |
| 28 | // heuristics below would flag every one of its models forever. |
| 29 | $formats_own_names = [ 'google' ]; |
| 30 | |
| 31 | $issues_total = 0; |
| 32 | |
| 33 | foreach ( (array) $core->get_option( 'ai_envs' ) as $env ) { |
| 34 | $type = $env['type'] ?? ''; |
| 35 | if ( !in_array( $type, $dynamic_types, true ) ) { |
| 36 | continue; |
| 37 | } |
| 38 | if ( $only && $type !== $only ) { |
| 39 | continue; |
| 40 | } |
| 41 | if ( empty( $env['apikey'] ) ) { |
| 42 | echo "SKIP {$type} ({$env['name']}): no API key\n"; |
| 43 | continue; |
| 44 | } |
| 45 | |
| 46 | echo "\n=== {$type} / {$env['name']} ===\n"; |
| 47 | |
| 48 | try { |
| 49 | $engine = Meow_MWAI_Engines_Factory::get( $core, $env['id'] ); |
| 50 | $models = $engine->retrieve_models(); |
| 51 | } |
| 52 | catch ( Exception $e ) { |
| 53 | echo " ERROR: " . $e->getMessage() . "\n"; |
| 54 | continue; |
| 55 | } |
| 56 | |
| 57 | echo " models classified: " . count( $models ) . "\n"; |
| 58 | $issues = []; |
| 59 | |
| 60 | // 1. Duplicate display names. Google ships preview and GA variants of the same model |
| 61 | // and the name formatter strips the suffix from both, which makes the dropdown |
| 62 | // ambiguous. Anything still colliding here needs disambiguating. |
| 63 | $by_name = []; |
| 64 | foreach ( $models as $m ) { |
| 65 | $by_name[$m['name']][] = $m['model']; |
| 66 | } |
| 67 | foreach ( $by_name as $name => $ids ) { |
| 68 | if ( count( $ids ) > 1 ) { |
| 69 | $issues[] = "duplicate name \"{$name}\" for: " . implode( ', ', $ids ); |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | // 2. Names that clearly fell through the formatter: still carrying a raw id, or a |
| 74 | // lowercased fragment that should have become a proper word. Only meaningful for the |
| 75 | // engines that format their own names. |
| 76 | if ( in_array( $type, $formats_own_names, true ) ) { |
| 77 | foreach ( $models as $m ) { |
| 78 | if ( $m['name'] === $m['model'] ) { |
| 79 | $issues[] = "unformatted name for {$m['model']} (name === id)"; |
| 80 | } |
| 81 | else if ( preg_match( '/(preview|latest|exp|[0-9]{4}-[0-9]{2}|customtools|nano-banana)/i', $m['name'] ) |
| 82 | && !preg_match( '/\(/', $m['name'] ) ) { |
| 83 | $issues[] = "raw id fragment left in name \"{$m['name']}\" ({$m['model']})"; |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // 3. Capability mismatches: the id advertises something the features do not. |
| 89 | // These are the exact shapes that were silently wrong before 2026-07-26. Each engine |
| 90 | // names its features in its own vocabulary (Google says 'image-generation', OpenRouter |
| 91 | // says 'text-to-image'), so any one of the listed synonyms satisfies the expectation. |
| 92 | $expectations = [ |
| 93 | // id pattern => any one of these features is enough |
| 94 | '/-image(-preview)?$|nano-banana/' => [ 'image-generation', 'text-to-image' ], |
| 95 | '/embedding/' => [ 'embedding', 'embeddings' ], |
| 96 | '/(native-audio|-live-)/' => [ 'realtime' ], |
| 97 | '/(veo|video)/' => [ 'video-generation', 'text-to-video' ], |
| 98 | ]; |
| 99 | foreach ( $models as $m ) { |
| 100 | $features = (array) ( $m['features'] ?? [] ); |
| 101 | foreach ( $expectations as $pattern => $accepted ) { |
| 102 | if ( preg_match( $pattern, $m['model'] ) && !array_intersect( $accepted, $features ) ) { |
| 103 | $issues[] = "{$m['model']} looks like '{$accepted[0]}' but features are: " |
| 104 | . ( implode( ',', $features ) ?: '(none)' ); |
| 105 | } |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | // 4. Models with no usable feature at all: they will render in the dropdown and then |
| 110 | // fail at query time. |
| 111 | foreach ( $models as $m ) { |
| 112 | if ( empty( $m['features'] ) ) { |
| 113 | $issues[] = "{$m['model']} has no features"; |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // 5. Anything the provider exposes that never made it into the list. Only the engine |
| 118 | // knows the raw payload, so this compares against the provider call where possible. |
| 119 | if ( $type === 'google' ) { |
| 120 | $res = wp_remote_get( |
| 121 | 'https://generativelanguage.googleapis.com/v1beta/models?pageSize=200&key=' . $env['apikey'], |
| 122 | [ 'timeout' => 30 ] |
| 123 | ); |
| 124 | if ( !is_wp_error( $res ) ) { |
| 125 | $body = json_decode( wp_remote_retrieve_body( $res ), true ); |
| 126 | $classified = array_column( $models, 'model' ); |
| 127 | foreach ( (array) ( $body['models'] ?? [] ) as $raw ) { |
| 128 | $id = str_replace( 'models/', '', $raw['name'] ); |
| 129 | $methods = (array) ( $raw['supportedGenerationMethods'] ?? [] ); |
| 130 | // Only care about ids we would plausibly want to offer. |
| 131 | $wanted = preg_match( '/^(gemini|nano-banana|imagen|veo)/', $id ) |
| 132 | && ( in_array( 'generateContent', $methods, true ) || in_array( 'embedContent', $methods, true ) ); |
| 133 | // Deliberate exclusions: dated snapshots, TTS, robotics. |
| 134 | $excluded = preg_match( '/-(tts|robotics)|(preview|exp)-\d{2}-\d{2,4}$|-\d{8}$/', $id ); |
| 135 | if ( $wanted && !$excluded && !in_array( $id, $classified, true ) ) { |
| 136 | $issues[] = "provider exposes {$id} (" . implode( ',', $methods ) . ") but it was dropped"; |
| 137 | } |
| 138 | } |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | if ( empty( $issues ) ) { |
| 143 | echo " no issues found\n"; |
| 144 | } |
| 145 | else { |
| 146 | $issues = array_values( array_unique( $issues ) ); |
| 147 | $issues_total += count( $issues ); |
| 148 | foreach ( $issues as $i ) { |
| 149 | echo " ISSUE: {$i}\n"; |
| 150 | } |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | echo "\nTOTAL ISSUES: {$issues_total}\n"; |
| 155 |