anthropic.php
2 years ago
core.php
2 years ago
factory.php
2 years ago
google.php
2 years ago
huggingface.php
2 years ago
openai.php
2 years ago
openrouter.php
2 years ago
replicate.php
2 years ago
replicate.php
821 lines
| 1 | <?php |
| 2 | |
| 3 | class Meow_MWAI_Engines_Replicate extends Meow_MWAI_Engines_Core |
| 4 | { |
| 5 | // Base (Replicate) |
| 6 | protected $apiKey = null; |
| 7 | protected $organizationId = null; |
| 8 | |
| 9 | // Response |
| 10 | protected $inModel = null; |
| 11 | protected $inId = null; |
| 12 | |
| 13 | // Streaming |
| 14 | protected $streamFunctionCall = null; |
| 15 | protected $streamToolCalls = []; |
| 16 | protected $streamLastMessage = null; |
| 17 | protected $streamImageIds = []; |
| 18 | protected $streamInTokens = null; |
| 19 | protected $streamOutTokens = null; |
| 20 | |
| 21 | public function __construct( $core, $env ) |
| 22 | { |
| 23 | parent::__construct( $core, $env ); |
| 24 | $this->set_environment(); |
| 25 | } |
| 26 | |
| 27 | public function reset_stream() { |
| 28 | $this->streamContent = null; |
| 29 | $this->streamBuffer = null; |
| 30 | $this->streamFunctionCall = null; |
| 31 | $this->streamToolCalls = []; |
| 32 | $this->streamLastMessage = null; |
| 33 | $this->streamInTokens = null; |
| 34 | $this->streamOutTokens = null; |
| 35 | $this->inModel = null; |
| 36 | $this->inId = null; |
| 37 | } |
| 38 | |
| 39 | protected function set_environment() { |
| 40 | $env = $this->env; |
| 41 | $this->apiKey = $env['apikey']; |
| 42 | } |
| 43 | |
| 44 | protected function build_messages( $query ) { |
| 45 | $messages = []; |
| 46 | |
| 47 | // First, we need to add the first message (the instructions). |
| 48 | if ( !empty( $query->instructions ) ) { |
| 49 | $messages[] = [ 'role' => 'system', 'content' => $query->instructions ]; |
| 50 | } |
| 51 | |
| 52 | // Then, if any, we need to add the 'messages', they are already formatted. |
| 53 | foreach ( $query->messages as $message ) { |
| 54 | $messages[] = $message; |
| 55 | } |
| 56 | |
| 57 | // If there is a context, we need to add it. |
| 58 | if ( !empty( $query->context ) ) { |
| 59 | $messages[] = [ 'role' => 'system', 'content' => $query->context ]; |
| 60 | } |
| 61 | |
| 62 | // Finally, we need to add the message, but if there is an image, we need to add it as a system message. |
| 63 | if ( $query->attachedFile ) { |
| 64 | $finalUrl = null; |
| 65 | if ( $query->image_remote_upload ) { |
| 66 | $finalUrl = $query->attachedFile->get_url(); |
| 67 | } |
| 68 | else { |
| 69 | $finalUrl = $query->attachedFile->get_inline_base64_url(); |
| 70 | } |
| 71 | $messages[] = [ |
| 72 | 'role' => 'user', |
| 73 | 'content' => [ |
| 74 | [ |
| 75 | "type" => "text", |
| 76 | "text" => $query->get_message() |
| 77 | ], |
| 78 | [ |
| 79 | "type" => "image_url", |
| 80 | "image_url" => [ |
| 81 | "url" => $finalUrl |
| 82 | ] |
| 83 | ] |
| 84 | ] |
| 85 | ]; |
| 86 | } |
| 87 | else { |
| 88 | $messages[] = [ 'role' => 'user', 'content' => $query->get_message() ]; |
| 89 | } |
| 90 | |
| 91 | return $messages; |
| 92 | } |
| 93 | |
| 94 | protected function build_body( $query, $streamCallback = null, $extra = null ) { |
| 95 | if ( $query instanceof Meow_MWAI_Query_Text ) { |
| 96 | $body = array( |
| 97 | "model" => $query->model, |
| 98 | "n" => $query->maxResults, |
| 99 | "max_tokens" => $query->maxTokens, |
| 100 | "temperature" => $query->temperature, |
| 101 | "stream" => !is_null( $streamCallback ), |
| 102 | ); |
| 103 | |
| 104 | if ( !empty( $query->stop ) ) { |
| 105 | $body['stop'] = $query->stop; |
| 106 | } |
| 107 | |
| 108 | if ( !empty( $query->responseFormat ) ) { |
| 109 | if ( $query->responseFormat === 'json' ) { |
| 110 | $body['response_format'] = [ 'type' => 'json_object' ]; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | |
| 115 | // Usage Data (only for Replicate) |
| 116 | // https://cookbook.openai.com/examples/how_to_stream_completions#4-how-to-get-token-usage-data-for-streamed-chat-completion-response |
| 117 | if ( !empty( $streamCallback ) && $this->envType === 'openai' ) { |
| 118 | $body['stream_options'] = [ |
| 119 | 'include_usage' => true, |
| 120 | ]; |
| 121 | } |
| 122 | |
| 123 | if ( !empty( $query->functions ) ) { |
| 124 | $model = $this->retrieve_model_info( $query->model ); |
| 125 | if ( !empty( $model['tags'] ) && !in_array( 'functions', $model['tags'] ) ) { |
| 126 | Meow_MWAI_Logging::warn( 'The model ' . $query->model . ' doesn\'t support Function Calling.' ); |
| 127 | } |
| 128 | else if ( strpos( $query->model, 'ft:' ) === 0 ) { |
| 129 | Meow_MWAI_Logging::warn( 'Replicate doesn\'t support Function Calling with fine-tuned models yet.' ); |
| 130 | } |
| 131 | else { |
| 132 | $body['tools'] = []; |
| 133 | // Dynamic function: they will interactively enhance the completion (tools). |
| 134 | foreach ( $query->functions as $function ) { |
| 135 | $body['tools'][] = [ |
| 136 | 'type' => 'function', |
| 137 | 'function' => $function->serializeForReplicate() |
| 138 | ]; |
| 139 | } |
| 140 | // Static functions: they will be executed at the end of the completion. |
| 141 | //$body['function_call'] = $query->functionCall; |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | $body['messages'] = $this->build_messages( $query ); |
| 146 | |
| 147 | // Add the feedback if it's a feedback query. |
| 148 | if ( $query instanceof Meow_MWAI_Query_Feedback ) { |
| 149 | if ( !empty( $query->blocks ) ) { |
| 150 | foreach ( $query->blocks as $feedback_block ) { |
| 151 | $body['messages'][] = $feedback_block['rawMessage']; |
| 152 | foreach ( $feedback_block['feedbacks'] as $feedback ) { |
| 153 | $body['messages'][] = [ |
| 154 | 'tool_call_id' => $feedback['request']['toolId'], |
| 155 | "role" => "tool", |
| 156 | 'name' => $feedback['request']['name'], |
| 157 | 'content' => $feedback['reply']['value'] |
| 158 | ]; |
| 159 | } |
| 160 | } |
| 161 | } |
| 162 | return $body; |
| 163 | } |
| 164 | |
| 165 | return $body; |
| 166 | } |
| 167 | else if ( $query instanceof Meow_MWAI_Query_Transcribe ) { |
| 168 | $body = array( |
| 169 | 'prompt' => $query->message, |
| 170 | 'model' => $query->model, |
| 171 | 'response_format' => 'text', |
| 172 | 'file' => basename( $query->url ), |
| 173 | 'data' => $extra |
| 174 | ); |
| 175 | return $body; |
| 176 | } |
| 177 | else if ( $query instanceof Meow_MWAI_Query_Embed ) { |
| 178 | $body = array( 'input' => $query->message, 'model' => $query->model ); |
| 179 | if ( $this->envType === 'azure' ) { |
| 180 | $body = array( "input" => $query->message ); |
| 181 | } |
| 182 | // Dimensions are only supported by v3 models |
| 183 | if ( !empty( $query->dimensions ) && strpos( $query->model, 'ada-002' ) === false ) { |
| 184 | $body['dimensions'] = $query->dimensions; |
| 185 | } |
| 186 | return $body; |
| 187 | } |
| 188 | else if ( $query instanceof Meow_MWAI_Query_Image ) { |
| 189 | $model = $query->model; |
| 190 | $modelInfo = $this->retrieve_model_info( $model ); |
| 191 | $body = array( "input" => [] ); |
| 192 | |
| 193 | if ( isset( $modelInfo['version'] ) ) { |
| 194 | $body['version'] = $modelInfo['version']; |
| 195 | } |
| 196 | |
| 197 | // From Replicate: |
| 198 | // Files should be passed as HTTP URLs or data URLs. |
| 199 | |
| 200 | if ( $query->feature === 'text-to-image' ) { |
| 201 | |
| 202 | // This works with Flux |
| 203 | // The model name starts with black-forest-labs/ |
| 204 | if ( strpos( $model, 'black-forest-labs/' ) === 0 ) { |
| 205 | $body['input']['steps'] = 25; |
| 206 | $body['input']['prompt'] = $query->message; |
| 207 | $body['input']['safety_tolerance'] = 5; |
| 208 | if ( !empty( $query->resolution ) ) { |
| 209 | $body['input']['aspect_ratio'] = $query->resolution; |
| 210 | } |
| 211 | } |
| 212 | else if ( strpos( $model, 'stability-ai/' ) === 0 ) { |
| 213 | $body['input']['prompt'] = $query->message; |
| 214 | $body['input']['num_inference_steps'] = 25; |
| 215 | if ( !empty( $query->resolution ) ) { |
| 216 | // $query->resolution is actually a string like 1024x1024 |
| 217 | $parts = explode( 'x', $query->resolution ); |
| 218 | $width = intval( $parts[0] ); |
| 219 | $height = intval( $parts[1] ); |
| 220 | $body['input']['width'] = $width; |
| 221 | $body['input']['height'] = $height; |
| 222 | } |
| 223 | } |
| 224 | else { |
| 225 | throw new Exception( 'The model ' . $model . ' is not supported for text-to-image.' ); |
| 226 | } |
| 227 | |
| 228 | // seed |
| 229 | // steps: Number of diffusion steps |
| 230 | // guidance: Controls the balance between adherence to the text prompt and image quality/diversity. Higher values make the output more closely match the prompt but may reduce overall image quality. Lower values allow for more creative freedom but might produce results less relevant to the prompt. |
| 231 | // interval: Interval is a setting that increases the variance in possible outputs letting the model be a tad more dynamic in what outputs it may produce in terms of composition, color, detail, and prompt interpretation. Setting this value low will ensure strong prompt following with more consistent outputs, setting it higher will produce more dynamic or varied outputs. |
| 232 | // aspect_ratio: Aspect ratio for the generated image |
| 233 | // safety_tolerance: Safety tolerance, 1 is most strict and 5 is most permissive |
| 234 | } |
| 235 | return $body; |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | protected function build_url( $query, $endpoint = null ) { |
| 240 | $url = ""; |
| 241 | $env = $this->env; |
| 242 | // This endpoint is basically Replicate or Azure, but in the case this class |
| 243 | // is overriden, we can pass the endpoint directly (for OpenRouter or HuggingFace, for example). |
| 244 | if ( empty( $endpoint ) ) { |
| 245 | $endpoint = apply_filters( 'mwai_replicate_endpoint', 'https://api.replicate.com/v1', $this->env ); |
| 246 | } |
| 247 | // Add the base API to the URL |
| 248 | if ( $query instanceof Meow_MWAI_Query_Text || $query instanceof Meow_MWAI_Query_Feedback ) { |
| 249 | throw new Exception( 'Not implemented yet.' ); |
| 250 | return $url; |
| 251 | } |
| 252 | else if ( $query instanceof Meow_MWAI_Query_Transcribe ) { |
| 253 | throw new Exception( 'Not implemented yet.' ); |
| 254 | return $url; |
| 255 | } |
| 256 | else if ( $query instanceof Meow_MWAI_Query_Embed ) { |
| 257 | throw new Exception( 'Not implemented yet.' ); |
| 258 | return $url; |
| 259 | } |
| 260 | else if ( $query instanceof Meow_MWAI_Query_Image ) { |
| 261 | //$url .= trailingslashit( $endpoint ) . 'models/' . $query->model . '/predictions'; |
| 262 | $url .= trailingslashit( $endpoint ) . 'predictions'; |
| 263 | return $url; |
| 264 | } |
| 265 | throw new Exception( 'The query is not supported by build_url().' ); |
| 266 | } |
| 267 | |
| 268 | protected function build_headers( $query ) { |
| 269 | if ( $query->apiKey ) { |
| 270 | $this->apiKey = $query->apiKey; |
| 271 | } |
| 272 | if ( empty( $this->apiKey ) ) { |
| 273 | throw new Exception( 'No API Key provided. Please visit the Settings.' ); |
| 274 | } |
| 275 | $headers = array( |
| 276 | 'Content-Type' => 'application/json', |
| 277 | 'Authorization' => 'Bearer ' . $this->apiKey, |
| 278 | ); |
| 279 | if ( $this->organizationId ) { |
| 280 | $headers['Replicate-Organization'] = $this->organizationId; |
| 281 | } |
| 282 | if ( $this->envType === 'azure' ) { |
| 283 | $headers = array( 'Content-Type' => 'application/json', 'api-key' => $this->apiKey ); |
| 284 | } |
| 285 | return $headers; |
| 286 | } |
| 287 | |
| 288 | protected function build_options( $headers, $json = null, $forms = null, $method = 'POST' ) { |
| 289 | $body = null; |
| 290 | if ( !empty( $forms ) ) { |
| 291 | $boundary = wp_generate_password ( 24, false ); |
| 292 | $headers['Content-Type'] = 'multipart/form-data; boundary=' . $boundary; |
| 293 | $body = $this->build_form_body( $forms, $boundary ); |
| 294 | } |
| 295 | else if ( !empty( $json ) ) { |
| 296 | $body = json_encode( $json ); |
| 297 | } |
| 298 | $options = array( |
| 299 | 'headers' => $headers, |
| 300 | 'method' => $method, |
| 301 | 'timeout' => MWAI_TIMEOUT, |
| 302 | 'body' => $body, |
| 303 | 'sslverify' => false |
| 304 | ); |
| 305 | return $options; |
| 306 | } |
| 307 | |
| 308 | public function run_query( $url, $options, $isStream = false ) { |
| 309 | try { |
| 310 | $options['stream'] = $isStream; |
| 311 | if ( $isStream ) { |
| 312 | $options['filename'] = tempnam( sys_get_temp_dir(), 'mwai-stream-' ); |
| 313 | } |
| 314 | $res = wp_remote_get( $url, $options ); |
| 315 | |
| 316 | if ( is_wp_error( $res ) ) { |
| 317 | throw new Exception( $res->get_error_message() ); |
| 318 | } |
| 319 | |
| 320 | $responseCode = wp_remote_retrieve_response_code( $res ); |
| 321 | if ( $responseCode === 404 ) { |
| 322 | throw new Exception( 'The model\'s API URL was not found: ' . $url ); |
| 323 | } |
| 324 | if ( $responseCode === 400 ) { |
| 325 | $message = wp_remote_retrieve_body( $res ); |
| 326 | if ( empty( $message ) ) { |
| 327 | $message = wp_remote_retrieve_response_message( $res ); |
| 328 | } |
| 329 | if ( empty( $message ) ) { |
| 330 | $message = 'Bad Request'; |
| 331 | } |
| 332 | throw new Exception( $message ); |
| 333 | } |
| 334 | |
| 335 | if ( $isStream ) { |
| 336 | return [ 'stream' => true ]; |
| 337 | } |
| 338 | |
| 339 | $response = wp_remote_retrieve_body( $res ); |
| 340 | $headersRes = wp_remote_retrieve_headers( $res ); |
| 341 | $headers = $headersRes->getAll(); |
| 342 | |
| 343 | // Check if Content-Type is 'multipart/form-data' or 'text/plain' |
| 344 | // If so, we don't need to decode the response |
| 345 | $normalizedHeaders = array_change_key_case( $headers, CASE_LOWER ); |
| 346 | $resContentType = $normalizedHeaders['content-type'] ?? ''; |
| 347 | if ( strpos( $resContentType, 'multipart/form-data' ) !== false || strpos( $resContentType, 'text/plain' ) !== false ) { |
| 348 | return [ 'stream' => false, 'headers' => $headers, 'data' => $response ]; |
| 349 | } |
| 350 | |
| 351 | $data = json_decode( $response, true ); |
| 352 | $this->handle_response_errors( $data ); |
| 353 | return [ 'headers' => $headers, 'data' => $data ]; |
| 354 | } |
| 355 | catch ( Exception $e ) { |
| 356 | Meow_MWAI_Logging::error( 'Replicate: ' . $e->getMessage() ); |
| 357 | throw $e; |
| 358 | } |
| 359 | finally { |
| 360 | if ( $isStream && file_exists( $options['filename'] ) ) { |
| 361 | unlink( $options['filename'] ); |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | public function try_decode_error( $data ) { |
| 367 | $json = json_decode( $data, true ); |
| 368 | if ( isset( $json['error']['message'] ) ) { |
| 369 | return $json['error']['message']; |
| 370 | } |
| 371 | return null; |
| 372 | } |
| 373 | |
| 374 | public function run_completion_query( $query, $streamCallback = null ) : Meow_MWAI_Reply { |
| 375 | $isStreaming = !is_null( $streamCallback ); |
| 376 | |
| 377 | if ( $isStreaming ) { |
| 378 | $this->streamCallback = $streamCallback; |
| 379 | add_action( 'http_api_curl', [ $this, 'stream_handler' ], 10, 3 ); |
| 380 | } |
| 381 | |
| 382 | $this->reset_stream(); |
| 383 | $body = $this->build_body( $query, $streamCallback ); |
| 384 | $url = $this->build_url( $query ); |
| 385 | $headers = $this->build_headers( $query ); |
| 386 | $options = $this->build_options( $headers, $body ); |
| 387 | |
| 388 | try { |
| 389 | $res = $this->run_query( $url, $options, $streamCallback ); |
| 390 | $reply = new Meow_MWAI_Reply( $query ); |
| 391 | |
| 392 | $returned_id = null; |
| 393 | $returned_model = $this->inModel; |
| 394 | $returned_in_tokens = null; |
| 395 | $returned_out_tokens = null; |
| 396 | $returned_price = null; |
| 397 | $returned_choices = []; |
| 398 | |
| 399 | // Streaming Mode |
| 400 | if ( $isStreaming ) { |
| 401 | if ( empty( $this->streamContent ) ) { |
| 402 | $error = $this->try_decode_error( $this->streamBuffer ); |
| 403 | if ( !is_null( $error ) ) { |
| 404 | throw new Exception( $error ); |
| 405 | } |
| 406 | } |
| 407 | $returned_id = $this->inId; |
| 408 | $returned_model = $this->inModel ? $this->inModel : $query->model; |
| 409 | $message = [ 'role' => 'assistant', 'content' => $this->streamContent ]; |
| 410 | if ( !empty( $this->streamFunctionCall ) ) { |
| 411 | $message['function_call'] = $this->streamFunctionCall; |
| 412 | } |
| 413 | if ( !empty( $this->streamToolCalls ) ) { |
| 414 | $message['tool_calls'] = $this->streamToolCalls; |
| 415 | } |
| 416 | if ( !is_null( $this->streamInTokens ) ) { |
| 417 | $returned_in_tokens = $this->streamInTokens; |
| 418 | } |
| 419 | if ( !is_null( $this->streamOutTokens ) ) { |
| 420 | $returned_out_tokens = $this->streamOutTokens; |
| 421 | } |
| 422 | $returned_choices = [ [ 'message' => $message ] ]; |
| 423 | } |
| 424 | // Standard Mode |
| 425 | else { |
| 426 | $data = $res['data']; |
| 427 | if ( empty( $data ) ) { |
| 428 | throw new Exception( 'No content received (res is null).' ); |
| 429 | } |
| 430 | if ( !$data['model'] ) { |
| 431 | Meow_MWAI_Logging::error( 'Replicate: Invalid response (no model information).' ); |
| 432 | Meow_MWAI_Logging::error( print_r( $data, 1 ) ); |
| 433 | throw new Exception( 'Invalid response (no model information).' ); |
| 434 | } |
| 435 | $returned_id = $data['id']; |
| 436 | $returned_model = $data['model']; |
| 437 | $returned_in_tokens = isset( $data['usage']['prompt_tokens'] ) ? |
| 438 | $data['usage']['prompt_tokens'] : null; |
| 439 | $returned_out_tokens = isset( $data['usage']['completion_tokens'] ) ? |
| 440 | $data['usage']['completion_tokens'] : null; |
| 441 | $returned_price = isset( $data['usage']['total_cost'] ) ? |
| 442 | $data['usage']['total_cost'] : null; |
| 443 | $returned_choices = $data['choices']; |
| 444 | } |
| 445 | |
| 446 | // Set the results. |
| 447 | $reply->set_choices( $returned_choices ); |
| 448 | if ( !empty( $returned_id ) ) { |
| 449 | $reply->set_id( $returned_id ); |
| 450 | } |
| 451 | if ( !empty( $returned_id ) ) { |
| 452 | $reply->set_id( $returned_id ); |
| 453 | } |
| 454 | |
| 455 | return $reply; |
| 456 | } |
| 457 | catch ( Exception $e ) { |
| 458 | Meow_MWAI_Logging::error( 'Replicate: ' . $e->getMessage() ); |
| 459 | $message = "From Replicate: " . $e->getMessage(); |
| 460 | throw new Exception( $message ); |
| 461 | } |
| 462 | finally { |
| 463 | if ( !is_null( $streamCallback ) ) { |
| 464 | remove_action( 'http_api_curl', [ $this, 'stream_handler' ] ); |
| 465 | } |
| 466 | } |
| 467 | } |
| 468 | |
| 469 | // TODO: We should find a way to add text-to-image somewhere in this query |
| 470 | public function run_image_query( $query ) { |
| 471 | $body = $this->build_body( $query ); |
| 472 | $url = $this->build_url( $query ); |
| 473 | $headers = $this->build_headers( $query ); |
| 474 | $options = $this->build_options( $headers, $body ); |
| 475 | |
| 476 | try { |
| 477 | $res = $this->run_query( $url, $options ); |
| 478 | $data = $res['data']; |
| 479 | if ( $data['status'] === 422 ) { |
| 480 | if ( isset( $data['title'] ) && isset( $data['detail'] ) ) { |
| 481 | throw new Exception( $data['title'] . ': ' . $data['detail'] ); |
| 482 | } |
| 483 | throw new Exception( 'The image generation failed.' ); |
| 484 | } |
| 485 | $getUrl = $data['urls']['get']; |
| 486 | $status = $data['status']; |
| 487 | while ( $status === 'processing' || $status === 'starting' ) { |
| 488 | sleep( 1 ); |
| 489 | $data = $this->execute( 'GET', $getUrl, null, null, true, null, null, true ); |
| 490 | $status = $data['status']; |
| 491 | } |
| 492 | if ( $status !== 'succeeded' ) { |
| 493 | // if $data has title and detail, we can use them to throw a more detailed error |
| 494 | if ( isset( $data['title'] ) && isset( $data['detail'] ) ) { |
| 495 | throw new Exception( $data['title'] . ': ' . $data['detail'] ); |
| 496 | } |
| 497 | throw new Exception( 'The image generation failed.' ); |
| 498 | } |
| 499 | $choices = []; |
| 500 | $output = isset( $data['output'] ) ? $data['output'] : []; |
| 501 | // Flux Schnell returns an array of urls in 'output' |
| 502 | if ( is_array( $output ) ) { |
| 503 | foreach ( $output as $item ) { |
| 504 | $choices[] = [ 'url' => $item ]; |
| 505 | } |
| 506 | } |
| 507 | // Flux Schnell returns 'url' in 'output' |
| 508 | else if ( is_string( $output ) ) { |
| 509 | $choices[] = [ 'url' => $output ]; |
| 510 | } |
| 511 | if ( empty( $choices ) ) { |
| 512 | throw new Exception( 'No output URL received.' ); |
| 513 | } |
| 514 | $reply = new Meow_MWAI_Reply( $query ); |
| 515 | $model = $query->model; |
| 516 | $resolution = null; |
| 517 | if ( isset( $data['metrics']['width'] ) && isset( $data['metrics']['height'] ) ) { |
| 518 | $resolution = $data['metrics']['width'] . 'x' . $data['metrics']['height']; |
| 519 | } |
| 520 | else { |
| 521 | $raw_resolution = Meow_MWAI_Core::get_image_resolution( $choices[0]['url'] ); |
| 522 | if ( !empty( $raw_resolution ) ) { |
| 523 | $resolution = $raw_resolution['width'] . 'x' . $raw_resolution['height']; |
| 524 | } |
| 525 | |
| 526 | } |
| 527 | if ( !empty( $resolution ) ) { |
| 528 | $usage = $this->core->record_images_usage( $model, $resolution, $query->maxResults ); |
| 529 | $reply->set_usage( $usage ); |
| 530 | } |
| 531 | |
| 532 | $reply->set_choices( $choices ); |
| 533 | $reply->set_type( 'images' ); |
| 534 | |
| 535 | if ( $query->localDownload === 'uploads' || $query->localDownload === 'library' ) { |
| 536 | foreach ( $reply->results as &$result ) { |
| 537 | $fileId = $this->core->files->upload_file( $result, null, 'generated', [ |
| 538 | 'query_envId' => $query->envId, |
| 539 | 'query_session' => $query->session, |
| 540 | 'query_model' => $query->model, |
| 541 | ], $query->envId, $query->localDownload, $query->localDownloadExpiry ); |
| 542 | $fileUrl = $this->core->files->get_url( $fileId ); |
| 543 | $result = $fileUrl; |
| 544 | } |
| 545 | } |
| 546 | $reply->result = $reply->results[0]; |
| 547 | return $reply; |
| 548 | } |
| 549 | catch ( Exception $e ) { |
| 550 | Meow_MWAI_Logging::error( 'Replicate: ' . $e->getMessage() ); |
| 551 | throw new Exception( "From Replicate: " . $e->getMessage() ); |
| 552 | } |
| 553 | } |
| 554 | |
| 555 | /* |
| 556 | This is the rest of the Replicate API support, not related to the models directly. |
| 557 | */ |
| 558 | |
| 559 | // Check if there are errors in the response from Replicate, and throw an exception if so. |
| 560 | protected function handle_response_errors( $data ) { |
| 561 | if ( isset( $data['error'] ) && !empty( $data['error'] ) ) { |
| 562 | $message = $data['error']; |
| 563 | throw new Exception( $message ); |
| 564 | } |
| 565 | } |
| 566 | |
| 567 | /** |
| 568 | * Build the body of a form request. |
| 569 | * If the field name is 'file', then the field value is the filename of the file to upload. |
| 570 | * The file contents are taken from the 'data' field. |
| 571 | * |
| 572 | * @param array $fields |
| 573 | * @param string $boundary |
| 574 | * @return string |
| 575 | */ |
| 576 | public function build_form_body( $fields, $boundary ) |
| 577 | { |
| 578 | $body = ''; |
| 579 | foreach ( $fields as $name => $value ) { |
| 580 | if ( $name == 'data' ) { |
| 581 | continue; |
| 582 | } |
| 583 | $body .= "--$boundary\r\n"; |
| 584 | $body .= "Content-Disposition: form-data; name=\"$name\""; |
| 585 | if ( $name == 'file' ) { |
| 586 | $body .= "; filename=\"{$value}\"\r\n"; |
| 587 | $body .= "Content-Type: application/json\r\n\r\n"; |
| 588 | $body .= $fields['data'] . "\r\n"; |
| 589 | } |
| 590 | else { |
| 591 | $body .= "\r\n\r\n$value\r\n"; |
| 592 | } |
| 593 | } |
| 594 | $body .= "--$boundary--\r\n"; |
| 595 | return $body; |
| 596 | } |
| 597 | |
| 598 | /** |
| 599 | * Run a request to the Replicate API. |
| 600 | * Fore more information about the $formFields, refer to the build_form_body method. |
| 601 | * |
| 602 | * @param string $method POST, PUT, GET, DELETE... |
| 603 | * @param string $url The API endpoint |
| 604 | * @param array $query The query parameters (json) |
| 605 | * @param array $formFields The form fields (multipart/form-data) |
| 606 | * @param bool $json Whether to return the response as json or not |
| 607 | * @return array |
| 608 | */ |
| 609 | public function execute( $method, $url, $query = null, $formFields = null, |
| 610 | $json = true, $extraHeaders = null, $streamCallback = null, $overrideUrl = false ) { |
| 611 | $headers = "Content-Type: application/json\r\n" . "Authorization: Bearer " . $this->apiKey . "\r\n"; |
| 612 | $body = $query ? json_encode( $query ) : null; |
| 613 | if ( !empty( $formFields ) ) { |
| 614 | $boundary = wp_generate_password( 24, false ); |
| 615 | $headers = [ |
| 616 | 'Content-Type' => 'multipart/form-data; boundary=' . $boundary, |
| 617 | 'Authorization' => 'Bearer ' . $this->apiKey |
| 618 | ]; |
| 619 | $body = $this->build_form_body( $formFields, $boundary ); |
| 620 | } |
| 621 | |
| 622 | // Maybe we should have headers always as an array... not sure why we have it as a string. |
| 623 | if ( !empty( $extraHeaders ) ) { |
| 624 | foreach ( $extraHeaders as $key => $value ) { |
| 625 | if ( is_array( $headers ) ) { |
| 626 | $headers[$key] = $value; |
| 627 | } |
| 628 | else { |
| 629 | $headers .= "$key: $value\r\n"; |
| 630 | } |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | // If it's a GET, body should be null, and we should append the query to the URL. |
| 635 | if ( $method === 'GET' ) { |
| 636 | if ( !empty( $query ) ) { |
| 637 | $url .= '?' . http_build_query( $query ); |
| 638 | } |
| 639 | $body = null; |
| 640 | } |
| 641 | |
| 642 | $url = $overrideUrl ? $url : ( 'https://api.replicate.com/v1' . $url ); |
| 643 | $options = [ |
| 644 | "headers" => $headers, |
| 645 | "method" => $method, |
| 646 | "timeout" => MWAI_TIMEOUT, |
| 647 | "body" => $body, |
| 648 | "sslverify" => false |
| 649 | ]; |
| 650 | |
| 651 | try { |
| 652 | if ( !is_null( $streamCallback ) ) { |
| 653 | $options['stream'] = true; |
| 654 | $options['filename'] = tempnam( sys_get_temp_dir(), 'mwai-stream-' ); |
| 655 | // The stream handler calls the streamCallback every time there is content |
| 656 | // TODO: For assistants, we should probably have a different stream handler to |
| 657 | // handle the assistant's specific reply and perform the necessary actions. |
| 658 | $this->streamCallback = $streamCallback; |
| 659 | add_action( 'http_api_curl', [ $this, 'stream_handler' ], 10, 3 ); |
| 660 | } |
| 661 | $res = wp_remote_request( $url, $options ); |
| 662 | if ( is_wp_error( $res ) ) { |
| 663 | throw new Exception( $res->get_error_message() ); |
| 664 | } |
| 665 | $res = wp_remote_retrieve_body( $res ); |
| 666 | $data = $json ? json_decode( $res, true ) : $res; |
| 667 | $this->handle_response_errors( $data ); |
| 668 | return $data; |
| 669 | } |
| 670 | catch ( Exception $e ) { |
| 671 | Meow_MWAI_Logging::error( 'Replicate: ' . $e->getMessage() ); |
| 672 | throw new Exception( $e->getMessage() ); |
| 673 | //throw new Exception( 'From Replicate: ' . $e->getMessage() ); |
| 674 | } |
| 675 | finally { |
| 676 | if ( !is_null( $streamCallback ) ) { |
| 677 | remove_action( 'http_api_curl', [ $this, 'stream_handler' ] ); |
| 678 | } |
| 679 | if ( !empty( $options['stream'] ) && file_exists( $options['filename'] ) ) { |
| 680 | unlink( $options['filename'] ); |
| 681 | } |
| 682 | } |
| 683 | } |
| 684 | |
| 685 | public function get_models() { |
| 686 | return $this->core->get_engine_models( 'replicate' ); |
| 687 | } |
| 688 | |
| 689 | public function get_price( Meow_MWAI_Query_Base $query, Meow_MWAI_Reply $reply ) |
| 690 | { |
| 691 | return null; |
| 692 | } |
| 693 | |
| 694 | function generate_resolutions( $widths, $heights ) { |
| 695 | $resolutions = []; |
| 696 | $acceptable_ratios = [ |
| 697 | '1:1' => [ 'name' => 'Square', 'ratio' => 1 ], |
| 698 | '16:9' => [ 'name' => 'Widescreen', 'ratio' => 16/9 ], |
| 699 | '2:3' => [ 'name' => 'Portrait', 'ratio' => 2/3 ], |
| 700 | '3:2' => [ 'name' => 'Landscape', 'ratio' => 3/2 ], |
| 701 | '4:5' => [ 'name' => 'Portrait', 'ratio' => 4/5 ], |
| 702 | '5:4' => [ 'name' => 'Landscape', 'ratio' => 5/4 ], |
| 703 | '9:16' => [ 'name' => 'Story', 'ratio' => 9/16 ] |
| 704 | ]; |
| 705 | |
| 706 | foreach ( $widths as $width ) { |
| 707 | foreach ( $heights as $height ) { |
| 708 | if ( $height <= 1024 && $width <= 1536 && $height >= 64 && $width >= 64 ) { |
| 709 | $ratio = $width / $height; |
| 710 | $ratio_name = null; |
| 711 | |
| 712 | foreach ( $acceptable_ratios as $key => $ratio_info ) { |
| 713 | if ( abs( $ratio - $ratio_info['ratio'] ) < 0.01 ) { |
| 714 | $ratio_name = $key; |
| 715 | $ratio_label = $ratio_info['name']; |
| 716 | break; |
| 717 | } |
| 718 | } |
| 719 | |
| 720 | if ( $ratio_name ) { |
| 721 | $label = "{$ratio_label} ({$ratio_name}): {$width}x{$height}"; |
| 722 | $resolutions[] = [ |
| 723 | 'name' => "{$width}x{$height}", |
| 724 | 'label' => $label |
| 725 | ]; |
| 726 | } |
| 727 | } |
| 728 | } |
| 729 | } |
| 730 | |
| 731 | // Sort resolutions by total pixel count |
| 732 | usort( $resolutions, function( $a, $b ) { |
| 733 | $aPixels = explode( 'x', $a['name'] ); |
| 734 | $bPixels = explode( 'x', $b['name'] ); |
| 735 | return ( $aPixels[0] * $aPixels[1] ) - ( $bPixels[0] * $bPixels[1] ); |
| 736 | } ); |
| 737 | |
| 738 | return $resolutions; |
| 739 | } |
| 740 | |
| 741 | public function retrieve_models() { |
| 742 | $rawModels = []; |
| 743 | $next = '/models'; |
| 744 | $cursor = null; |
| 745 | $allowed_owners = [ 'black-forest-labs', 'stability-ai' ]; |
| 746 | while ( $next ) { |
| 747 | $query_args = $cursor ? [ 'cursor' => $cursor ] : []; |
| 748 | $response = $this->execute( 'GET', $next, $query_args ); |
| 749 | if ( !is_array( $response ) || empty( $response['results'] ) ) { |
| 750 | break; |
| 751 | } |
| 752 | $filtered_results = array_filter( $response['results'], function( $model ) use ( $allowed_owners ) { |
| 753 | $isAllowedOwner = isset( $model['owner'] ) && in_array( $model['owner'], $allowed_owners ); |
| 754 | $isPublic = isset( $model['visibility'] ) && $model['visibility'] === 'public'; |
| 755 | return $isAllowedOwner && $isPublic; |
| 756 | } ); |
| 757 | $rawModels = array_merge( $rawModels, $filtered_results ); |
| 758 | if ( empty( $response['next'] ) ) { |
| 759 | break; |
| 760 | } |
| 761 | $parsed_url = wp_parse_url( $response['next'] ); |
| 762 | parse_str( $parsed_url['query'] ?? '', $query_params ); |
| 763 | $cursor = $query_params['cursor'] ?? ''; |
| 764 | $next = '/models'; |
| 765 | } |
| 766 | $models = array(); |
| 767 | foreach ( $rawModels as $rawModel ) { |
| 768 | |
| 769 | $name = trim( $rawModel['name'] ); |
| 770 | $family = trim( $rawModel['owner'] ); |
| 771 | $tags = ['image', 'text-to-image']; |
| 772 | $model = $family . '/' . $name; |
| 773 | $version = isset( $rawModel['latest_version']['id'] ) ? $rawModel['latest_version']['id'] : null; |
| 774 | |
| 775 | // outpainting, super-resolution, |
| 776 | if ( $family === 'stability-ai' ) { |
| 777 | $tags[] = 'image-to-image'; |
| 778 | $tags[] = 'inpainting'; |
| 779 | } |
| 780 | |
| 781 | $resolutions = []; |
| 782 | |
| 783 | // Black Forest Labs |
| 784 | if ( $family === 'black-forest-labs' ) { |
| 785 | |
| 786 | // Those works at least for Flux Pro |
| 787 | $resolutions[] = [ 'name' => '1:1', 'label' => 'Square (1:1)' ]; |
| 788 | $resolutions[] = [ 'name' => '16:9', 'label' => 'Widescreen (16:9)' ]; |
| 789 | $resolutions[] = [ 'name' => '2:3', 'label' => 'Portrait (2:3)' ]; |
| 790 | $resolutions[] = [ 'name' => '3:2', 'label' => 'Landscape (3:2)' ]; |
| 791 | $resolutions[] = [ 'name' => '4:5', 'label' => 'Portrait (4:5)' ]; |
| 792 | $resolutions[] = [ 'name' => '5:4', 'label' => 'Landscape (5:4)' ]; |
| 793 | $resolutions[] = [ 'name' => '9:16', 'label' => 'Story (9:16)' ]; |
| 794 | } |
| 795 | |
| 796 | // Stability AI |
| 797 | if ( $family === 'stability-ai' ) { |
| 798 | $heights = [64, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1152, 1216, 1344, 1536]; |
| 799 | $widths = [64, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024]; |
| 800 | $resolutions = $this->generate_resolutions( $widths, $heights ); |
| 801 | } |
| 802 | |
| 803 | $models[] = array( |
| 804 | 'model' => $model, |
| 805 | 'name' => $name, |
| 806 | 'family' => $family, |
| 807 | 'version' => $version, |
| 808 | 'features' => ['text-to-image'], |
| 809 | 'price' => null, |
| 810 | 'type' => 'image', |
| 811 | 'resolutions' => $resolutions, |
| 812 | 'unit' => 1 / 1000, |
| 813 | 'maxCompletionTokens' => null, |
| 814 | 'maxContextualTokens' => null, |
| 815 | 'tags' => $tags |
| 816 | ); |
| 817 | } |
| 818 | return $models; |
| 819 | } |
| 820 | } |
| 821 |