DefaultVocabLoader.php
87 lines
| 1 | <?php |
| 2 | |
| 3 | declare(strict_types=1); |
| 4 | |
| 5 | namespace Yethee\Tiktoken\Vocab\Loader; |
| 6 | |
| 7 | use RuntimeException; |
| 8 | use Yethee\Tiktoken\Vocab\Vocab; |
| 9 | use Yethee\Tiktoken\Vocab\VocabLoader; |
| 10 | |
| 11 | use function assert; |
| 12 | use function fclose; |
| 13 | use function file_exists; |
| 14 | use function fopen; |
| 15 | use function is_dir; |
| 16 | use function is_writable; |
| 17 | use function mkdir; |
| 18 | use function preg_match; |
| 19 | use function sha1; |
| 20 | use function sprintf; |
| 21 | use function stream_copy_to_stream; |
| 22 | |
| 23 | use const DIRECTORY_SEPARATOR; |
| 24 | |
| 25 | final class DefaultVocabLoader implements VocabLoader |
| 26 | { |
| 27 | public function __construct(private string|null $cacheDir = null) |
| 28 | { |
| 29 | } |
| 30 | |
| 31 | public function load(string $uri): Vocab |
| 32 | { |
| 33 | if ($this->cacheDir !== null && preg_match('@^https?://@i', $uri)) { |
| 34 | $cacheFile = $this->cacheDir . DIRECTORY_SEPARATOR . sha1($uri); |
| 35 | } else { |
| 36 | $cacheFile = null; |
| 37 | } |
| 38 | |
| 39 | if ($cacheFile !== null) { |
| 40 | if (file_exists($cacheFile)) { |
| 41 | return Vocab::fromFile($cacheFile); |
| 42 | } |
| 43 | |
| 44 | assert($this->cacheDir !== null); |
| 45 | |
| 46 | if (! is_dir($this->cacheDir) && ! @mkdir($this->cacheDir, 0750, true)) { |
| 47 | throw new RuntimeException(sprintf( |
| 48 | 'Directory does not exist and cannot be created: %s', |
| 49 | $this->cacheDir, |
| 50 | )); |
| 51 | } |
| 52 | |
| 53 | if (! is_writable($this->cacheDir)) { |
| 54 | throw new RuntimeException(sprintf('Directory is not writable: %s', $this->cacheDir)); |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | $stream = fopen($uri, 'r'); |
| 59 | |
| 60 | if ($stream === false) { |
| 61 | throw new RuntimeException(sprintf('Could not open stream for URI: %s', $uri)); |
| 62 | } |
| 63 | |
| 64 | try { |
| 65 | if ($cacheFile !== null) { |
| 66 | $cacheStream = fopen($cacheFile, 'w+'); |
| 67 | |
| 68 | if ($cacheStream === false) { |
| 69 | throw new RuntimeException(sprintf('Could not open file for write: %s', $cacheFile)); |
| 70 | } |
| 71 | |
| 72 | try { |
| 73 | stream_copy_to_stream($stream, $cacheStream); |
| 74 | |
| 75 | return Vocab::fromStream($cacheStream); |
| 76 | } finally { |
| 77 | fclose($cacheStream); |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | return Vocab::fromStream($stream); |
| 82 | } finally { |
| 83 | fclose($stream); |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 |