| 1 |
/** |
| 2 |
* Detect when an LLM response is actually a "I need more info" refusal |
| 3 |
* disguised as content, instead of the structured output the prompt |
| 4 |
* asked for. The model occasionally does this when the form has too |
| 5 |
* many empty fields; without detection we'd render the question text |
| 6 |
* as if it were the operator's content (e.g. push the questions into |
| 7 |
* the highlights array, or parse zero days from an itinerary). |
| 8 |
* |
| 9 |
* The detector is intentionally conservative — false positives hide |
| 10 |
* real content from the operator, which is worse than letting the odd |
| 11 |
* clarification slip through. We require BOTH a recognizable phrase |
| 12 |
* AND a low-content-ish shape (short overall, lots of questions, no |
| 13 |
* `Day N:` heads when itinerary was requested, etc.). |
| 14 |
*/ |
| 15 |
|
| 16 |
/** |
| 17 |
* Phrases the LLM commonly uses when refusing. Lowercased, partial-match. |
| 18 |
* Keep this list short — broader patterns cause false positives on |
| 19 |
* legitimate content that happens to mention "information" or "please". |
| 20 |
*/ |
| 21 |
const CLARIFICATION_PHRASES: string[] = [ |
| 22 |
"i don't have enough", |
| 23 |
"i do not have enough", |
| 24 |
"i need more information", |
| 25 |
"i need more details", |
| 26 |
"i need more context", |
| 27 |
"could you provide", |
| 28 |
"could you please provide", |
| 29 |
"can you provide", |
| 30 |
"please provide", |
| 31 |
"please share", |
| 32 |
"to write accurate", |
| 33 |
"to write specific", |
| 34 |
"once i have these details", |
| 35 |
"once you provide", |
| 36 |
"with these details i'll", |
| 37 |
"with these details i can", |
| 38 |
"without more information", |
| 39 |
"without more details", |
| 40 |
"the operator's notes show empty", |
| 41 |
"operator's notes are empty", |
| 42 |
"i'll need to know", |
| 43 |
"to give you a meaningful", |
| 44 |
]; |
| 45 |
|
| 46 |
export interface ClarificationResult { |
| 47 |
isClarification: boolean; |
| 48 |
/** AI's prose explanation (top of the response, before any questions). */ |
| 49 |
message: string; |
| 50 |
/** Individual questions the AI asked, if any (one per array slot). */ |
| 51 |
questions: string[]; |
| 52 |
} |
| 53 |
|
| 54 |
export function detectClarification( |
| 55 |
rawText: string, |
| 56 |
opts: { expectDayHeads?: boolean } = {}, |
| 57 |
): ClarificationResult { |
| 58 |
const text = (rawText || "").trim(); |
| 59 |
if (text === "") { |
| 60 |
return { isClarification: false, message: "", questions: [] }; |
| 61 |
} |
| 62 |
|
| 63 |
const lower = text.toLowerCase(); |
| 64 |
const matchedPhrase = CLARIFICATION_PHRASES.find((p) => lower.includes(p)); |
| 65 |
if (!matchedPhrase) { |
| 66 |
return { isClarification: false, message: "", questions: [] }; |
| 67 |
} |
| 68 |
|
| 69 |
// If the task expected day heads (itinerary) and we got zero, that's |
| 70 |
// strong evidence — even a partial refusal is unusable. Pair with the |
| 71 |
// phrase match. |
| 72 |
if (opts.expectDayHeads) { |
| 73 |
const hasDayHead = /^\s*(?:#+\s*)?(?:\*+\s*)?day\s+\d+/im.test(text); |
| 74 |
if (hasDayHead) { |
| 75 |
return { isClarification: false, message: "", questions: [] }; |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
// Otherwise require multiple question marks OR a short response — a |
| 80 |
// long form-letter-style answer that happens to mention "please |
| 81 |
// provide" but actually delivers content shouldn't trip the detector. |
| 82 |
const questionCount = (text.match(/\?/g) || []).length; |
| 83 |
const wordCount = text.split(/\s+/).filter(Boolean).length; |
| 84 |
if (!opts.expectDayHeads && questionCount < 2 && wordCount > 80) { |
| 85 |
return { isClarification: false, message: "", questions: [] }; |
| 86 |
} |
| 87 |
|
| 88 |
// Split into a leading prose block (everything before the first |
| 89 |
// question or bullet) and a list of question lines. Tolerates bullet |
| 90 |
// styles: `-`, `*`, `•`. |
| 91 |
const lines = text.split(/\r?\n/); |
| 92 |
const messageLines: string[] = []; |
| 93 |
const questions: string[] = []; |
| 94 |
let pastIntro = false; |
| 95 |
for (const raw of lines) { |
| 96 |
const line = raw.trim(); |
| 97 |
if (line === "") { |
| 98 |
if (messageLines.length > 0) pastIntro = true; |
| 99 |
continue; |
| 100 |
} |
| 101 |
const stripped = line.replace(/^[\s\-\*•·●]+/, "").trim(); |
| 102 |
const looksLikeQuestion = |
| 103 |
stripped.endsWith("?") || |
| 104 |
/^(could|can|please|would|will|what|when|where|who|why|how)\b/i.test( |
| 105 |
stripped, |
| 106 |
); |
| 107 |
const isBullet = /^[\s\-\*•·●]/.test(line); |
| 108 |
if (pastIntro || isBullet || looksLikeQuestion) { |
| 109 |
questions.push(stripped); |
| 110 |
pastIntro = true; |
| 111 |
} else { |
| 112 |
messageLines.push(line); |
| 113 |
} |
| 114 |
} |
| 115 |
|
| 116 |
return { |
| 117 |
isClarification: true, |
| 118 |
message: messageLines.join("\n").trim(), |
| 119 |
questions, |
| 120 |
}; |
| 121 |
} |
| 122 |
|
