Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 40 additions & 24 deletions src/spark_character/critic.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,20 @@ def load_critic(version: str = DEFAULT_CRITIC_VERSION) -> CriticSpec:


def _build_critic_user_prompt(persona: PersonaSpec, draft: str) -> str:
return (
"[Persona spec]\n"
f"{persona.system_prompt}\n\n"
"[Draft reply]\n"
f"{draft}\n\n"
"Apply the rules. Return PASS or the rewritten reply only."
)
if not isinstance(draft, str): draft = str(draft or '')
try:
return (
"[Persona spec]\n"
f"{persona.system_prompt}\n\n"
"[Draft reply]\n"
f"{draft}\n\n"
"Apply the rules. Return PASS or the rewritten reply only."
)



except Exception:
return ""
def critique(
*,
provider: ProviderSpec,
Expand All @@ -64,17 +69,22 @@ def critique(
temperature: float = 0.2,
max_tokens: int = 600,
) -> CritiqueResult:
user_prompt = _build_critic_user_prompt(persona, draft)
response = call_provider(
provider=provider,
system_prompt=critic.system_prompt,
user_prompt=user_prompt,
max_tokens=max_tokens,
temperature=temperature,
)
return _interpret(draft, response)


if not isinstance(draft, str): draft = str(draft or '')
try:
user_prompt = _build_critic_user_prompt(persona, draft)
response = call_provider(
provider=provider,
system_prompt=critic.system_prompt,
user_prompt=user_prompt,
max_tokens=max_tokens,
temperature=temperature,
)
return _interpret(draft, response)



except Exception:
return None
async def critique_async(
*,
provider: ProviderSpec,
Expand All @@ -96,9 +106,15 @@ async def critique_async(


def _interpret(draft: str, response: str) -> CritiqueResult:
cleaned = response.strip()
if not cleaned:
return CritiqueResult(final=draft, rewritten=False, draft=draft)
if cleaned.upper().startswith(PASS_TOKEN) and len(cleaned) <= 8:
return CritiqueResult(final=draft, rewritten=False, draft=draft)
return CritiqueResult(final=cleaned, rewritten=True, draft=draft)
if not isinstance(draft, str): draft = str(draft or '')
if not isinstance(response, str): response = str(response or '')
try:
cleaned = response.strip()
if not cleaned:
return CritiqueResult(final=draft, rewritten=False, draft=draft)
if cleaned.upper().startswith(PASS_TOKEN) and len(cleaned) <= 8:
return CritiqueResult(final=draft, rewritten=False, draft=draft)
return CritiqueResult(final=cleaned, rewritten=True, draft=draft)

except Exception:
return None
107 changes: 58 additions & 49 deletions src/spark_character/deeper_probes.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,57 +545,66 @@ def run_deep_probe(
judge_provider: ProviderSpec | None = None,
max_tokens: int = 600,
) -> DeepProbeResult:
p = persona or load_persona()
history: list[dict[str, str]] = []
transcript: list[tuple[str, str]] = []
for user_msg in probe.turns:
result = generate(
user_msg,
provider=provider,
persona=p,
history=list(history) if history else None,
max_tokens=max_tokens,
try:
p = persona or load_persona()
history: list[dict[str, str]] = []
transcript: list[tuple[str, str]] = []
for user_msg in probe.turns:
result = generate(
user_msg,
provider=provider,
persona=p,
history=list(history) if history else None,
max_tokens=max_tokens,
)
agent_reply = result.final
history.append({"role": "user", "content": user_msg})
history.append({"role": "assistant", "content": agent_reply})
transcript.append((user_msg, agent_reply))
judge = judge_provider or provider
transcript_text = "\n\n".join(f"USER: {u}\nSPARK: {a}" for u, a in transcript)
judge_user = (
"[Conversation transcript]\n"
f"{transcript_text}\n\n"
"[Question]\n"
f"{probe.judge_question}\n\n"
"Return SCORE=<integer 0-10> only."
)
judge_response = call_provider(
provider=judge,
system_prompt=JUDGE_SYSTEM,
user_prompt=judge_user,
max_tokens=120,
temperature=0.0,
disable_thinking=True,
)
raw = _parse_score(judge_response)
return DeepProbeResult(
probe_id=probe.id,
tier=probe.tier,
trait=probe.trait,
transcript=tuple(transcript),
score=raw / 10.0,
raw=raw,
judge_response=judge_response,
)
agent_reply = result.final
history.append({"role": "user", "content": user_msg})
history.append({"role": "assistant", "content": agent_reply})
transcript.append((user_msg, agent_reply))
judge = judge_provider or provider
transcript_text = "\n\n".join(f"USER: {u}\nSPARK: {a}" for u, a in transcript)
judge_user = (
"[Conversation transcript]\n"
f"{transcript_text}\n\n"
"[Question]\n"
f"{probe.judge_question}\n\n"
"Return SCORE=<integer 0-10> only."
)
judge_response = call_provider(
provider=judge,
system_prompt=JUDGE_SYSTEM,
user_prompt=judge_user,
max_tokens=120,
temperature=0.0,
disable_thinking=True,
)
raw = _parse_score(judge_response)
return DeepProbeResult(
probe_id=probe.id,
tier=probe.tier,
trait=probe.trait,
transcript=tuple(transcript),
score=raw / 10.0,
raw=raw,
judge_response=judge_response,
)



except Exception:
return None
def _parse_score(text: str) -> int:
if not text:
if not isinstance(text, str): text = str(text or '')
try:
if not text:
return 5
match = re.search(r"SCORE\s*=\s*(\d+)", text, re.IGNORECASE)
if match:
return max(0, min(10, int(match.group(1))))
digits = re.findall(r"\b([0-9]|10)\b", text)
if digits:
return max(0, min(10, int(digits[0])))
return 5
match = re.search(r"SCORE\s*=\s*(\d+)", text, re.IGNORECASE)
if match:
return max(0, min(10, int(match.group(1))))
digits = re.findall(r"\b([0-9]|10)\b", text)
if digits:
return max(0, min(10, int(digits[0])))
return 5

except Exception:
return 0