Prompt Reliability Checker

# Hermes Dispatch Tool — Prompt Reliability Checker
# Source: https://hermesdispatch.dev/tools/prompt-reliability-checker/
# Description: Run a system prompt multiple times against a local LLM and score consistency.
# License: MIT (generated by hermesdispatch.dev)
#
# INSTALL:
#   1. Save this file as ~/.hermes/hermes-agent/tools/prompt_reliability_checker.py
#   2. Restart Hermes or run /reset in a session
#
# MANUAL REGISTRY:
#   from tools.prompt_reliability_checker import register
#   register()

import json
import urllib.request

def _ok(result):
    return json.dumps({"success": True, "data": result}, indent=2)

def _err(message):
    return json.dumps({"success": False, "error": message}, indent=2)

TOOL_NAME = "prompt_reliability_checker"
TOOLSET = "agents"

SCHEMA = {
    "type": "function",
    "function": {
        "name": TOOL_NAME,
        "description": "Run a system prompt N times against a local Ollama LLM and report consistency, failure rate, and unique outputs.",
        "parameters": {
            "type": "object",
            "properties": {
                "base_url": {"type": "string", "default": "http://localhost:11434", "description": "Ollama base URL."},
                "model": {"type": "string", "default": "llama3.2", "description": "Ollama model name."},
                "system_prompt": {"type": "string", "default": "Reply YES or NO only.", "description": "System prompt to test."},
                "user_message": {"type": "string", "default": "Is red wine heart-healthy?", "description": "User message to send each run."},
                "runs": {"type": "integer", "default": 5, "description": "Number of runs (max 20)."},
                "temperature": {"type": "number", "default": 0.7, "description": "Model temperature."},
            },
            "required": []
        }
    }
}

def generate(base_url, model, system_prompt, user_message, temperature):
    data = json.dumps({
        "model": model,
        "system": system_prompt,
        "prompt": user_message,
        "stream": False,
        "options": {"temperature": temperature}
    }).encode()
    req = urllib.request.Request(
        f"{base_url}/api/generate",
        data=data,
        headers={"Content-Type": "application/json"},
        method="POST"
    )
    with urllib.request.urlopen(req, timeout=120) as resp:
        return json.loads(resp.read().decode()).get("response", "")

def check_reliability(base_url="http://localhost:11434", model="llama3.2",
                      system_prompt="Reply YES or NO only.", user_message="Is red wine heart-healthy?",
                      runs=5, temperature=0.7):
    runs = min(int(runs), 20)
    responses = []
    failures = 0
    for _ in range(runs):
        try:
            r = generate(base_url, model, system_prompt, user_message, temperature).strip()
        except Exception as e:
            r = ""
            failures += 1
        responses.append(r)

    non_empty = [r for r in responses if r]
    unique = list(set(non_empty))
    consistency = (len(non_empty) / len(unique) * 100) if unique else 0
    return _ok({
        "responses": responses,
        "consistency_pct": round(consistency, 1),
        "failure_rate": f"{failures}/{runs}",
        "unique_outputs": len(unique),
    })

def HANDLER(args):
    try:
        return check_reliability(
            base_url=args.get("base_url", "http://localhost:11434"),
            model=args.get("model", "llama3.2"),
            system_prompt=args.get("system_prompt", "Reply YES or NO only."),
            user_message=args.get("user_message", "Is red wine heart-healthy?"),
            runs=int(args.get("runs", 5)),
            temperature=float(args.get("temperature", 0.7)),
        )
    except Exception as e:
        return _err(str(e))

def register():
    try:
        from tools.registry import registry
        registry.register(name=TOOL_NAME, toolset=TOOLSET, schema=SCHEMA, handler=HANDLER)
    except ImportError:
        print("Hermes registry not found; skipping manual registration.")

if __name__ == "__main__":
    print(HANDLER({}))