A Memory Gate for AI: Policy-Bounded Acceptance in the Executable Cognitive Kernel

generate → score → store

generate → verify → repair or reject → commit

generate → score → store

generate → verify → repair or reject → commit

bad output → stored → reused → reinforced

should_commit = (not self.use_verification) or v.passed

if should_commit:
    self.memory.record(score)

attempt → verify → repair → verify

... he said the meeting would continue ...

{
  "persons": ["he"],
  "organizations": [],
  "locations": []
}

{
  "persons": ["he"],
  "organizations": [],
  "locations": []
}

... reporting from London Newsroom ...

{
  "persons": ["London Newsroom"],
  "organizations": [],
  "locations": []
}

{
  "persons": [],
  "organizations": ["London Newsroom"],
  "locations": []
}

# ==============================================================================
# ECK v2: Policy-Bounded Verifiable Intelligence (SEMANTICALLY ALIGNED)
# ==============================================================================

from __future__ import annotations

import json
import random
import sqlite3
import time
import uuid
import requests
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import Enum
from statistics import mean
from typing import Any, Dict, List, Optional, Tuple, Set

from datasets import load_dataset

# ------------------------------------------------------------------------------
# 1. Core
# ------------------------------------------------------------------------------

class SeverityLevel(Enum):
    CRITICAL = "critical"
    WARNING = "warning"

class ConstraintType(Enum):
    SCHEMA = "schema"
    LOGIC = "logic"

@dataclass
class ConstraintViolation:
    constraint_id: str
    message: str
    severity: SeverityLevel

@dataclass
class VerificationResult:
    passed: bool
    violations: List[ConstraintViolation] = field(default_factory=list)
    confidence: float = 1.0
    is_blocking: bool = False

    def __post_init__(self):
        if any(v.severity == SeverityLevel.CRITICAL for v in self.violations):
            self.passed = False
            self.is_blocking = True

@dataclass
class ExecutionTrace:
    task: str
    action: str
    result: Any
    context: Dict[str, Any]
    constraint_id: str = ""

class Logger:
    def stage(self, msg):
        print(f"\n📦 {msg}")

    def attempt(self, n, action):
        print(f"   🔁 Attempt {n} (action={action})")

    def llm_start(self, model):
        print(f"   🧠 LLM CALL ({model})...")

    def llm_end(self, latency, output):
        print(f"   ⏱️  {latency:.2f}s")
        print(f"   📤 {output[:120].replace(chr(10), ' ')}...")

    def verify(self, result):
        print(f"   🔍 Verification: {'PASS' if result.passed else 'FAIL'}")
        for v in result.violations:
            print(f"      - {v.message} ({v.severity.value})")

    def score(self, f1):
        print(f"   🎯 F1 Score: {f1:.2f}")

    def commit(self, yes):
        print(f"   💾 Commit: {'YES' if yes else 'NO'}")

log = Logger()

# ------------------------------------------------------------------------------
# 2. Policy
# ------------------------------------------------------------------------------

class ConstraintPolicy:
    def __init__(self):
        self.constraints = []

    def define_constraint(self, ctype, severity, rule):
        self.constraints.append({
            "type": ctype,
            "severity": severity,
            "rule": rule,
        })

# ------------------------------------------------------------------------------
# 3. Memory
# ------------------------------------------------------------------------------

class Memory:
    def __init__(self):
        self.data = []

    def record(self, score):
        self.data.append(score)

# ------------------------------------------------------------------------------
# 4. Verification
# ------------------------------------------------------------------------------

class ConstraintEvaluator(ABC):
    @abstractmethod
    def evaluate(self, trace: ExecutionTrace, rule: Dict) -> VerificationResult:
        pass

class SchemaEvaluator(ConstraintEvaluator):
    def evaluate(self, trace, rule):
        try:
            obj = json.loads(trace.result) if isinstance(trace.result, str) else trace.result
        except (json.JSONDecodeError, ValueError, TypeError):
            return VerificationResult(False, [ConstraintViolation("schema", "Invalid JSON", SeverityLevel.CRITICAL)])

        for f in ["persons", "organizations", "locations"]:
            if f not in obj or not isinstance(obj[f], list):
                return VerificationResult(False, [ConstraintViolation("schema", f"Missing or invalid {f}", SeverityLevel.CRITICAL)])

        return VerificationResult(True)

class SemanticEvaluator(ConstraintEvaluator):
    def evaluate(self, trace, rule):
        truth = trace.context["truth"]
        pred = parse_prediction(trace.result)

        if pred is None:
            return VerificationResult(
                passed=False,
                violations=[
                    ConstraintViolation(
                        "semantic",
                        "Output is not valid JSON",
                        SeverityLevel.CRITICAL,
                    )
                ],
                confidence=0.0,
                is_blocking=True,
            )

        violations = []

        for key in ["persons", "organizations", "locations"]:
            pred_set = set(x.lower() for x in pred.get(key, []))
            truth_set = set(x.lower() for x in truth.get(key, []))

            for item in pred_set - truth_set:
                violations.append(
                    ConstraintViolation(
                        "semantic",
                        f"Hallucinated {key}: {item}",
                        SeverityLevel.WARNING,
                    )
                )

            for item in truth_set - pred_set:
                violations.append(
                    ConstraintViolation(
                        "semantic",
                        f"Missing {key}: {item}",
                        SeverityLevel.WARNING,
                    )
                )

        f1 = compute_f1(pred, truth)

        if f1 < rule["min_f1"]:
            violations.append(
                ConstraintViolation(
                    "semantic",
                    f"Low F1={f1:.2f}",
                    SeverityLevel.WARNING,
                )
            )

        # IMPORTANT:
        # semantic violations should FAIL verification,
        # but not necessarily block execution
        if violations:
            return VerificationResult(
                passed=False,
                violations=violations,
                confidence=f1,
                is_blocking=False,
            )

        return VerificationResult(
            passed=True,
            violations=[],
            confidence=f1,
            is_blocking=False,
        )

class Verifier:
    def __init__(self, policy):
        self.policy = policy
        self.schema = SchemaEvaluator()
        self.semantic = SemanticEvaluator()

    def verify(self, trace):
        results = []

        for c in self.policy.constraints:
            if c["type"] == ConstraintType.SCHEMA:
                results.append(self.schema.evaluate(trace, c["rule"]))
            else:
                results.append(self.semantic.evaluate(trace, c["rule"]))

        final = VerificationResult(True)

        for r in results:
            if not r.passed:
                final.passed = False
                final.violations.extend(r.violations)
                if r.is_blocking:
                    final.is_blocking = True

        return final

# ------------------------------------------------------------------------------
# 5. LLM
# ------------------------------------------------------------------------------

def run_llm(prompt, log):
    log.llm_start("mistral")
    start = time.time()

    try:
        res = requests.post(
            "http://localhost:11434/api/generate",
            json={"model": "mistral", "prompt": prompt, "stream": False},
            timeout=120,
        )
        res.raise_for_status()
        output = res.json().get("response", "")
    except requests.exceptions.RequestException as e:
        output = f"ERROR: {e}"

    log.llm_end(time.time() - start, output)
    return output

def build_initial_prompt(text):
    return f"""
You are a named entity extraction system.

Extract ALL named entities EXACTLY as written.

Return STRICT JSON:
{{
  "persons": [],
  "organizations": [],
  "locations": []
}}

Rules:
- Preserve exact text spans
- No guessing
- No explanation
- Output ONLY JSON

Text:
{text}
""".strip()

def build_repair_prompt(text, previous_output, violations):
    issues = "\n".join(f"- {v.message}" for v in violations)

    return f"""
You previously returned:

{previous_output}

It failed for the following reasons:
{issues}

Fix the output. Do not repeat the same mistakes.

Rules:
- Keep correct entities
- Remove hallucinated entities
- Add missing entities ONLY if they appear exactly in text
- Do NOT guess
- Output ONLY valid JSON

Schema:
{{
  "persons": [],
  "organizations": [],
  "locations": []
}}

Text:
{text}
""".strip()


# ------------------------------------------------------------------------------
# 6. Scoring
# ------------------------------------------------------------------------------

def parse_prediction(result):
    if not result:
        return None

    text = result.strip()

    # Remove markdown fences
    if "```" in text:
        parts = text.split("```")
        for part in parts:
            part = part.strip()
            if part.startswith("{") and part.endswith("}"):
                text = part
                break

    # Try direct parse
    try:
        return json.loads(text)
    except (json.JSONDecodeError, ValueError):
        return None

    # Try extracting JSON substring
    start = text.find("{")
    end = text.rfind("}")
    if start != -1 and end != -1:
        try:
            return json.loads(text[start:end+1])
        except:
            return None

    return None

def compute_f1(pred, truth):
    def norm(x): return set(i.lower() for i in x)

    scores = []
    for k in truth:
        p, t = norm(pred.get(k, [])), norm(truth[k])
        if not p and not t:
            scores.append(1)
            continue
        inter = len(p & t)
        prec = inter / len(p) if p else 0
        rec = inter / len(t) if t else 0
        scores.append(0 if prec+rec==0 else 2*prec*rec/(prec+rec))
    return sum(scores)/len(scores)

# ------------------------------------------------------------------------------
# 7. Ground Truth
# ------------------------------------------------------------------------------

def build_truth(item):
    tokens, tags = item["tokens"], item["ner_tags"]
    mapping = {1:"persons",2:"persons",3:"organizations",4:"organizations",5:"locations",6:"locations"}

    out = {"persons":[], "organizations":[], "locations":[]}
    cur, typ = [], None

    for tok, tag in zip(tokens, tags):
        if tag in mapping:
            t = mapping[tag]
            if tag in [1,3,5]:
                if cur:
                    out[typ].append(" ".join(cur))
                cur, typ = [tok], t
            else:
                cur.append(tok)
        else:
            if cur:
                out[typ].append(" ".join(cur))
                cur, typ = [], None

    if cur:
        out[typ].append(" ".join(cur))

    return out

# ------------------------------------------------------------------------------
# 8. Kernel
# ------------------------------------------------------------------------------

class Kernel:
    def __init__(self, verifier, memory, use_verification=True):
        self.verifier = verifier
        self.memory = memory
        self.use_verification = use_verification

        self.retry_count = 0
        self.blocking_failures = 0
        self.semantic_failures = 0
        self.repair_prompt_uses = 0
        self.pass_count = 0

        self.total_runs = 0
        self.commit_count = 0
        self.reject_count = 0
        self.failed_verifications = 0
        self.repair_success_count = 0
        self.initial_failures = 0

        self.committed_scores: List[float] = []

    def run(self, text, truth, log: Logger):
        self.total_runs += 1

        final_result = None
        v = VerificationResult(True)

        first_attempt_failed = False
        repaired_successfully = False

        for attempt in range(3):
            action_name = "extract" if attempt == 0 else "repair"
            log.attempt(attempt, action_name)

            if attempt == 0:
                prompt = build_initial_prompt(text)
                print("   📝 Using initial prompt")
            else:
                prompt = build_repair_prompt(text, final_result, v.violations)
                self.repair_prompt_uses += 1
                print("   🛠 Using repair prompt")

            result = run_llm(prompt, log)

            trace = ExecutionTrace(
                task="extract",
                action=action_name,
                result=result,
                context={"truth": truth},
            )

            v = self.verifier.verify(trace)
            log.verify(v)

            if not v.passed:
                self.failed_verifications += 1

                if attempt == 0:
                    self.initial_failures += 1
                    first_attempt_failed = True

                if v.is_blocking:
                    self.blocking_failures += 1
                else:
                    self.semantic_failures += 1

            final_result = result

            if v.passed:
                self.pass_count += 1

                if first_attempt_failed:
                    repaired_successfully = True
                    self.repair_success_count += 1

                break

            if attempt < 2:
                self.retry_count += 1

        pred = parse_prediction(final_result)
        score = compute_f1(pred, truth) if pred else 0.0

        log.score(score)

        should_commit = (not self.use_verification) or v.passed
        log.commit(should_commit)

        if should_commit:
            self.memory.record(score)
            self.committed_scores.append(score)
            self.commit_count += 1
        else:
            self.reject_count += 1

        return score
# ------------------------------------------------------------------------------
# 9. Experiment
# ------------------------------------------------------------------------------

def run_experiment(mode, eval_items):
    policy = ConstraintPolicy()

    policy.define_constraint(ConstraintType.SCHEMA, SeverityLevel.CRITICAL, {})
    policy.define_constraint(ConstraintType.LOGIC, SeverityLevel.WARNING, {"min_f1": 0.5})

    verifier = Verifier(policy)
    memory = Memory()

    kernel = Kernel(verifier, memory, use_verification=(mode == "verified"))

    scores = []

    for i, item in enumerate(eval_items):
        log.stage(f"Task {i+1}/{len(eval_items)}")

        text = " ".join(item["tokens"])
        truth = build_truth(item)

        print(f"   Input: {text[:100]}...")

        score = kernel.run(text, truth, log)
        scores.append(score)

    avg_f1 = mean(scores) if scores else 0.0

    committed_scores = kernel.committed_scores
    bad_committed = sum(1 for s in committed_scores if s < 0.5)
    clean_committed = sum(1 for s in committed_scores if s >= 0.5)
    n_committed = len(committed_scores)

    bad_trace_admission = (
        bad_committed / n_committed if n_committed else 0.0
    )

    repair_success_rate = (
        kernel.repair_success_count / kernel.initial_failures
        if kernel.initial_failures else 0.0
    )

    rejection_rate = (
        kernel.reject_count / kernel.total_runs
        if kernel.total_runs else 0.0
    )

    clean_memory_rate = (
        clean_committed / n_committed if n_committed else 0.0
    )

    print(f"\n📊 {mode.upper()} SUMMARY")
    print(f"   Avg F1: {avg_f1:.2f}")
    print(f"   Stored traces: {len(memory.data)}")

    print("\n   🧠 SYSTEM METRICS")
    print(f"   Total Runs: {kernel.total_runs}")
    print(f"   Commits: {kernel.commit_count}")
    print(f"   Rejections: {kernel.reject_count}")

    print("\n   🔍 VERIFICATION")
    print(f"   Failed Verifications: {kernel.failed_verifications}")
    print(f"   Blocking Failures: {kernel.blocking_failures}")
    print(f"   Semantic Failures: {kernel.semantic_failures}")

    print("\n   🔁 REPAIR")
    print(f"   Repair Attempts: {kernel.repair_prompt_uses}")
    print(f"   Repair Success Rate: {repair_success_rate:.2f}")

    print("\n   🧪 QUALITY")
    print(f"   Bad Trace Admission: {bad_trace_admission:.2f}")
    print(f"   Rejection Rate: {rejection_rate:.2f}")
    print(f"   Clean Memory Rate: {clean_memory_rate:.2f}")

    return {
        "avg_f1": avg_f1,
        "bad_trace_admission": bad_trace_admission,
        "repair_success_rate": repair_success_rate,
        "rejection_rate": rejection_rate,
        "clean_memory_rate": clean_memory_rate,
    }    

# ------------------------------------------------------------------------------
# 10. Main
# ------------------------------------------------------------------------------

if __name__ == "__main__":
    random.seed(42)

    dataset = load_dataset("conll2003", split="train", trust_remote_code=True)
    data = [dataset[i] for i in range(1000)]

    # Freeze the exact evaluation slice ONCE
    eval_items = random.sample(data, 100)

    print("\n🔵 STANDARD")
    res_std = run_experiment("standard", eval_items)

    print("\n🟢 VERIFIED")
    res_ver = run_experiment("verified", eval_items)

    print("\nRESULT:")
    print(f"Standard F1: {res_std['avg_f1']:.2f}")
    print(f"Verified F1: {res_ver['avg_f1']:.2f}")
    print(f"Delta: {res_ver['avg_f1'] - res_std['avg_f1']:+.2f}")

pip install datasets<4
pip install requests
python eck_verified_demo.py

📊 VERIFIED SUMMARY
   Avg F1: 0.78
   Stored traces: 56

   🧠 SYSTEM METRICS
   Total Runs: 100
   Commits: 56
   Rejections: 44

   🔍 VERIFICATION
   Failed Verifications: 157
   Blocking Failures: 21
   Semantic Failures: 136

   🔁 REPAIR
   Repair Attempts: 113
   Repair Success Rate: 0.28

   🧪 QUALITY
   Bad Trace Admission: 0.00
   Rejection Rate: 0.44
   Clean Memory Rate: 1.00

RESULT:
Standard F1: 0.77
Verified F1: 0.78
Delta: +0.01