From 52c56f7fede099117008aed98f34c36800fc5337 Mon Sep 17 00:00:00 2001 From: Jacob Mathison Date: Tue, 21 Jul 2026 11:25:45 -0500 Subject: [PATCH] feat: add compact dictionary codebook --- .gitignore | 2 + AGENTS.md | 17 +- Cargo.lock | 74 ++++++- Cargo.toml | 1 + README.md | 16 +- dictionary/static-base.json | 20 ++ llm.txt | 7 +- manifest.llm.json | 30 ++- scripts/precommit-check.sh | 20 ++ src/dictionary.rs | 388 ++++++++++++++++++++++++++++++++++-- src/dispatcher.rs | 4 + src/llm.rs | 129 +++++++++++- src/main.rs | 18 +- 13 files changed, 673 insertions(+), 53 deletions(-) create mode 100644 dictionary/static-base.json diff --git a/.gitignore b/.gitignore index 07fd07f..e870cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,7 @@ /runtime/ /dictionary/actions.jsonl /dictionary/actions.index.json +/dictionary/model.codebook.json +/dictionary/model.examples.jsonl *.log .DS_Store diff --git a/AGENTS.md b/AGENTS.md index 41dfd8e..1a734ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project -`mp-ai-module-controller` is a Rust CLI scaffold for local llama.cpp-driven action dispatch. It starts a local `llama-server`, generates an action dictionary for small-model training and runtime lookup, queries the server through its OpenAI-compatible chat endpoint, and dispatches exact-match Rust scripts. +`mp-ai-module-controller` is a Rust CLI scaffold for local llama.cpp-driven action dispatch. It starts a local `llama-server`, generates verbose and compact action dictionaries for small-model training/runtime lookup, queries the server through its OpenAI-compatible chat endpoint, and dispatches exact-match Rust scripts. ## Commands @@ -32,15 +32,20 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ ## Dictionary Rules -- `src/registry.rs` is the source of truth for action definitions. -- `dictionary/actions.jsonl` is generated canonical training data. -- `dictionary/actions.index.json` is the generated runtime lookup file. +- `src/registry.rs` is the source of truth for executable action definitions. +- `dictionary/static-base.json` is the tracked static compact-code base. Change it only when the compact protocol changes. +- `dictionary/actions.jsonl` is generated canonical verbose training/action data. +- `dictionary/actions.index.json` is the generated verbose runtime lookup file. +- `dictionary/model.codebook.json` is the generated model-facing compact codebook. +- `dictionary/model.examples.jsonl` is generated template/example data for future synthetic-data tooling. - Regenerate dictionary files with `cargo run -- dictionary generate` after changing registered actions. - Do not manually edit generated dictionary files. +- Generated metadata includes `dictionary_version`, `registry_checksum`, `static_base_checksum`, and `codebook_checksum`. ## Dispatch Rules - Dispatch only exact `action_id` values present in `dictionary/actions.index.json`. +- Compact model outputs must include matching `v` and `c` values from `dictionary/model.codebook.json` before expansion. - Aliases and intent examples are training hints only; they are not executable ids. - Unknown, malformed, ambiguous, missing, or empty action ids must be rejected without running scripts. - Script processes are detached. The controller spawns them and does not monitor completion. @@ -53,6 +58,8 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ - `runtime/` - `dictionary/actions.jsonl` - `dictionary/actions.index.json` +- `dictionary/model.codebook.json` +- `dictionary/model.examples.jsonl` ## CI And Hooks @@ -61,7 +68,7 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ - Woodpecker workflow lives at `.woodpecker.yml`. - The committed pre-commit hook lives at `.githooks/pre-commit`. - Configure local hooks with `git config core.hooksPath .githooks`. -- All CI and pre-commit checks should call `scripts/precommit-check.sh` so dictionary generation stays validated consistently. +- All CI and pre-commit checks should call `scripts/precommit-check.sh` so full dictionary generation stays validated consistently. ## Coding Guidelines diff --git a/Cargo.lock b/Cargo.lock index 181ec7b..0ef389e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -100,6 +100,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -141,7 +150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core", ] @@ -209,6 +218,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -218,6 +236,26 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -315,6 +353,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -675,6 +723,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "shell-words", "tempfile", "tracing", @@ -1030,6 +1079,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1336,6 +1396,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1378,6 +1444,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index f36b58b..761a5d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ dotenvy = "0.15" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10" shell-words = "1.1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/README.md b/README.md index 83aed3d..ac1282d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mp-ai-module-controller -Rust controller for starting a local `llama.cpp` server, asking it to map natural-language requests to known local actions, and dispatching exact-match Rust scripts. +Rust controller for starting a local `llama.cpp` server, asking it to map natural-language requests to known local actions, and dispatching exact-match Rust scripts. It supports both verbose action JSON and compact dictionary-coded outputs for hypersmall local models. ## Setup @@ -42,8 +42,12 @@ Run `cargo run -- dictionary generate` to create: - `dictionary/actions.jsonl`: canonical training dictionary, one action record per line. - `dictionary/actions.index.json`: compact runtime lookup index. +- `dictionary/model.codebook.json`: static-plus-adaptive compact codebook. +- `dictionary/model.examples.jsonl`: hand-authored examples/templates for future synthetic-data tooling. -The runtime only dispatches exact `action_id` matches from `actions.index.json`. Aliases and examples are training hints, not executable ids. +`dictionary/static-base.json` is the tracked static base dictionary. Generated metadata includes `dictionary_version`, `registry_checksum`, `static_base_checksum`, and `codebook_checksum`. + +The runtime only dispatches exact `action_id` matches from `actions.index.json`. Aliases and examples are training hints, not executable ids. Compact model outputs must match the generated codebook version and checksum before expansion. ## LLM Response Contract @@ -53,7 +57,13 @@ The local model must return compact JSON: {"action_id":"logger","payload":{"message":"hello"}} ``` -Malformed JSON, missing `action_id`, empty `action_id`, aliases, and unknown action ids are rejected without running anything. +It may also return compact codebook JSON: + +```json +{"v":"1.0.0","c":"","a":"A001","p":{"F001":"hello"}} +``` + +Malformed JSON, missing action codes, stale versions/checksums, aliases, and unknown action ids are rejected without running anything. ## Initial Script diff --git a/dictionary/static-base.json b/dictionary/static-base.json new file mode 100644 index 0000000..cd6a98f --- /dev/null +++ b/dictionary/static-base.json @@ -0,0 +1,20 @@ +{ + "dictionary_version": "1.0.0", + "keys": { + "K001": "a", + "K002": "p", + "K003": "v", + "K004": "c" + }, + "types": { + "T001": "action", + "T002": "field", + "T003": "error", + "T004": "module" + }, + "errors": { + "E001": "unknown_action", + "E002": "invalid_payload", + "E003": "checksum_mismatch" + } +} diff --git a/llm.txt b/llm.txt index 2a502bd..c793f63 100644 --- a/llm.txt +++ b/llm.txt @@ -1,12 +1,15 @@ # mp-ai-module-controller -Rust CLI for local llama.cpp action dispatch. +Rust CLI for local llama.cpp action dispatch with verbose and compact dictionary-coded model outputs. Important files: - `src/main.rs`: CLI entrypoint. - `src/registry.rs`: source-of-truth action registry. - `src/dictionary.rs`: generates `dictionary/actions.jsonl` and `dictionary/actions.index.json`. +- `dictionary/static-base.json`: tracked static compact-code base. +- `dictionary/model.codebook.json`: generated compact codebook. +- `dictionary/model.examples.jsonl`: generated templates/examples for future synthetic-data tooling. - `src/llm.rs`: sends OpenAI-compatible chat requests to the local llama.cpp server and parses action JSON. - `src/dispatcher.rs`: exact-match runtime dispatch and detached script spawning. - `src/scripts/logger.rs`: proof-of-concept logger script. @@ -37,7 +40,7 @@ Environment: - `LLAMA_EXTRA_ARGS`: optional extra llama-server args. - `CONTROLLER_LOG_LEVEL`: default `info`. -The LLM must return compact JSON like `{"action_id":"logger","payload":{"message":"hello"}}`. The runtime only executes exact `action_id` matches from `dictionary/actions.index.json`; aliases are not executable. Invalid or unknown outputs are rejected without running anything. +The LLM may return verbose JSON like `{"action_id":"logger","payload":{"message":"hello"}}` or compact codebook JSON like `{"v":"1.0.0","c":"","a":"A001","p":{"F001":"hello"}}`. Compact outputs must match `dictionary/model.codebook.json` version and checksum before expansion. The runtime only executes exact `action_id` matches from `dictionary/actions.index.json`; aliases are not executable. Invalid or unknown outputs are rejected without running anything. Codex environment: `.codex/environments/environment.toml` defines setup plus Build, Test, Generate Dictionary, Precommit Check, Run Logger, Query Logger, and Serve Llama actions. diff --git a/manifest.llm.json b/manifest.llm.json index 9690e31..bd3bedf 100644 --- a/manifest.llm.json +++ b/manifest.llm.json @@ -75,20 +75,37 @@ "fallback_binary": "llama-server", "api": "OpenAI-compatible /v1/chat/completions", "response_contract": { - "format": "compact JSON", + "format": "verbose JSON or compact codebook JSON", "example": { "action_id": "logger", "payload": { "message": "hello" } + }, + "compact_example": { + "v": "1.0.0", + "c": "", + "a": "A001", + "p": { + "F001": "hello" + } } } }, "dictionary": { "source_of_truth": "src/registry.rs", + "static_base": "dictionary/static-base.json", "generated_jsonl": "dictionary/actions.jsonl", "generated_index": "dictionary/actions.index.json", - "runtime_policy": "Only exact action_id matches from the generated index are executable." + "generated_codebook": "dictionary/model.codebook.json", + "generated_examples": "dictionary/model.examples.jsonl", + "dictionary_version": "1.0.0", + "metadata": [ + "registry_checksum", + "static_base_checksum", + "codebook_checksum" + ], + "runtime_policy": "Verbose outputs require exact action_id matches; compact outputs require matching version/checksum and exact codebook expansion before dispatch." }, "actions": [ { @@ -106,7 +123,9 @@ "logs/", "runtime/", "dictionary/actions.jsonl", - "dictionary/actions.index.json" + "dictionary/actions.index.json", + "dictionary/model.codebook.json", + "dictionary/model.examples.jsonl" ], "ci": { "codex_environment": ".codex/environments/environment.toml", @@ -119,7 +138,10 @@ "cargo test", "cargo run -- dictionary generate", "test -s dictionary/actions.jsonl", - "test -s dictionary/actions.index.json" + "test -s dictionary/actions.index.json", + "test -s dictionary/model.codebook.json", + "test -s dictionary/model.examples.jsonl", + "test -s dictionary/static-base.json" ] }, "agent_guidance": "See AGENTS.md. Keep src/registry.rs as the source of truth, regenerate dictionary files after action changes, and reject non-exact LLM action outputs." diff --git a/scripts/precommit-check.sh b/scripts/precommit-check.sh index 84125ac..1df8382 100755 --- a/scripts/precommit-check.sh +++ b/scripts/precommit-check.sh @@ -9,5 +9,25 @@ cargo run -- dictionary generate test -s dictionary/actions.jsonl test -s dictionary/actions.index.json +test -s dictionary/model.codebook.json +test -s dictionary/model.examples.jsonl +test -s dictionary/static-base.json + +python3 - <<'PY' +import json +from pathlib import Path + +for path in [ + "dictionary/actions.index.json", + "dictionary/model.codebook.json", + "dictionary/static-base.json", +]: + json.loads(Path(path).read_text()) + +for path in ["dictionary/actions.jsonl", "dictionary/model.examples.jsonl"]: + for line_number, line in enumerate(Path(path).read_text().splitlines(), start=1): + if line.strip(): + json.loads(line) +PY echo "precommit check complete" diff --git a/src/dictionary.rs b/src/dictionary.rs index 214abeb..2645326 100644 --- a/src/dictionary.rs +++ b/src/dictionary.rs @@ -1,6 +1,7 @@ -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File}; use std::io::Write; #[cfg(test)] @@ -9,6 +10,8 @@ use std::path::{Path, PathBuf}; use crate::registry::ActionDefinition; +pub const DICTIONARY_SCHEMA_VERSION: &str = "1.0"; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ActionDictionaryRecord { pub action_id: String, @@ -22,14 +25,56 @@ pub struct ActionDictionaryRecord { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ActionIndex { pub schema_version: String, + pub dictionary_version: String, + pub registry_checksum: String, + pub static_base_checksum: String, + pub codebook_checksum: String, pub actions: BTreeMap, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StaticBaseDictionary { + pub dictionary_version: String, + pub keys: BTreeMap, + pub types: BTreeMap, + pub errors: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct AdaptiveCodebook { + pub actions: BTreeMap, + pub fields: BTreeMap, + pub modules: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ModelCodebook { + pub schema_version: String, + pub dictionary_version: String, + pub registry_checksum: String, + pub static_base_checksum: String, + pub codebook_checksum: String, + pub static_base: StaticBaseDictionary, + pub adaptive: AdaptiveCodebook, +} + +#[derive(Debug, Clone, Serialize)] +struct ModelCodebookForChecksum<'a> { + schema_version: &'a str, + dictionary_version: &'a str, + registry_checksum: &'a str, + static_base_checksum: &'a str, + static_base: &'a StaticBaseDictionary, + adaptive: &'a AdaptiveCodebook, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DictionarySummary { pub action_count: usize, pub jsonl_path: PathBuf, pub index_path: PathBuf, + pub codebook_path: PathBuf, + pub examples_path: PathBuf, } pub fn default_dictionary_dir() -> PathBuf { @@ -43,37 +88,49 @@ pub fn generate_dictionary_files( fs::create_dir_all(out_dir) .with_context(|| format!("creating dictionary directory {}", out_dir.display()))?; + let static_base = load_static_base_dictionary(out_dir)?; + validate_static_base_dictionary(&static_base)?; + + let records = sorted_records(actions); + let registry_checksum = checksum_json(&records)?; + let static_base_checksum = checksum_json(&static_base)?; + let adaptive = build_adaptive_codebook(&records); + let codebook = build_model_codebook( + static_base, + adaptive, + registry_checksum, + static_base_checksum, + )?; + let jsonl_path = out_dir.join("actions.jsonl"); let index_path = out_dir.join("actions.index.json"); - let records = actions - .iter() - .cloned() - .map(ActionDictionaryRecord::from) - .collect::>(); + let codebook_path = out_dir.join("model.codebook.json"); + let examples_path = out_dir.join("model.examples.jsonl"); - let mut jsonl = - File::create(&jsonl_path).with_context(|| format!("creating {}", jsonl_path.display()))?; - for record in &records { - serde_json::to_writer(&mut jsonl, record)?; - jsonl.write_all(b"\n")?; - } + write_actions_jsonl(&jsonl_path, &records)?; let index = ActionIndex { - schema_version: "1.0".to_string(), + schema_version: DICTIONARY_SCHEMA_VERSION.to_string(), + dictionary_version: codebook.dictionary_version.clone(), + registry_checksum: codebook.registry_checksum.clone(), + static_base_checksum: codebook.static_base_checksum.clone(), + codebook_checksum: codebook.codebook_checksum.clone(), actions: records - .into_iter() + .iter() + .cloned() .map(|record| (record.action_id.clone(), record)) .collect(), }; - - let pretty = serde_json::to_string_pretty(&index)?; - fs::write(&index_path, format!("{pretty}\n")) - .with_context(|| format!("writing {}", index_path.display()))?; + write_pretty_json(&index_path, &index)?; + write_pretty_json(&codebook_path, &codebook)?; + write_model_examples_jsonl(&examples_path, &codebook)?; Ok(DictionarySummary { action_count: index.actions.len(), jsonl_path, index_path, + codebook_path, + examples_path, }) } @@ -90,6 +147,236 @@ pub fn load_action_index(dictionary_dir: &Path) -> Result { Ok(index) } +pub fn load_model_codebook(dictionary_dir: &Path) -> Result { + let path = dictionary_dir.join("model.codebook.json"); + let raw = fs::read_to_string(&path).with_context(|| { + format!( + "reading {}; run `cargo run -- dictionary generate` first", + path.display() + ) + })?; + let codebook = serde_json::from_str::(&raw) + .with_context(|| format!("parsing {}", path.display()))?; + Ok(codebook) +} + +pub fn resolve_action_code<'a>(codebook: &'a ModelCodebook, code: &str) -> Option<&'a str> { + codebook.adaptive.actions.get(code).map(String::as_str) +} + +pub fn resolve_field_code<'a>(codebook: &'a ModelCodebook, code: &str) -> Option<&'a str> { + codebook.adaptive.fields.get(code).map(String::as_str) +} + +fn load_static_base_dictionary(dictionary_dir: &Path) -> Result { + let path = dictionary_dir.join("static-base.json"); + let raw = fs::read_to_string(&path).with_context(|| { + format!( + "reading tracked static base dictionary {}; create this file before generating dictionaries", + path.display() + ) + })?; + serde_json::from_str::(&raw) + .with_context(|| format!("parsing {}", path.display())) +} + +fn validate_static_base_dictionary(static_base: &StaticBaseDictionary) -> Result<()> { + let expected_keys = [("K001", "a"), ("K002", "p"), ("K003", "v"), ("K004", "c")]; + let expected_types = [ + ("T001", "action"), + ("T002", "field"), + ("T003", "error"), + ("T004", "module"), + ]; + let expected_errors = [ + ("E001", "unknown_action"), + ("E002", "invalid_payload"), + ("E003", "checksum_mismatch"), + ]; + + for (code, value) in expected_keys { + if static_base.keys.get(code).map(String::as_str) != Some(value) { + bail!("static base dictionary must reserve {code}={value}"); + } + } + for (code, value) in expected_types { + if static_base.types.get(code).map(String::as_str) != Some(value) { + bail!("static base dictionary must reserve {code}={value}"); + } + } + for (code, value) in expected_errors { + if static_base.errors.get(code).map(String::as_str) != Some(value) { + bail!("static base dictionary must reserve {code}={value}"); + } + } + Ok(()) +} + +fn sorted_records(actions: &[ActionDefinition]) -> Vec { + let mut records = actions + .iter() + .cloned() + .map(ActionDictionaryRecord::from) + .collect::>(); + records.sort_by(|left, right| left.action_id.cmp(&right.action_id)); + records +} + +fn build_adaptive_codebook(records: &[ActionDictionaryRecord]) -> AdaptiveCodebook { + let actions = records + .iter() + .enumerate() + .map(|(index, record)| (format_code("A", index), record.action_id.clone())) + .collect::>(); + + let fields = records + .iter() + .flat_map(payload_fields) + .collect::>() + .into_iter() + .enumerate() + .map(|(index, field)| (format_code("F", index), field)) + .collect::>(); + + let modules = records + .iter() + .map(|record| record.executable.script.clone()) + .collect::>() + .into_iter() + .enumerate() + .map(|(index, module)| (format_code("M", index), module)) + .collect::>(); + + AdaptiveCodebook { + actions, + fields, + modules, + } +} + +fn build_model_codebook( + static_base: StaticBaseDictionary, + adaptive: AdaptiveCodebook, + registry_checksum: String, + static_base_checksum: String, +) -> Result { + let mut codebook = ModelCodebook { + schema_version: DICTIONARY_SCHEMA_VERSION.to_string(), + dictionary_version: static_base.dictionary_version.clone(), + registry_checksum, + static_base_checksum, + codebook_checksum: String::new(), + static_base, + adaptive, + }; + + codebook.codebook_checksum = checksum_codebook_without_self(&codebook)?; + Ok(codebook) +} + +fn checksum_codebook_without_self(codebook: &ModelCodebook) -> Result { + checksum_json(&ModelCodebookForChecksum { + schema_version: &codebook.schema_version, + dictionary_version: &codebook.dictionary_version, + registry_checksum: &codebook.registry_checksum, + static_base_checksum: &codebook.static_base_checksum, + static_base: &codebook.static_base, + adaptive: &codebook.adaptive, + }) +} + +fn checksum_json(value: &T) -> Result { + let encoded = serde_json::to_vec(value)?; + let digest = Sha256::digest(encoded); + Ok(format!("{digest:x}")) +} + +fn payload_fields(record: &ActionDictionaryRecord) -> Vec { + record + .payload_schema + .get("properties") + .and_then(serde_json::Value::as_object) + .map(|properties| properties.keys().cloned().collect()) + .unwrap_or_default() +} + +fn format_code(prefix: &str, zero_based_index: usize) -> String { + format!("{prefix}{:03}", zero_based_index + 1) +} + +fn write_actions_jsonl(path: &Path, records: &[ActionDictionaryRecord]) -> Result<()> { + let mut jsonl = File::create(path).with_context(|| format!("creating {}", path.display()))?; + for record in records { + serde_json::to_writer(&mut jsonl, record)?; + jsonl.write_all(b"\n")?; + } + Ok(()) +} + +fn write_model_examples_jsonl(path: &Path, codebook: &ModelCodebook) -> Result<()> { + let mut jsonl = File::create(path).with_context(|| format!("creating {}", path.display()))?; + let examples = [ + serde_json::json!({ + "kind": "verbose_example", + "input": "log hello from codex", + "target": { + "action_id": "logger", + "payload": { + "message": "hello from codex" + } + } + }), + serde_json::json!({ + "kind": "compact_example", + "input": "log hello from codex", + "target": { + "v": codebook.dictionary_version, + "c": codebook.codebook_checksum, + "a": "A001", + "p": { + "F001": "hello from codex" + } + } + }), + serde_json::json!({ + "kind": "invalid_checksum_example", + "input": "log hello with a stale dictionary", + "target": { + "v": codebook.dictionary_version, + "c": "0000000000000000000000000000000000000000000000000000000000000000", + "a": "A001", + "p": { + "F001": "hello" + } + }, + "expected_runtime_result": "reject_checksum_mismatch" + }), + serde_json::json!({ + "kind": "synthetic_template", + "input": "{{natural_language_request}}", + "target": { + "v": codebook.dictionary_version, + "c": codebook.codebook_checksum, + "a": "{{action_code}}", + "p": { + "{{field_code}}": "{{field_value}}" + } + } + }), + ]; + + for example in examples { + serde_json::to_writer(&mut jsonl, &example)?; + jsonl.write_all(b"\n")?; + } + Ok(()) +} + +fn write_pretty_json(path: &Path, value: &T) -> Result<()> { + let pretty = serde_json::to_string_pretty(value)?; + fs::write(path, format!("{pretty}\n")).with_context(|| format!("writing {}", path.display())) +} + #[cfg(test)] fn read_jsonl_records(path: &Path) -> Result> { let file = File::open(path).with_context(|| format!("opening {}", path.display()))?; @@ -126,13 +413,57 @@ mod tests { use crate::registry::registered_actions; #[test] - fn generates_jsonl_and_index() { - let temp = tempfile::tempdir().expect("temp dir"); + fn static_base_dictionary_reserves_expected_codes() { + let temp = temp_dictionary_dir(); + let static_base = load_static_base_dictionary(temp.path()).unwrap(); + validate_static_base_dictionary(&static_base).unwrap(); + assert_eq!(static_base.keys["K001"], "a"); + assert_eq!(static_base.types["T002"], "field"); + assert_eq!(static_base.errors["E003"], "checksum_mismatch"); + } + + #[test] + fn adaptive_layer_maps_logger_and_message_deterministically() { + let records = sorted_records(®istered_actions()); + let adaptive = build_adaptive_codebook(&records); + assert_eq!(adaptive.actions["A001"], "logger"); + assert_eq!(adaptive.fields["F001"], "message"); + assert_eq!(adaptive.modules["M001"], "logger"); + } + + #[test] + fn checksums_are_stable_for_unchanged_inputs() { + let temp = temp_dictionary_dir(); + let first = generate_dictionary_files(®istered_actions(), temp.path()).unwrap(); + let first_codebook = load_model_codebook(temp.path()).unwrap(); + let second = generate_dictionary_files(®istered_actions(), temp.path()).unwrap(); + let second_codebook = load_model_codebook(temp.path()).unwrap(); + + assert_eq!(first.action_count, second.action_count); + assert_eq!( + first_codebook.registry_checksum, + second_codebook.registry_checksum + ); + assert_eq!( + first_codebook.static_base_checksum, + second_codebook.static_base_checksum + ); + assert_eq!( + first_codebook.codebook_checksum, + second_codebook.codebook_checksum + ); + } + + #[test] + fn generates_all_dictionary_outputs() { + let temp = temp_dictionary_dir(); let summary = generate_dictionary_files(®istered_actions(), temp.path()).unwrap(); assert_eq!(summary.action_count, 1); assert!(summary.jsonl_path.exists()); assert!(summary.index_path.exists()); + assert!(summary.codebook_path.exists()); + assert!(summary.examples_path.exists()); let records = read_jsonl_records(&summary.jsonl_path).unwrap(); assert_eq!(records.len(), 1); @@ -141,5 +472,20 @@ mod tests { let index = load_action_index(temp.path()).unwrap(); assert!(index.actions.contains_key("logger")); assert!(!index.actions.contains_key("log")); + + let codebook = load_model_codebook(temp.path()).unwrap(); + assert_eq!(codebook.adaptive.actions["A001"], "logger"); + assert_eq!(codebook.adaptive.fields["F001"], "message"); + assert!(!codebook.codebook_checksum.is_empty()); + } + + fn temp_dictionary_dir() -> tempfile::TempDir { + let temp = tempfile::tempdir().expect("temp dir"); + fs::write( + temp.path().join("static-base.json"), + include_str!("../dictionary/static-base.json"), + ) + .unwrap(); + temp } } diff --git a/src/dispatcher.rs b/src/dispatcher.rs index bd0f717..acfa98c 100644 --- a/src/dispatcher.rs +++ b/src/dispatcher.rs @@ -76,6 +76,10 @@ mod tests { ActionIndex { schema_version: "1.0".to_string(), + dictionary_version: "1.0.0".to_string(), + registry_checksum: "test-registry-checksum".to_string(), + static_base_checksum: "test-static-base-checksum".to_string(), + codebook_checksum: "test-codebook-checksum".to_string(), actions, } } diff --git a/src/llm.rs b/src/llm.rs index c494917..5d8f413 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -2,10 +2,11 @@ use anyhow::{Context, Result, anyhow, bail}; use reqwest::blocking::Client; use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::collections::BTreeMap; use std::time::Duration; use crate::config::ControllerConfig; -use crate::dictionary::ActionIndex; +use crate::dictionary::{ActionIndex, ModelCodebook, resolve_action_code, resolve_field_code}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ActionRequest { @@ -14,6 +15,15 @@ pub struct ActionRequest { pub payload: Value, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct CompactActionRequest { + pub v: String, + pub c: String, + pub a: String, + #[serde(default)] + pub p: BTreeMap, +} + #[derive(Debug, Serialize)] struct ChatRequest { model: String, @@ -43,10 +53,16 @@ struct ChatChoiceMessage { content: String, } -pub fn query_llama(config: &ControllerConfig, index: &ActionIndex, text: &str) -> Result { +pub fn query_llama( + config: &ControllerConfig, + index: &ActionIndex, + codebook: &ModelCodebook, + text: &str, +) -> Result { + let codebook_json = serde_json::to_string(codebook)?; let actions = serde_json::to_string(&index.actions)?; let system = format!( - "You map user requests to exact local action ids. Return only compact JSON with shape {{\"action_id\":\"...\",\"payload\":{{...}}}}. Valid actions index: {actions}" + "You map user requests to local actions. Prefer compact JSON using the codebook with shape {{\"v\":\"...\",\"c\":\"...\",\"a\":\"A001\",\"p\":{{\"F001\":\"...\"}}}}. Return only JSON. Compact codebook: {codebook_json}. Verbose fallback is allowed with shape {{\"action_id\":\"...\",\"payload\":{{...}}}}. Valid verbose actions index: {actions}" ); let request = ChatRequest { model: "local".to_string(), @@ -87,7 +103,7 @@ pub fn query_llama(config: &ControllerConfig, index: &ActionIndex, text: &str) - .ok_or_else(|| anyhow!("llama.cpp response had no choices")) } -pub fn parse_action_response(raw: &str) -> Result { +pub fn parse_action_response(raw: &str, codebook: &ModelCodebook) -> Result { let trimmed = raw.trim(); let json_text = if trimmed.starts_with('{') { trimmed @@ -96,9 +112,18 @@ pub fn parse_action_response(raw: &str) -> Result { .ok_or_else(|| anyhow!("LLM response did not contain a JSON object"))? }; - let request = serde_json::from_str::(json_text) + let value = serde_json::from_str::(json_text) .with_context(|| format!("parsing LLM action JSON: {json_text}"))?; + let request = if value.get("action_id").is_some() { + serde_json::from_value::(value) + .with_context(|| format!("parsing verbose LLM action JSON: {json_text}"))? + } else if value.get("a").is_some() { + expand_compact_action_response(value, codebook)? + } else { + bail!("LLM action JSON had neither action_id nor compact action code a"); + }; + if request.action_id.trim().is_empty() { bail!("LLM action JSON had an empty action_id"); } @@ -106,6 +131,39 @@ pub fn parse_action_response(raw: &str) -> Result { Ok(request) } +fn expand_compact_action_response(value: Value, codebook: &ModelCodebook) -> Result { + let compact = serde_json::from_value::(value) + .context("parsing compact LLM action JSON")?; + + if compact.v != codebook.dictionary_version { + bail!( + "compact LLM action version mismatch: got {}, expected {}", + compact.v, + codebook.dictionary_version + ); + } + + if compact.c != codebook.codebook_checksum { + bail!("compact LLM action checksum mismatch"); + } + + let action_id = resolve_action_code(codebook, &compact.a) + .ok_or_else(|| anyhow!("unknown compact action code `{}`", compact.a))? + .to_string(); + + let mut payload = serde_json::Map::new(); + for (field_code, field_value) in compact.p { + let field_name = resolve_field_code(codebook, &field_code) + .ok_or_else(|| anyhow!("unknown compact field code `{field_code}`"))?; + payload.insert(field_name.to_string(), field_value); + } + + Ok(ActionRequest { + action_id, + payload: Value::Object(payload), + }) +} + fn extract_first_json_object(text: &str) -> Option<&str> { let start = text.find('{')?; let mut depth = 0_i32; @@ -139,29 +197,78 @@ fn extract_first_json_object(text: &str) -> Option<&str> { #[cfg(test)] mod tests { use super::*; + use crate::dictionary::{generate_dictionary_files, load_model_codebook}; + use crate::registry::registered_actions; + use std::fs; #[test] fn parses_valid_action_response() { - let parsed = - parse_action_response(r#"{"action_id":"logger","payload":{"message":"hello"}}"#) - .unwrap(); + let codebook = test_codebook(); + let parsed = parse_action_response( + r#"{"action_id":"logger","payload":{"message":"hello"}}"#, + &codebook, + ) + .unwrap(); assert_eq!(parsed.action_id, "logger"); assert_eq!(parsed.payload["message"], "hello"); } #[test] fn parses_embedded_json_response() { + let codebook = test_codebook(); let parsed = parse_action_response( r#"Here is the action: {"action_id":"logger","payload":{"message":"hello"}}"#, + &codebook, ) .unwrap(); assert_eq!(parsed.action_id, "logger"); } + #[test] + fn expands_valid_compact_action_response() { + let codebook = test_codebook(); + let raw = format!( + r#"{{"v":"{}","c":"{}","a":"A001","p":{{"F001":"hello"}}}}"#, + codebook.dictionary_version, codebook.codebook_checksum + ); + let parsed = parse_action_response(&raw, &codebook).unwrap(); + assert_eq!(parsed.action_id, "logger"); + assert_eq!(parsed.payload["message"], "hello"); + } + + #[test] + fn rejects_compact_checksum_or_version_mismatch() { + let codebook = test_codebook(); + let bad_checksum = format!( + r#"{{"v":"{}","c":"0000000000000000000000000000000000000000000000000000000000000000","a":"A001","p":{{"F001":"hello"}}}}"#, + codebook.dictionary_version + ); + let bad_version = format!( + r#"{{"v":"0.0.0","c":"{}","a":"A001","p":{{"F001":"hello"}}}}"#, + codebook.codebook_checksum + ); + assert!(parse_action_response(&bad_checksum, &codebook).is_err()); + assert!(parse_action_response(&bad_version, &codebook).is_err()); + } + #[test] fn rejects_malformed_response() { - assert!(parse_action_response("run logger please").is_err()); - assert!(parse_action_response(r#"{"payload":{"message":"missing id"}}"#).is_err()); - assert!(parse_action_response(r#"{"action_id":"","payload":{}}"#).is_err()); + let codebook = test_codebook(); + assert!(parse_action_response("run logger please", &codebook).is_err()); + assert!( + parse_action_response(r#"{"payload":{"message":"missing id"}}"#, &codebook).is_err() + ); + assert!(parse_action_response(r#"{"action_id":"","payload":{}}"#, &codebook).is_err()); + } + + fn test_codebook() -> ModelCodebook { + let temp = tempfile::tempdir().expect("temp dir"); + fs::write( + temp.path().join("static-base.json"), + include_str!("../dictionary/static-base.json"), + ) + .unwrap(); + generate_dictionary_files(®istered_actions(), temp.path()).unwrap(); + load_model_codebook(temp.path()).unwrap() } } diff --git a/src/main.rs b/src/main.rs index 1c60795..098b23f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,9 @@ use serde_json::Value; use std::path::PathBuf; use crate::config::ControllerConfig; -use crate::dictionary::{default_dictionary_dir, generate_dictionary_files, load_action_index}; +use crate::dictionary::{ + default_dictionary_dir, generate_dictionary_files, load_action_index, load_model_codebook, +}; use crate::dispatcher::{dispatch_action, run_internal_script}; use crate::llm::{parse_action_response, query_llama}; use crate::registry::registered_actions; @@ -83,18 +85,22 @@ fn main() -> Result<()> { let out_dir = out_dir.unwrap_or_else(default_dictionary_dir); let summary = generate_dictionary_files(®istered_actions(), &out_dir)?; println!( - "generated {} actions: {} and {}", + "generated {} actions: {}, {}, {}, and {}", summary.action_count, summary.jsonl_path.display(), - summary.index_path.display() + summary.index_path.display(), + summary.codebook_path.display(), + summary.examples_path.display() ); Ok(()) } Command::Query { text } => { let config = ControllerConfig::from_env(); - let index = load_action_index(&default_dictionary_dir())?; - let response = query_llama(&config, &index, &text)?; - let request = parse_action_response(&response)?; + let dictionary_dir = default_dictionary_dir(); + let index = load_action_index(&dictionary_dir)?; + let codebook = load_model_codebook(&dictionary_dir)?; + let response = query_llama(&config, &index, &codebook, &text)?; + let request = parse_action_response(&response, &codebook)?; dispatch_action(&index, &request.action_id, request.payload) } Command::Run { action_id, payload } => {