{ "schema": "nsrl.github_pages_results.v1", "generatedAt": "2026-08-30T08:57:50.126Z", "commit": "c63e3ca96e4192110880e735b9b7ed0721212800", "run": { "id": "33302836210", "number": "33", "url": "https://github.com/atimics/nsrl/actions/runs/33302836210" }, "headline": { "id": "nsrl-mme-v0", "label": "NSRL-MME v0", "schema": "nsrl.multimodal_llm_eval.v0", "targetScorePerMille": 700, "minimumRowsPerFamily": 72, "status": "failed", "scorePerMille": 371, "scoreLabel": "37.1%", "targetLabel": "70.0%", "hero": "NSRL-MME v0 Is Measured And Failing", "summary": "The current headline floor is 37.1% against the 70.0% target; the gates below show what must be fixed next.", "policy": "Sampler, replay, browser-probe, and memory-assisted sample metrics are diagnostics only; they do not define the headline score.", "evidence": { "artifact": "data/processed/nsrl-mme-v0.json", "qualityReport": "data/processed/nsrl-mme-v0-local-smoke-20260709T0752Z/quality-report-clean-72-eval500-v2.json", "objectiveCoverage": "data/processed/objective-coverage.json", "qualityReportOk": false, "confidenceLabel": "incomplete" }, "components": [ { "id": "text_prompt_to_image_plan", "label": "text prompt -> symbolic image plan", "kind": "model_native_directional_task", "rows": 432, "scorePerMille": 439, "source": "confidence_trace.directional_native_eval.groups.text_prompt_to_image_plan.stats" }, { "id": "seal_image_to_text", "label": "seal image -> identity / attributes / source text", "kind": "model_native_directional_task", "rows": 720, "scorePerMille": 434, "source": "confidence_trace.directional_native_eval.groups.seal_image_to_text.stats" }, { "id": "text_and_seal_to_explanation", "label": "text + seal -> grounded explanation / match", "kind": "model_native_directional_task", "rows": 576, "scorePerMille": 371, "source": "confidence_trace.directional_native_eval.groups.text_and_seal_to_explanation.stats" }, { "id": "identity_source_binding", "label": "prompt/name -> identity / source binding", "kind": "model_native_directional_task", "rows": 720, "scorePerMille": 375, "source": "confidence_trace.directional_native_eval.groups.identity_source_binding.stats" }, { "id": "hard_negative_match", "label": "match / no-match hard-negative agreement", "kind": "model_native_cross_modal_agreement", "rows": 313, "scorePerMille": 1000, "source": "confidence_trace.cross_modal_agreement" } ], "gates": [ { "id": "quality_report", "label": "green quality report with confidence trace", "ok": false }, { "id": "objective_coverage", "label": "local objective coverage proof", "ok": true }, { "id": "source_grounding", "label": "source-grounded text/image evidence", "ok": true }, { "id": "generated_output_integrity", "label": "held-out generated output integrity", "ok": false } ] }, "assets": [ { "id": "attention", "label": "NSRLLMM1 attention artifact", "path": "web/assets/solomon-attention.nsrllmm", "status": "present", "bytes": 63576, "sha256": "60826f8b823f3bc6e0cb45a769ab2797eaae2a3581deac295e6937dc91f7e475" }, { "id": "multimodal", "label": "NSRLMOD1 multimodal artifact", "path": "web/assets/solomon-multimodal.nsrlmod", "status": "present", "bytes": 17948048, "sha256": "ef57fc5ad94cda625dae307049be80078d4016521a0815596cff492e32605bd4" }, { "id": "denoiser", "label": "NSRLTCH denoiser artifact", "path": "web/assets/solomon-model.nsrltch", "status": "present", "bytes": 10596, "sha256": "a60497cd37201ede88a95be89bbfdc42eb7945c5be48c8f24bcab035cb573ebd" }, { "id": "text-index", "label": "Solomon text signature index", "path": "web/assets/solomon-spirit-text-signatures.tsv", "status": "present", "bytes": 107359, "sha256": "34c0227d0f54ab5c2237fd243a6e41c93a2bea1ce353c7a2cbe5013f33c58b29" } ], "sampleAssets": [ { "id": "text-conditioned-seals", "label": "NSRLTCH text-conditioned seal panel", "source": "docs/assets/solomon-text-conditioned-seals.png", "path": "assets/solomon-text-conditioned-seals.png", "status": "present", "bytes": 651666, "sha256": "3d5e4e2be6ac7a91b453d8e64ad9fce12ec15f7392804f0d63516ae2f02868ee" }, { "id": "sample-gallery-trace", "label": "Fixed prompt gallery trace", "source": "docs/solomon-sample-gallery.tsv", "path": "assets/solomon-sample-gallery.tsv", "status": "present", "bytes": 2532, "sha256": "2aead8cf6a9b0c6bcb5351bae7a0275921a2446c413d4747a4933c261fbc4697" }, { "id": "prompt-sample-bael", "label": "Prompt sample: seal of Bael", "source": "docs/assets/results-samples/bael.png", "path": "assets/results-samples/bael.png", "status": "present", "bytes": 1158, "sha256": "b7bad1609716d125e41e0ec8c70017777b8f6e14f3bea324d77bf70511b206cb" }, { "id": "prompt-sample-stolas", "label": "Prompt sample: seal of Stolas", "source": "docs/assets/results-samples/stolas.png", "path": "assets/results-samples/stolas.png", "status": "present", "bytes": 1139, "sha256": "1b026f0894c0070069df65e8b0bda5b1ca7bedd29727649b0e6895aeb1c3dd25" }, { "id": "prompt-sample-marbas", "label": "Prompt sample: seal of Marbas", "source": "docs/assets/results-samples/marbas.png", "path": "assets/results-samples/marbas.png", "status": "present", "bytes": 1152, "sha256": "828c80b1a1a0f1930a7a106664e7571547e0c03106d36c17c96bb28ad844c96d" }, { "id": "prompt-sample-generic", "label": "Prompt sample: king solomon seal", "source": "docs/assets/results-samples/generic.png", "path": "assets/results-samples/generic.png", "status": "present", "bytes": 1124, "sha256": "7f77674d9b456475be3a5fadfed0c85f7621bbf80df4dcf963eaef6b8598b640" }, { "id": "prompt-sample-eastern-king", "label": "Prompt sample: eastern king invisible spirit", "source": "docs/assets/results-samples/eastern-king.png", "path": "assets/results-samples/eastern-king.png", "status": "present", "bytes": 1152, "sha256": "828c80b1a1a0f1930a7a106664e7571547e0c03106d36c17c96bb28ad844c96d" }, { "id": "prompt-sample-owl-astronomy", "label": "Prompt sample: owl astronomy seal", "source": "docs/assets/results-samples/owl-astronomy.png", "path": "assets/results-samples/owl-astronomy.png", "status": "present", "bytes": 1190, "sha256": "da0d171cae35795c3cedf0a99711501d167a157b6717e556013c1de086b3ffab" } ], "probes": [ { "id": "web-quality", "label": "Browser artifact quality", "status": "passed", "command": "node scripts/check-solomon-attention-web-quality.mjs --all-names --summary", "data": { "schema": "nsrl.solomon_attention_web_quality_summary.v1", "model": "web/assets/solomon-attention.nsrllmm", "prompts": 72, "allNames": true } }, { "id": "raw-name-rank", "label": "Raw prompt-name rank", "status": "passed", "command": "node scripts/probe-solomon-attention-raw-rank.mjs --all-names --summary", "data": { "schema": "nsrl.solomon_attention_raw_rank_summary.v1", "model": "web/assets/solomon-attention.nsrllmm", "textPrefix": "Solomon selects ", "prompts": 72, "ranked": 72, "top1": 72, "top5": 72, "top10": 72, "medianRank": 1, "meanRank": 1, "worstRank": 1, "medianMarginQ8": 4064, "worstMarginQ8": 4064, "misses": [] } }, { "id": "body-start-rank", "label": "Body-start rank", "status": "passed", "command": "node scripts/probe-solomon-attention-body-start-rank.mjs --summary", "data": { "schema": "nsrl.solomon_attention_body_start_rank_summary.v1", "model": "web/assets/solomon-attention.nsrllmm", "prompts": 72, "ranked": 72, "top1": 72, "top5": 72, "top10": 72, "medianRank": 1, "meanRank": 1, "worstRank": 1, "medianMarginQ8": 1391, "worstMarginQ8": 1, "top1Misses": [], "misses": [] } } ], "tables": [ { "id": "prior-scaling", "title": "Prior Scaling Curve", "path": "docs/solomon-eval-scaling-curve.tsv", "columns": [ "prompt_rows", "n_train_prompts", "latent_dim", "text_features", "eval_top1_per_mille", "eval_top5_per_mille", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash" ], "highlights": [ "eval_top1_per_mille", "eval_top5_per_mille", "gold_top1_per_mille" ], "status": "present", "header": [ "prompt_rows", "n_train_prompts", "latent_dim", "text_features", "epochs", "eval_count", "eval_top1_per_mille", "eval_top5_per_mille", "novel_count", "novel_top1_per_mille", "cluster_count", "cluster_top1_per_mille", "gold_count", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash", "model_dir" ], "rows": [ { "prompt_rows": "288", "n_train_prompts": "115", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "29", "eval_top1_per_mille": "34", "eval_top5_per_mille": "275", "novel_count": "12", "novel_top1_per_mille": "0", "cluster_count": "17", "cluster_top1_per_mille": "58", "gold_count": "144", "gold_top1_per_mille": "250", "gold_top5_per_mille": "506", "model_hash": "c98fc237", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n288-ld32-tf512-e12" }, { "prompt_rows": "288", "n_train_prompts": "115", "latent_dim": "64", "text_features": "512", "epochs": "12", "eval_count": "29", "eval_top1_per_mille": "0", "eval_top5_per_mille": "241", "novel_count": "12", "novel_top1_per_mille": "0", "cluster_count": "17", "cluster_top1_per_mille": "0", "gold_count": "144", "gold_top1_per_mille": "104", "gold_top5_per_mille": "326", "model_hash": "ca117469", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n288-ld64-tf512-e12" }, { "prompt_rows": "288", "n_train_prompts": "115", "latent_dim": "128", "text_features": "512", "epochs": "12", "eval_count": "29", "eval_top1_per_mille": "34", "eval_top5_per_mille": "241", "novel_count": "12", "novel_top1_per_mille": "0", "cluster_count": "17", "cluster_top1_per_mille": "58", "gold_count": "144", "gold_top1_per_mille": "97", "gold_top5_per_mille": "326", "model_hash": "f67981cf", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n288-ld128-tf512-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "200", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "173", "gold_count": "144", "gold_top1_per_mille": "76", "gold_top5_per_mille": "277", "model_hash": "0ce31085", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n576-ld32-tf512-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "64", "text_features": "512", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "137", "eval_top5_per_mille": "362", "novel_count": "24", "novel_top1_per_mille": "125", "cluster_count": "46", "cluster_top1_per_mille": "130", "gold_count": "144", "gold_top1_per_mille": "97", "gold_top5_per_mille": "229", "model_hash": "be26f0d7", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n576-ld64-tf512-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "128", "text_features": "512", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "187", "eval_top5_per_mille": "362", "novel_count": "24", "novel_top1_per_mille": "166", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "83", "gold_top5_per_mille": "222", "model_hash": "57d13997", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n576-ld128-tf512-e12" }, { "prompt_rows": "1152", "n_train_prompts": "822", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "186", "eval_top1_per_mille": "86", "eval_top5_per_mille": "231", "novel_count": "61", "novel_top1_per_mille": "49", "cluster_count": "100", "cluster_top1_per_mille": "90", "gold_count": "144", "gold_top1_per_mille": "62", "gold_top5_per_mille": "222", "model_hash": "65047fbd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1152-ld32-tf512-e12" }, { "prompt_rows": "1152", "n_train_prompts": "822", "latent_dim": "64", "text_features": "512", "epochs": "12", "eval_count": "186", "eval_top1_per_mille": "64", "eval_top5_per_mille": "215", "novel_count": "61", "novel_top1_per_mille": "49", "cluster_count": "100", "cluster_top1_per_mille": "50", "gold_count": "144", "gold_top1_per_mille": "76", "gold_top5_per_mille": "256", "model_hash": "a2a510ad", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1152-ld64-tf512-e12" }, { "prompt_rows": "1152", "n_train_prompts": "822", "latent_dim": "128", "text_features": "512", "epochs": "12", "eval_count": "186", "eval_top1_per_mille": "75", "eval_top5_per_mille": "231", "novel_count": "61", "novel_top1_per_mille": "114", "cluster_count": "100", "cluster_top1_per_mille": "30", "gold_count": "144", "gold_top1_per_mille": "104", "gold_top5_per_mille": "256", "model_hash": "e8f9bf91", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1152-ld128-tf512-e12" }, { "prompt_rows": "1425", "n_train_prompts": "1040", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "241", "eval_top1_per_mille": "66", "eval_top5_per_mille": "240", "novel_count": "76", "novel_top1_per_mille": "39", "cluster_count": "130", "cluster_top1_per_mille": "76", "gold_count": "144", "gold_top1_per_mille": "41", "gold_top5_per_mille": "222", "model_hash": "d3ea03af", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1425-ld32-tf512-e12" }, { "prompt_rows": "1425", "n_train_prompts": "1040", "latent_dim": "64", "text_features": "512", "epochs": "12", "eval_count": "241", "eval_top1_per_mille": "49", "eval_top5_per_mille": "244", "novel_count": "76", "novel_top1_per_mille": "39", "cluster_count": "130", "cluster_top1_per_mille": "61", "gold_count": "144", "gold_top1_per_mille": "62", "gold_top5_per_mille": "243", "model_hash": "134230e1", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1425-ld64-tf512-e12" }, { "prompt_rows": "1425", "n_train_prompts": "1040", "latent_dim": "128", "text_features": "512", "epochs": "12", "eval_count": "241", "eval_top1_per_mille": "49", "eval_top5_per_mille": "219", "novel_count": "76", "novel_top1_per_mille": "78", "cluster_count": "130", "cluster_top1_per_mille": "46", "gold_count": "144", "gold_top1_per_mille": "97", "gold_top5_per_mille": "284", "model_hash": "4e63b0ad", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n1425-ld128-tf512-e12" } ], "best": [ { "column": "eval_top1_per_mille", "value": 200, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "200", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "173", "gold_count": "144", "gold_top1_per_mille": "76", "gold_top5_per_mille": "277", "model_hash": "0ce31085", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n576-ld32-tf512-e12" } }, { "column": "eval_top5_per_mille", "value": 462, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "200", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "173", "gold_count": "144", "gold_top1_per_mille": "76", "gold_top5_per_mille": "277", "model_hash": "0ce31085", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n576-ld32-tf512-e12" } }, { "column": "gold_top1_per_mille", "value": 250, "row": { "prompt_rows": "288", "n_train_prompts": "115", "latent_dim": "32", "text_features": "512", "epochs": "12", "eval_count": "29", "eval_top1_per_mille": "34", "eval_top5_per_mille": "275", "novel_count": "12", "novel_top1_per_mille": "0", "cluster_count": "17", "cluster_top1_per_mille": "58", "gold_count": "144", "gold_top1_per_mille": "250", "gold_top5_per_mille": "506", "model_hash": "c98fc237", "model_dir": "data/processed/key-solomon-goetia-latent-v1/scaling-curve/n288-ld32-tf512-e12" } } ] }, { "id": "text-feature-shape", "title": "Text Feature Shape Probe", "path": "docs/solomon-text-feature-shape-probe.tsv", "columns": [ "prompt_rows", "latent_dim", "text_features", "eval_top1_per_mille", "eval_top5_per_mille", "novel_top1_per_mille", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash" ], "highlights": [ "eval_top1_per_mille", "novel_top1_per_mille", "gold_top5_per_mille" ], "status": "present", "header": [ "prompt_rows", "n_train_prompts", "latent_dim", "text_features", "epochs", "eval_count", "eval_top1_per_mille", "eval_top5_per_mille", "novel_count", "novel_top1_per_mille", "cluster_count", "cluster_top1_per_mille", "gold_count", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash", "model_dir" ], "rows": [ { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "2048", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "225", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "217", "gold_count": "144", "gold_top1_per_mille": "166", "gold_top5_per_mille": "416", "model_hash": "04993d13", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld32-tf2048-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "212", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "173", "gold_top5_per_mille": "388", "model_hash": "b801e8fd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld32-tf8192-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "64", "text_features": "2048", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "187", "eval_top5_per_mille": "487", "novel_count": "24", "novel_top1_per_mille": "125", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "159", "gold_top5_per_mille": "333", "model_hash": "127e2a41", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld64-tf2048-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "64", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "175", "eval_top5_per_mille": "437", "novel_count": "24", "novel_top1_per_mille": "166", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "152", "gold_top5_per_mille": "319", "model_hash": "5e6dfa75", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld64-tf8192-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "128", "text_features": "2048", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "137", "eval_top5_per_mille": "437", "novel_count": "24", "novel_top1_per_mille": "166", "cluster_count": "46", "cluster_top1_per_mille": "108", "gold_count": "144", "gold_top1_per_mille": "145", "gold_top5_per_mille": "326", "model_hash": "d1b4e7ed", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld128-tf2048-e12" }, { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "128", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "237", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "291", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "138", "gold_top5_per_mille": "277", "model_hash": "4fcc02bd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld128-tf8192-e12" } ], "best": [ { "column": "eval_top1_per_mille", "value": 237, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "128", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "237", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "291", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "138", "gold_top5_per_mille": "277", "model_hash": "4fcc02bd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld128-tf8192-e12" } }, { "column": "novel_top1_per_mille", "value": 291, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "128", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "237", "eval_top5_per_mille": "462", "novel_count": "24", "novel_top1_per_mille": "291", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "138", "gold_top5_per_mille": "277", "model_hash": "4fcc02bd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld128-tf8192-e12" } }, { "column": "gold_top5_per_mille", "value": 416, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "2048", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "225", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "217", "gold_count": "144", "gold_top1_per_mille": "166", "gold_top5_per_mille": "416", "model_hash": "04993d13", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-shape-probe/n576-ld32-tf2048-e12" } } ] }, { "id": "text-feature", "title": "Text Feature Probe", "path": "docs/solomon-text-feature-probe.tsv", "columns": [ "prompt_rows", "latent_dim", "text_features", "eval_top1_per_mille", "eval_top5_per_mille", "novel_top1_per_mille", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash" ], "highlights": [ "eval_top1_per_mille", "novel_top1_per_mille", "gold_top5_per_mille" ], "status": "present", "header": [ "prompt_rows", "n_train_prompts", "latent_dim", "text_features", "epochs", "eval_count", "eval_top1_per_mille", "eval_top5_per_mille", "novel_count", "novel_top1_per_mille", "cluster_count", "cluster_top1_per_mille", "gold_count", "gold_top1_per_mille", "gold_top5_per_mille", "model_hash", "model_dir" ], "rows": [ { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "212", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "173", "gold_top5_per_mille": "388", "model_hash": "b801e8fd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-probe/n576-ld32-tf8192-e12" } ], "best": [ { "column": "eval_top1_per_mille", "value": 212, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "212", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "173", "gold_top5_per_mille": "388", "model_hash": "b801e8fd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-probe/n576-ld32-tf8192-e12" } }, { "column": "novel_top1_per_mille", "value": 208, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "212", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "173", "gold_top5_per_mille": "388", "model_hash": "b801e8fd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-probe/n576-ld32-tf8192-e12" } }, { "column": "gold_top5_per_mille", "value": 388, "row": { "prompt_rows": "576", "n_train_prompts": "352", "latent_dim": "32", "text_features": "8192", "epochs": "12", "eval_count": "80", "eval_top1_per_mille": "212", "eval_top5_per_mille": "537", "novel_count": "24", "novel_top1_per_mille": "208", "cluster_count": "46", "cluster_top1_per_mille": "195", "gold_count": "144", "gold_top1_per_mille": "173", "gold_top5_per_mille": "388", "model_hash": "b801e8fd", "model_dir": "data/processed/key-solomon-goetia-latent-v1/text-feature-probe/n576-ld32-tf8192-e12" } } ] }, { "id": "generative-eval", "title": "Generative Eval", "path": "docs/solomon-generative-eval.tsv", "columns": [ "model", "prompts", "top1_per_mille", "top5_per_mille", "latent_top1_per_mille", "latent_top5_per_mille", "mean_generated_target_distance_q8", "text_weight" ], "highlights": [ "top1_per_mille", "top5_per_mille", "latent_top1_per_mille" ], "status": "present", "header": [ "model", "latent_model", "prompts", "top1", "top5", "top1_per_mille", "top5_per_mille", "top1_16", "top5_16", "top1_16_per_mille", "top5_16_per_mille", "top1_px", "top5_px", "top1_px_per_mille", "top5_px_per_mille", "latent_top1", "latent_top5", "latent_top1_per_mille", "latent_top5_per_mille", "mean_rank_q8", "mean_rank_16_q8", "mean_rank_px_q8", "mean_latent_rank_q8", "mean_generated_ink_q8", "mean_generated_outside_ink_q8", "mean_generated_edge_ink_q8", "mean_generated_target_distance_q8", "mean_generated_target_distance_16_q8", "mean_generated_target_distance_px_q8", "mean_latent_decoded_target_distance_q8", "mean_latent_target_distance_q8", "text_weight", "selected_mean_wash_penalty_q8" ], "rows": [ { "model": "centroid16", "latent_model": "data/local-runs-linux/local-centroid-30ch-smoke/latent/model.nsrllat", "prompts": "24", "top1": "1", "top5": "1", "top1_per_mille": "41", "top5_per_mille": "41", "top1_16": "1", "top5_16": "1", "top1_16_per_mille": "41", "top5_16_per_mille": "41", "top1_px": "1", "top5_px": "2", "top1_px_per_mille": "41", "top5_px_per_mille": "83", "latent_top1": "2", "latent_top5": "3", "latent_top1_per_mille": "83", "latent_top5_per_mille": "125", "mean_rank_q8": "10272", "mean_rank_16_q8": "10272", "mean_rank_px_q8": "9568", "mean_latent_rank_q8": "8234", "mean_generated_ink_q8": "31471", "mean_generated_outside_ink_q8": "20720", "mean_generated_edge_ink_q8": "24292", "mean_generated_target_distance_q8": "6062784", "mean_generated_target_distance_16_q8": "6062784", "mean_generated_target_distance_px_q8": "430279946", "mean_latent_decoded_target_distance_q8": "6256096", "mean_latent_target_distance_q8": "3280341", "text_weight": "96", "selected_mean_wash_penalty_q8": "15813888" }, { "model": "aws-e8", "latent_model": "data/aws-runs/solomon-prior-optfix-e8-div32-cm32-w4-20260623T2128Z/latent/model.nsrllat", "prompts": "24", "top1": "1", "top5": "1", "top1_per_mille": "41", "top5_per_mille": "41", "top1_16": "1", "top5_16": "1", "top1_16_per_mille": "41", "top5_16_per_mille": "41", "top1_px": "1", "top5_px": "1", "top1_px_per_mille": "41", "top5_px_per_mille": "41", "latent_top1": "1", "latent_top5": "1", "latent_top1_per_mille": "41", "latent_top5_per_mille": "41", "mean_rank_q8": "10560", "mean_rank_16_q8": "10560", "mean_rank_px_q8": "9504", "mean_latent_rank_q8": "7616", "mean_generated_ink_q8": "30390", "mean_generated_outside_ink_q8": "19698", "mean_generated_edge_ink_q8": "22927", "mean_generated_target_distance_q8": "5849621", "mean_generated_target_distance_16_q8": "5849621", "mean_generated_target_distance_px_q8": "416689909", "mean_latent_decoded_target_distance_q8": "6447978", "mean_latent_target_distance_q8": "3288640", "text_weight": "96", "selected_mean_wash_penalty_q8": "15779328" } ], "best": [ { "column": "top1_per_mille", "value": 41, "row": { "model": "centroid16", "latent_model": "data/local-runs-linux/local-centroid-30ch-smoke/latent/model.nsrllat", "prompts": "24", "top1": "1", "top5": "1", "top1_per_mille": "41", "top5_per_mille": "41", "top1_16": "1", "top5_16": "1", "top1_16_per_mille": "41", "top5_16_per_mille": "41", "top1_px": "1", "top5_px": "2", "top1_px_per_mille": "41", "top5_px_per_mille": "83", "latent_top1": "2", "latent_top5": "3", "latent_top1_per_mille": "83", "latent_top5_per_mille": "125", "mean_rank_q8": "10272", "mean_rank_16_q8": "10272", "mean_rank_px_q8": "9568", "mean_latent_rank_q8": "8234", "mean_generated_ink_q8": "31471", "mean_generated_outside_ink_q8": "20720", "mean_generated_edge_ink_q8": "24292", "mean_generated_target_distance_q8": "6062784", "mean_generated_target_distance_16_q8": "6062784", "mean_generated_target_distance_px_q8": "430279946", "mean_latent_decoded_target_distance_q8": "6256096", "mean_latent_target_distance_q8": "3280341", "text_weight": "96", "selected_mean_wash_penalty_q8": "15813888" } }, { "column": "top5_per_mille", "value": 41, "row": { "model": "centroid16", "latent_model": "data/local-runs-linux/local-centroid-30ch-smoke/latent/model.nsrllat", "prompts": "24", "top1": "1", "top5": "1", "top1_per_mille": "41", "top5_per_mille": "41", "top1_16": "1", "top5_16": "1", "top1_16_per_mille": "41", "top5_16_per_mille": "41", "top1_px": "1", "top5_px": "2", "top1_px_per_mille": "41", "top5_px_per_mille": "83", "latent_top1": "2", "latent_top5": "3", "latent_top1_per_mille": "83", "latent_top5_per_mille": "125", "mean_rank_q8": "10272", "mean_rank_16_q8": "10272", "mean_rank_px_q8": "9568", "mean_latent_rank_q8": "8234", "mean_generated_ink_q8": "31471", "mean_generated_outside_ink_q8": "20720", "mean_generated_edge_ink_q8": "24292", "mean_generated_target_distance_q8": "6062784", "mean_generated_target_distance_16_q8": "6062784", "mean_generated_target_distance_px_q8": "430279946", "mean_latent_decoded_target_distance_q8": "6256096", "mean_latent_target_distance_q8": "3280341", "text_weight": "96", "selected_mean_wash_penalty_q8": "15813888" } }, { "column": "latent_top1_per_mille", "value": 83, "row": { "model": "centroid16", "latent_model": "data/local-runs-linux/local-centroid-30ch-smoke/latent/model.nsrllat", "prompts": "24", "top1": "1", "top5": "1", "top1_per_mille": "41", "top5_per_mille": "41", "top1_16": "1", "top5_16": "1", "top1_16_per_mille": "41", "top5_16_per_mille": "41", "top1_px": "1", "top5_px": "2", "top1_px_per_mille": "41", "top5_px_per_mille": "83", "latent_top1": "2", "latent_top5": "3", "latent_top1_per_mille": "83", "latent_top5_per_mille": "125", "mean_rank_q8": "10272", "mean_rank_16_q8": "10272", "mean_rank_px_q8": "9568", "mean_latent_rank_q8": "8234", "mean_generated_ink_q8": "31471", "mean_generated_outside_ink_q8": "20720", "mean_generated_edge_ink_q8": "24292", "mean_generated_target_distance_q8": "6062784", "mean_generated_target_distance_16_q8": "6062784", "mean_generated_target_distance_px_q8": "430279946", "mean_latent_decoded_target_distance_q8": "6256096", "mean_latent_target_distance_q8": "3280341", "text_weight": "96", "selected_mean_wash_penalty_q8": "15813888" } } ] }, { "id": "latent-planner-eval", "title": "Latent Planner Eval", "path": "docs/solomon-latent-planner-eval.tsv", "columns": [ "run_id", "model", "prompts", "top1_per_mille", "top5_per_mille", "mean_rank_q8", "mean_target_distance_q8", "latent_model_hash", "conclusion" ], "highlights": [ "top1_per_mille", "top5_per_mille" ], "status": "present", "header": [ "run_id", "model", "latent_model", "latent_model_hash", "source_samples", "prompts", "top1", "top5", "top1_per_mille", "top5_per_mille", "mean_rank_q8", "mean_target_distance_q8", "source_samples_hash", "selected_prompt_hash", "conclusion" ], "rows": [ { "run_id": "clean72-ld128tf1024", "model": "ld128tf1024", "latent_model": "data/processed/key-solomon-goetia-latent-16x16-v2-sharp/n1425-ld128-tf1024-e60/model.nsrllat", "latent_model_hash": "0xa620d349f9f336ad", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "prompts": "72", "top1": "3", "top5": "6", "top1_per_mille": "41", "top5_per_mille": "83", "mean_rank_q8": "9255", "mean_target_distance_q8": "4265852", "source_samples_hash": "0x71724acec159c539", "selected_prompt_hash": "0xdf68fb043e8b00e4", "conclusion": "learned decoded latent plans only identify 3 of 72 held-out prompts at top-1, confirming the prompt-to-signature planner is the first blocker before bitmap rendering" } ], "best": [ { "column": "top1_per_mille", "value": 41, "row": { "run_id": "clean72-ld128tf1024", "model": "ld128tf1024", "latent_model": "data/processed/key-solomon-goetia-latent-16x16-v2-sharp/n1425-ld128-tf1024-e60/model.nsrllat", "latent_model_hash": "0xa620d349f9f336ad", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "prompts": "72", "top1": "3", "top5": "6", "top1_per_mille": "41", "top5_per_mille": "83", "mean_rank_q8": "9255", "mean_target_distance_q8": "4265852", "source_samples_hash": "0x71724acec159c539", "selected_prompt_hash": "0xdf68fb043e8b00e4", "conclusion": "learned decoded latent plans only identify 3 of 72 held-out prompts at top-1, confirming the prompt-to-signature planner is the first blocker before bitmap rendering" } }, { "column": "top5_per_mille", "value": 83, "row": { "run_id": "clean72-ld128tf1024", "model": "ld128tf1024", "latent_model": "data/processed/key-solomon-goetia-latent-16x16-v2-sharp/n1425-ld128-tf1024-e60/model.nsrllat", "latent_model_hash": "0xa620d349f9f336ad", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "prompts": "72", "top1": "3", "top5": "6", "top1_per_mille": "41", "top5_per_mille": "83", "mean_rank_q8": "9255", "mean_target_distance_q8": "4265852", "source_samples_hash": "0x71724acec159c539", "selected_prompt_hash": "0xdf68fb043e8b00e4", "conclusion": "learned decoded latent plans only identify 3 of 72 held-out prompts at top-1, confirming the prompt-to-signature planner is the first blocker before bitmap rendering" } } ] }, { "id": "oracle-condition-diagnostic", "title": "Oracle Conditioning Diagnostic", "path": "docs/solomon-oracle-condition-diagnostic.tsv", "columns": [ "run_id", "diagnostic_only", "prompts", "signature_top1_per_mille", "signature_top5_per_mille", "retrieval_top1_per_mille", "retrieval_top5_per_mille", "mean_signature_rank_q8", "mean_retrieval_rank_q8", "conclusion" ], "highlights": [ "signature_top1_per_mille", "retrieval_top1_per_mille", "retrieval_top5_per_mille" ], "status": "present", "header": [ "run_id", "diagnostic_only", "prompts", "sampler_model", "source_samples", "condition_source", "signature_top1_per_mille", "signature_top5_per_mille", "retrieval_top1_per_mille", "retrieval_top5_per_mille", "mean_signature_rank_q8", "mean_signature_target_distance_q8", "mean_retrieval_rank_q8", "conclusion" ], "rows": [ { "run_id": "v1-oracle-l72-c1-p1", "diagnostic_only": "true", "prompts": "72", "sampler_model": "data/processed/key-solomon-goetia-denoise-v1/text-multichannel-conv/model.nsrltch", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "condition_source": "true_held_out_16x16_signature_attention_plan", "signature_top1_per_mille": "986", "signature_top5_per_mille": "1000", "retrieval_top1_per_mille": "180", "retrieval_top5_per_mille": "527", "mean_signature_rank_q8": "259", "mean_signature_target_distance_q8": "5207292", "mean_retrieval_rank_q8": "2549", "conclusion": "true 16x16 conditioning nearly solves signature identity, so the learned prompt-to-signature planner is the first blocker; rendered retrieval identity remains weak and separate" } ], "best": [ { "column": "signature_top1_per_mille", "value": 986, "row": { "run_id": "v1-oracle-l72-c1-p1", "diagnostic_only": "true", "prompts": "72", "sampler_model": "data/processed/key-solomon-goetia-denoise-v1/text-multichannel-conv/model.nsrltch", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "condition_source": "true_held_out_16x16_signature_attention_plan", "signature_top1_per_mille": "986", "signature_top5_per_mille": "1000", "retrieval_top1_per_mille": "180", "retrieval_top5_per_mille": "527", "mean_signature_rank_q8": "259", "mean_signature_target_distance_q8": "5207292", "mean_retrieval_rank_q8": "2549", "conclusion": "true 16x16 conditioning nearly solves signature identity, so the learned prompt-to-signature planner is the first blocker; rendered retrieval identity remains weak and separate" } }, { "column": "retrieval_top1_per_mille", "value": 180, "row": { "run_id": "v1-oracle-l72-c1-p1", "diagnostic_only": "true", "prompts": "72", "sampler_model": "data/processed/key-solomon-goetia-denoise-v1/text-multichannel-conv/model.nsrltch", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "condition_source": "true_held_out_16x16_signature_attention_plan", "signature_top1_per_mille": "986", "signature_top5_per_mille": "1000", "retrieval_top1_per_mille": "180", "retrieval_top5_per_mille": "527", "mean_signature_rank_q8": "259", "mean_signature_target_distance_q8": "5207292", "mean_retrieval_rank_q8": "2549", "conclusion": "true 16x16 conditioning nearly solves signature identity, so the learned prompt-to-signature planner is the first blocker; rendered retrieval identity remains weak and separate" } }, { "column": "retrieval_top5_per_mille", "value": 527, "row": { "run_id": "v1-oracle-l72-c1-p1", "diagnostic_only": "true", "prompts": "72", "sampler_model": "data/processed/key-solomon-goetia-denoise-v1/text-multichannel-conv/model.nsrltch", "source_samples": "local-clean-72-eval500-v2/samples.tsv", "condition_source": "true_held_out_16x16_signature_attention_plan", "signature_top1_per_mille": "986", "signature_top5_per_mille": "1000", "retrieval_top1_per_mille": "180", "retrieval_top5_per_mille": "527", "mean_signature_rank_q8": "259", "mean_signature_target_distance_q8": "5207292", "mean_retrieval_rank_q8": "2549", "conclusion": "true 16x16 conditioning nearly solves signature identity, so the learned prompt-to-signature planner is the first blocker; rendered retrieval identity remains weak and separate" } } ] }, { "id": "multimodal-eval", "title": "Multimodal Eval", "path": "docs/solomon-multimodal-eval.tsv", "columns": [ "model", "eval_scope", "examples", "overall_top1_per_mille", "text_top1_per_mille", "image_top1_per_mille", "prompt_top1_per_mille", "exact_examples_per_mille", "context_hit_per_mille", "model_hash" ], "highlights": [ "overall_top1_per_mille", "text_top1_per_mille", "image_top1_per_mille" ], "status": "present", "header": [ "model", "model_path", "text_index", "eval_scope", "examples", "token_count", "model_hash", "token_hash", "overall_count", "overall_top1_per_mille", "overall_top5_per_mille", "prompt_count", "prompt_top1_per_mille", "prompt_top5_per_mille", "text_count", "text_top1_per_mille", "text_top5_per_mille", "image_count", "image_top1_per_mille", "image_top5_per_mille", "special_count", "special_top1_per_mille", "special_top5_per_mille", "exact_examples", "exact_examples_per_mille", "mean_rank_q8", "prompt_mean_rank_q8", "text_mean_rank_q8", "image_mean_rank_q8", "context_hit_per_mille" ], "rows": [ { "model": "deployed", "model_path": "web/assets/solomon-multimodal.nsrlmod", "text_index": "web/assets/solomon-spirit-text-signatures.tsv", "eval_scope": "tracked_corpus_replay", "examples": "313", "token_count": "124370", "model_hash": "0x8057e681de909abc", "token_hash": "0x5d778c1acce837c5", "overall_count": "124057", "overall_top1_per_mille": "994", "overall_top5_per_mille": "998", "prompt_count": "4343", "prompt_top1_per_mille": "882", "prompt_top5_per_mille": "968", "text_count": "38334", "text_top1_per_mille": "996", "text_top5_per_mille": "999", "image_count": "80128", "image_top1_per_mille": "1000", "image_top5_per_mille": "1000", "special_count": "1252", "special_top1_per_mille": "1000", "special_top5_per_mille": "1000", "exact_examples": "1", "exact_examples_per_mille": "3", "mean_rank_q8": "261", "prompt_mean_rank_q8": "372", "text_mean_rank_q8": "259", "image_mean_rank_q8": "256", "context_hit_per_mille": "1000" } ], "best": [ { "column": "overall_top1_per_mille", "value": 994, "row": { "model": "deployed", "model_path": "web/assets/solomon-multimodal.nsrlmod", "text_index": "web/assets/solomon-spirit-text-signatures.tsv", "eval_scope": "tracked_corpus_replay", "examples": "313", "token_count": "124370", "model_hash": "0x8057e681de909abc", "token_hash": "0x5d778c1acce837c5", "overall_count": "124057", "overall_top1_per_mille": "994", "overall_top5_per_mille": "998", "prompt_count": "4343", "prompt_top1_per_mille": "882", "prompt_top5_per_mille": "968", "text_count": "38334", "text_top1_per_mille": "996", "text_top5_per_mille": "999", "image_count": "80128", "image_top1_per_mille": "1000", "image_top5_per_mille": "1000", "special_count": "1252", "special_top1_per_mille": "1000", "special_top5_per_mille": "1000", "exact_examples": "1", "exact_examples_per_mille": "3", "mean_rank_q8": "261", "prompt_mean_rank_q8": "372", "text_mean_rank_q8": "259", "image_mean_rank_q8": "256", "context_hit_per_mille": "1000" } }, { "column": "text_top1_per_mille", "value": 996, "row": { "model": "deployed", "model_path": "web/assets/solomon-multimodal.nsrlmod", "text_index": "web/assets/solomon-spirit-text-signatures.tsv", "eval_scope": "tracked_corpus_replay", "examples": "313", "token_count": "124370", "model_hash": "0x8057e681de909abc", "token_hash": "0x5d778c1acce837c5", "overall_count": "124057", "overall_top1_per_mille": "994", "overall_top5_per_mille": "998", "prompt_count": "4343", "prompt_top1_per_mille": "882", "prompt_top5_per_mille": "968", "text_count": "38334", "text_top1_per_mille": "996", "text_top5_per_mille": "999", "image_count": "80128", "image_top1_per_mille": "1000", "image_top5_per_mille": "1000", "special_count": "1252", "special_top1_per_mille": "1000", "special_top5_per_mille": "1000", "exact_examples": "1", "exact_examples_per_mille": "3", "mean_rank_q8": "261", "prompt_mean_rank_q8": "372", "text_mean_rank_q8": "259", "image_mean_rank_q8": "256", "context_hit_per_mille": "1000" } }, { "column": "image_top1_per_mille", "value": 1000, "row": { "model": "deployed", "model_path": "web/assets/solomon-multimodal.nsrlmod", "text_index": "web/assets/solomon-spirit-text-signatures.tsv", "eval_scope": "tracked_corpus_replay", "examples": "313", "token_count": "124370", "model_hash": "0x8057e681de909abc", "token_hash": "0x5d778c1acce837c5", "overall_count": "124057", "overall_top1_per_mille": "994", "overall_top5_per_mille": "998", "prompt_count": "4343", "prompt_top1_per_mille": "882", "prompt_top5_per_mille": "968", "text_count": "38334", "text_top1_per_mille": "996", "text_top5_per_mille": "999", "image_count": "80128", "image_top1_per_mille": "1000", "image_top5_per_mille": "1000", "special_count": "1252", "special_top1_per_mille": "1000", "special_top5_per_mille": "1000", "exact_examples": "1", "exact_examples_per_mille": "3", "mean_rank_q8": "261", "prompt_mean_rank_q8": "372", "text_mean_rank_q8": "259", "image_mean_rank_q8": "256", "context_hit_per_mille": "1000" } } ] }, { "id": "attention-raw-eval", "title": "Attention Raw Eval", "path": "docs/solomon-attention-eval.tsv", "columns": [ "model", "eval_scope", "prompts", "prompt_name_match_per_mille", "mean_raw_quality_score", "min_raw_quality_score", "distinct_texts", "scaffold_output_per_mille", "model_hash" ], "highlights": [ "prompt_name_match_per_mille", "mean_raw_quality_score", "distinct_texts" ], "status": "present", "header": [ "model", "model_path", "eval_scope", "prompts", "generated", "prompt_name_matches", "prompt_name_match_per_mille", "mean_raw_quality_score", "min_raw_quality_score", "distinct_texts", "scaffold_outputs", "scaffold_output_per_mille", "mean_chars", "model_hash", "sample_prompt", "sample_output" ], "rows": [ { "model": "raw-no-memory", "model_path": "web/assets/solomon-attention.nsrllmm", "eval_scope": "native_raw_no_memory_no_conditioning", "prompts": "72", "generated": "72", "prompt_name_matches": "0", "prompt_name_match_per_mille": "0", "mean_raw_quality_score": "27", "min_raw_quality_score": "15", "distinct_texts": "4", "scaffold_outputs": "55", "scaffold_output_per_mille": "763", "mean_chars": "63", "model_hash": "0xff009204cd3d1767", "sample_prompt": "seal of Bael", "sample_output": "---He is of the Goetia and teacheth with his ART in LINE." } ], "best": [ { "column": "prompt_name_match_per_mille", "value": 0, "row": { "model": "raw-no-memory", "model_path": "web/assets/solomon-attention.nsrllmm", "eval_scope": "native_raw_no_memory_no_conditioning", "prompts": "72", "generated": "72", "prompt_name_matches": "0", "prompt_name_match_per_mille": "0", "mean_raw_quality_score": "27", "min_raw_quality_score": "15", "distinct_texts": "4", "scaffold_outputs": "55", "scaffold_output_per_mille": "763", "mean_chars": "63", "model_hash": "0xff009204cd3d1767", "sample_prompt": "seal of Bael", "sample_output": "---He is of the Goetia and teacheth with his ART in LINE." } }, { "column": "mean_raw_quality_score", "value": 27, "row": { "model": "raw-no-memory", "model_path": "web/assets/solomon-attention.nsrllmm", "eval_scope": "native_raw_no_memory_no_conditioning", "prompts": "72", "generated": "72", "prompt_name_matches": "0", "prompt_name_match_per_mille": "0", "mean_raw_quality_score": "27", "min_raw_quality_score": "15", "distinct_texts": "4", "scaffold_outputs": "55", "scaffold_output_per_mille": "763", "mean_chars": "63", "model_hash": "0xff009204cd3d1767", "sample_prompt": "seal of Bael", "sample_output": "---He is of the Goetia and teacheth with his ART in LINE." } }, { "column": "distinct_texts", "value": 4, "row": { "model": "raw-no-memory", "model_path": "web/assets/solomon-attention.nsrllmm", "eval_scope": "native_raw_no_memory_no_conditioning", "prompts": "72", "generated": "72", "prompt_name_matches": "0", "prompt_name_match_per_mille": "0", "mean_raw_quality_score": "27", "min_raw_quality_score": "15", "distinct_texts": "4", "scaffold_outputs": "55", "scaffold_output_per_mille": "763", "mean_chars": "63", "model_hash": "0xff009204cd3d1767", "sample_prompt": "seal of Bael", "sample_output": "---He is of the Goetia and teacheth with his ART in LINE." } } ] }, { "id": "sample-gallery", "title": "Prompt Sample Gallery", "path": "docs/solomon-sample-gallery.tsv", "columns": [ "model", "prompt_id", "prompt_kind", "prompt", "text_source", "image_source", "text_lm_fallback", "text", "image_path", "model_hash" ], "highlights": [], "status": "present", "header": [ "model", "prompt_id", "prompt_kind", "prompt", "seed", "top_k", "width", "height", "text_source", "image_source", "text_lm_fallback", "text", "image_path", "model_hash", "token_hash", "image_sha256" ], "rows": [ { "model": "NSRLLMM1", "prompt_id": "bael", "prompt_kind": "fixed-known", "prompt": "seal of Bael", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_lm_strict", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "false", "text": "Solomon selects Bael: He maketh thee to go Invisible.", "image_path": "docs/assets/results-samples/bael.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "b7bad1609716d125e41e0ec8c70017777b8f6e14f3bea324d77bf70511b206cb" }, { "model": "NSRLLMM1", "prompt_id": "stolas", "prompt_kind": "fixed-known", "prompt": "seal of Stolas", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_lm_strict", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "false", "text": "Solomon selects Stolas: He teacheth the Art of Astronomy, and the Virtues of Herbs and Precious Stones.", "image_path": "docs/assets/results-samples/stolas.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "1b026f0894c0070069df65e8b0bda5b1ca7bedd29727649b0e6895aeb1c3dd25" }, { "model": "NSRLLMM1", "prompt_id": "marbas", "prompt_kind": "fixed-known", "prompt": "seal of Marbas", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_memory_guard", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "true", "text": "Solomon selects Marbas: He is a Great President, and appeareth at first in the form of a Great Lion, but afterwards, at the request of the Master, he putteth on Human Shape.", "image_path": "docs/assets/results-samples/marbas.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "828c80b1a1a0f1930a7a106664e7571547e0c03106d36c17c96bb28ad844c96d" }, { "model": "NSRLLMM1", "prompt_id": "generic", "prompt_kind": "fixed-generic", "prompt": "king solomon seal", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_lm_strict", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "false", "text": "Solomon selects Ronove: He teacheth the Art of Rhetoric very well and giveth Good Servants, Knowledge of Tongues, and Favours with Friends or Foes.", "image_path": "docs/assets/results-samples/generic.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "7f77674d9b456475be3a5fadfed0c85f7621bbf80df4dcf963eaef6b8598b640" }, { "model": "NSRLLMM1", "prompt_id": "eastern-king", "prompt_kind": "fixed-held-out-phrase", "prompt": "eastern king invisible spirit", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_memory_guard", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "true", "text": "Solomon selects Marbas: He is a Great President, and appeareth at first in the form of a Great Lion, but afterwards, at the request of the Master, he putteth on Human Shape.", "image_path": "docs/assets/results-samples/eastern-king.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "828c80b1a1a0f1930a7a106664e7571547e0c03106d36c17c96bb28ad844c96d" }, { "model": "NSRLLMM1", "prompt_id": "owl-astronomy", "prompt_kind": "fixed-held-out-phrase", "prompt": "owl astronomy seal", "seed": "1", "top_k": "1", "width": "192", "height": "192", "text_source": "embedded_text_lm_strict", "image_source": "embedded_image_memory_strict", "text_lm_fallback": "false", "text": "Solomon selects Murmur: and appeareth in the Form of a Warrior riding upon a Gryphon, with a Ducal Crown upon his Head.", "image_path": "docs/assets/results-samples/owl-astronomy.png", "model_hash": "0xff009204cd3d1767", "token_hash": "0x4ddd78e1d39fc115", "image_sha256": "da0d171cae35795c3cedf0a99711501d167a157b6717e556013c1de086b3ffab" } ], "best": [] } ], "models": [ { "id": "nsrllmm1", "name": "NSRLLMM1", "role": "attention joint text/image-token model", "status": "probe-gated + raw control", "summary": "Current browser artifact path. The probe suite exercises prompt-scoped text, embedded image memory, raw name logits, and source-specific body-start logits. The raw no-memory row is a free-running negative control and should not be read as browser-path quality.", "metrics": [ { "label": "Artifact probe suite", "value": "72/72", "detail": "100%. All configured prompt samples passed the browser sampler gate.", "bar": 100 }, { "label": "Constrained name-logit top-1", "value": "72/72", "detail": "100%. Expected spirit name token after `Solomon selects `; median rank 1; worst rank 1.", "bar": 100 }, { "label": "Constrained body-start top-1", "value": "72/72", "detail": "100%. First source prose token after the name opening; median rank 1; worst rank 1.", "bar": 100 }, { "label": "Raw no-memory name match", "value": "0.0%", "detail": "0 per mille from raw-no-memory / 72 prompts / 0xff009204cd3d1767 (prompt_name_match_per_mille).", "bar": 0 }, { "label": "Raw text quality score", "value": "27/100", "detail": "mean_raw_quality_score from raw-no-memory / 72 prompts / 0xff009204cd3d1767.", "bar": 27 }, { "label": "Raw scaffold outputs", "value": "76.3%", "detail": "763 per mille from raw-no-memory / 72 prompts / 0xff009204cd3d1767 (scaffold_output_per_mille).", "bar": 76.3 } ], "evidence": [ "scripts/check-solomon-attention-web-quality.mjs --all-names --summary", "scripts/probe-solomon-attention-raw-rank.mjs --all-names --summary", "scripts/probe-solomon-attention-body-start-rank.mjs --summary", "docs/solomon-attention-eval.tsv" ] }, { "id": "nsrllat1", "name": "NSRLLAT1", "role": "prompt-to-layout latent prior", "status": "published evals", "summary": "Prompt/layout prior measured on checked-in train/eval/gold prompt partitions and the clean 72 held-out generation prompt set. The clean-72 row is the fast planner target to improve before rerunning bitmap generation.", "metrics": [ { "label": "Clean-72 planner top-1", "value": "4.1%", "detail": "41 per mille from ld128tf1024 / 72 prompts (top1_per_mille).", "bar": 4.1 }, { "label": "Clean-72 planner top-5", "value": "8.3%", "detail": "83 per mille from ld128tf1024 / 72 prompts (top5_per_mille).", "bar": 8.3 }, { "label": "Best eval top-1", "value": "20.0%", "detail": "200 per mille from 576 prompt rows / latent 32 / text 512 / 0ce31085 (eval_top1_per_mille).", "bar": 20 }, { "label": "Best eval top-5", "value": "46.2%", "detail": "462 per mille from 576 prompt rows / latent 32 / text 512 / 0ce31085 (eval_top5_per_mille).", "bar": 46.2 }, { "label": "Best gold top-1", "value": "25.0%", "detail": "250 per mille from 288 prompt rows / latent 32 / text 512 / c98fc237 (gold_top1_per_mille).", "bar": 25 }, { "label": "Best shape-probe eval top-1", "value": "23.7%", "detail": "237 per mille from 576 prompt rows / latent 128 / text 8192 / 4fcc02bd (eval_top1_per_mille).", "bar": 23.7 } ], "evidence": [ "docs/solomon-latent-planner-eval.tsv", "docs/solomon-eval-scaling-curve.tsv", "docs/solomon-text-feature-shape-probe.tsv" ] }, { "id": "nsrltch", "name": "NSRLTCH", "role": "text-conditioned bitmap denoiser", "status": "published evals", "summary": "The denoiser artifact is scored on learned latent-conditioned generations. The oracle-conditioning diagnostic is an upper bound only: it gives the denoiser the true 16x16 target plan to separate planner failure from renderer/retrieval failure.", "metrics": [ { "label": "Published artifact", "value": "present", "detail": "10.3 KiB / sha256 a60497cd3720", "bar": 100 }, { "label": "Best generated top-1", "value": "4.1%", "detail": "41 per mille from centroid16 / 24 prompts (top1_per_mille).", "bar": 4.1 }, { "label": "Best generated top-5", "value": "4.1%", "detail": "41 per mille from centroid16 / 24 prompts (top5_per_mille).", "bar": 4.1 }, { "label": "Best latent top-1", "value": "8.3%", "detail": "83 per mille from centroid16 / 24 prompts (latent_top1_per_mille).", "bar": 8.3 }, { "label": "Oracle-plan signature top-1", "value": "98.6%", "detail": "986 per mille from 72 prompts (signature_top1_per_mille).", "bar": 98.6 }, { "label": "Oracle-plan retrieval top-1", "value": "18.0%", "detail": "180 per mille from 72 prompts (retrieval_top1_per_mille).", "bar": 18 }, { "label": "Best target distance", "value": "5,849,621", "detail": "mean_generated_target_distance_q8 from aws-e8 / 24 prompts. Lower is better.", "bar": null } ], "evidence": [ "web/assets/solomon-model.nsrltch", "docs/solomon-generative-eval.tsv", "docs/solomon-oracle-condition-diagnostic.tsv" ] }, { "id": "nsrlmod1", "name": "NSRLMOD1", "role": "coarse joint text/image-token model", "status": "published evals", "summary": "The multimodal fallback artifact is scored on tracked corpus replay: next-token ranks over prompt bytes, generated text bytes, marker tokens, and 16x16 image bins. This is artifact-native replay quality, not broad free-running quality.", "metrics": [ { "label": "Published artifact", "value": "present", "detail": "17.1 MiB / sha256 ef57fc5ad94c", "bar": 100 }, { "label": "Overall replay top-1", "value": "99.4%", "detail": "994 per mille from deployed / 313 examples / 0x8057e681de909abc (overall_top1_per_mille).", "bar": 99.4 }, { "label": "Text replay top-1", "value": "99.6%", "detail": "996 per mille from deployed / 313 examples / 0x8057e681de909abc (text_top1_per_mille).", "bar": 99.6 }, { "label": "Image-token top-1", "value": "100.0%", "detail": "1000 per mille from deployed / 313 examples / 0x8057e681de909abc (image_top1_per_mille).", "bar": 100 }, { "label": "Exact examples", "value": "0.3%", "detail": "3 per mille from deployed / 313 examples / 0x8057e681de909abc (exact_examples_per_mille).", "bar": 0.3 } ], "evidence": [ "web/assets/solomon-multimodal.nsrlmod", "docs/solomon-multimodal-eval.tsv", "scripts/run-solomon-multimodal-eval.mjs" ] } ], "samplePanels": [ { "id": "nsrllat1-routing", "model": "NSRLLAT1", "title": "Latent Prior Routing", "outcome": "Best checked-in routing row", "summary": "Prompt-to-layout routing is scored on held-out Solomon prompt partitions. This panel shows the strongest eval top-1 row now published.", "metrics": [ { "label": "Eval top-1", "value": "20.0%" }, { "label": "Eval top-5", "value": "46.2%" }, { "label": "Gold top-5", "value": "27.7%" } ], "evidence": [ "docs/solomon-eval-scaling-curve.tsv", "576 prompt rows / latent 32 / text 512 / 0ce31085" ] }, { "id": "nsrltch-generative", "model": "NSRLTCH", "title": "Generated Seal Retrieval", "outcome": "Class-head generation eval", "summary": "Generated bitmap seals are checked against target retrieval. The image panel is the tracked visual sample; the numbers come from the generative eval table.", "image": { "src": "assets/solomon-text-conditioned-seals.png", "alt": "NSRLTCH text-conditioned seal panel" }, "metrics": [ { "label": "Generated top-1", "value": "4.1%" }, { "label": "Generated top-5", "value": "4.1%" }, { "label": "Latent top-1", "value": "8.3%" } ], "evidence": [ "docs/solomon-generative-eval.tsv", "docs/assets/solomon-text-conditioned-seals.png" ] }, { "id": "nsrltch-oracle-conditioning", "model": "NSRLTCH", "title": "Oracle Conditioning Split", "outcome": "Diagnostic upper bound; not headline evidence", "summary": "The denoiser receives the true held-out 16x16 target plan here. This isolates the current failure: the learned planner is the first blocker, while rendered-image retrieval remains a separate weak gate.", "metrics": [ { "label": "Signature top-1", "value": "98.6%" }, { "label": "Signature top-5", "value": "100.0%" }, { "label": "Retrieval top-1", "value": "18.0%" }, { "label": "Retrieval top-5", "value": "52.7%" } ], "evidence": [ "docs/solomon-oracle-condition-diagnostic.tsv", "scripts/run-solomon-oracle-condition-diagnostic.mjs" ] }, { "id": "nsrlmod1-replay", "model": "NSRLMOD1", "title": "Text/Image Token Replay", "outcome": "Tracked corpus replay", "summary": "The multimodal artifact is scored on next-token replay over prompt bytes, generated text bytes, marker tokens, and 16x16 image bins.", "metrics": [ { "label": "Overall top-1", "value": "99.4%" }, { "label": "Text top-1", "value": "99.6%" }, { "label": "Image top-1", "value": "100.0%" }, { "label": "Context hit", "value": "100.0%" } ], "evidence": [ "docs/solomon-multimodal-eval.tsv", "deployed / 313 examples / 0x8057e681de909abc" ] }, { "id": "nsrllmm1-control", "model": "NSRLLMM1", "title": "Probe-Gated Artifact, Raw Control", "outcome": "Browser probes pass; raw no-memory control fails name binding", "summary": "The browser artifact path is probe-gated separately from native free-running sampling with memory and conditioning disabled.", "sample": { "label": "seal of Bael", "text": "---He is of the Goetia and teacheth with his ART in LINE." }, "metrics": [ { "label": "Artifact prompts", "value": "72/72" }, { "label": "Name-logit top-1", "value": "72/72" }, { "label": "Body-start top-1", "value": "72/72" }, { "label": "Raw name match", "value": "0.0%" }, { "label": "Raw text score", "value": "27/100" } ], "evidence": [ "docs/solomon-attention-eval.tsv", "scripts/check-solomon-attention-web-quality.mjs --all-names --summary" ] } ], "promptSamples": [ { "id": "bael", "model": "NSRLLMM1", "promptKind": "fixed-known", "prompt": "seal of Bael", "textSource": "embedded_text_lm_strict", "imageSource": "embedded_image_memory_strict", "textFallback": "false", "text": "Solomon selects Bael: He maketh thee to go Invisible.", "image": { "src": "assets/results-samples/bael.png", "alt": "seal of Bael sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" }, { "id": "stolas", "model": "NSRLLMM1", "promptKind": "fixed-known", "prompt": "seal of Stolas", "textSource": "embedded_text_lm_strict", "imageSource": "embedded_image_memory_strict", "textFallback": "false", "text": "Solomon selects Stolas: He teacheth the Art of Astronomy, and the Virtues of Herbs and Precious Stones.", "image": { "src": "assets/results-samples/stolas.png", "alt": "seal of Stolas sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" }, { "id": "marbas", "model": "NSRLLMM1", "promptKind": "fixed-known", "prompt": "seal of Marbas", "textSource": "embedded_text_memory_guard", "imageSource": "embedded_image_memory_strict", "textFallback": "true", "text": "Solomon selects Marbas: He is a Great President, and appeareth at first in the form of a Great Lion, but afterwards, at the request of the Master, he putteth on Human Shape.", "image": { "src": "assets/results-samples/marbas.png", "alt": "seal of Marbas sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" }, { "id": "generic", "model": "NSRLLMM1", "promptKind": "fixed-generic", "prompt": "king solomon seal", "textSource": "embedded_text_lm_strict", "imageSource": "embedded_image_memory_strict", "textFallback": "false", "text": "Solomon selects Ronove: He teacheth the Art of Rhetoric very well and giveth Good Servants, Knowledge of Tongues, and Favours with Friends or Foes.", "image": { "src": "assets/results-samples/generic.png", "alt": "king solomon seal sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" }, { "id": "eastern-king", "model": "NSRLLMM1", "promptKind": "fixed-held-out-phrase", "prompt": "eastern king invisible spirit", "textSource": "embedded_text_memory_guard", "imageSource": "embedded_image_memory_strict", "textFallback": "true", "text": "Solomon selects Marbas: He is a Great President, and appeareth at first in the form of a Great Lion, but afterwards, at the request of the Master, he putteth on Human Shape.", "image": { "src": "assets/results-samples/eastern-king.png", "alt": "eastern king invisible spirit sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" }, { "id": "owl-astronomy", "model": "NSRLLMM1", "promptKind": "fixed-held-out-phrase", "prompt": "owl astronomy seal", "textSource": "embedded_text_lm_strict", "imageSource": "embedded_image_memory_strict", "textFallback": "false", "text": "Solomon selects Murmur: and appeareth in the Form of a Warrior riding upon a Gryphon, with a Ducal Crown upon his Head.", "image": { "src": "assets/results-samples/owl-astronomy.png", "alt": "owl astronomy seal sample" }, "traceHref": "assets/solomon-sample-gallery.tsv", "modelHash": "0xff009204cd3d1767" } ], "honesty": { "status": "passed", "errors": [] } }