{ "format_version": 1, "title": "RenderMasterBot Strata experiment history", "publication_date": "2026-10-12", "source_audit_sha256": "ccb0c0acad74da548cccb9f65b9e1ad60518fca8d76b48110be8af1623894b30", "public_overview": { "title": "Strata offline assistant: the complete experiment history", "period": "October 8–10, 2026 (UTC)", "summary": "A local engineering investigation of Strata with Qwen3.8-Flash-Next, alongside installed Ollama options. It covers deployment, answer-quality failures, quantization upgrades, memory policies, bounded throughput, long reasoning, coding and workflow fixtures, interrupted runs, and narrowly scoped recovery segments.", "hardware_context": "The historical machine used an RTX 5080, Ryzen 9 9950X3D, about 61.66 GiB physical RAM and an NVMe SSD. These are October 8 evidence bindings, not a live hardware check.", "outcome": "The runtime worked, but runtime success never established autonomous Unreal expertise. Several studies completed their requested calls while failing content, schema, resource or lifecycle gates. Human and product acceptance remain pending. Failed and interrupted histories are retained rather than replaced by later successful tails.", "not_a_ranking": "Profiles, prompts, budgets, memory modes, reviewers and run timing changed across studies. Separate studies are not pooled into an all-time model-quality or speed ranking." }, "public_timeline": [ { "id": "H01", "date_utc": "2026-10-08T17:58:01Z", "title": "Initial load and structured-output check", "kind": "deployment check", "status": "completed within narrow scope", "description": "IQ2_XS loaded in 42.25 seconds; a supplied strict-schema probe returned in 2.17 seconds and passed validation. The model had a configured 65,536-token context, but full-context stress was not run.", "limits": "Structured output was prompt-and-validate rather than constrained decoding. This tested the runtime and one contract, not live Unreal work or answer expertise." }, { "id": "H02", "date_utc": "2026-10-08T17:58:54Z", "title": "First four-case Unreal-assistance evaluation", "kind": "quality", "status": "requests completed; preliminary quality failed", "description": "Four complete answers and confirmed unload. Preliminary review scored 27/40 against a frozen 34/40 threshold. Scope and structured-plan cases were stronger; profiler interpretation and experimental design contained substantive errors.", "limits": "Four development fixtures, no live engine access. Review was agent preliminary judgment; human acceptance remains pending." }, { "id": "H03", "date_utc": "2026-10-08T18:00:01Z", "title": "Historical-profiler demonstration task", "kind": "demonstration", "status": "one response completed; acceptance pending", "description": "One supplied-evidence task returned a complete answer in 98.61 seconds. The input profiler record was a historical September 17 capture, not a new measurement.", "limits": "No diagnosis was verified through new profiling or an executed optimization. This is a demonstration separate from the four-case suite." }, { "id": "H04", "date_utc": "2026-10-08T18:17:17Z", "title": "First configured-system comparison attempt", "kind": "comparison attempt", "status": "aborted before Strata phase; cleanup gate failed", "description": "The planned 16-record comparison saved eight Ollama answers, then stopped because model release was not confirmed. No Strata response belongs to this attempt.", "limits": "Preserved diagnostic attempt. It is not a completed two-backend comparison and must not be merged with the later replacement protocol." }, { "id": "H05", "date_utc": "2026-10-08T18:23:22Z", "title": "Revised Qwen3.5 versus Strata comparison", "kind": "configured-system comparison", "status": "16 main responses completed; full content acceptance failed", "description": "Four tasks, two rounds per backend, plus a separate strict-schema request per backend. Preliminary totals were Ollama 23.5/40 in both rounds and Strata 29/40 then 32/40. All main rounds failed full acceptance. Median preloaded nonstreaming request times were 4.23 seconds and 7.54 seconds respectively.", "limits": "The directory calls the fixture simulated, but these were local model requests over simulated/supplied tasks, not mocked inference. No live Unreal operations. Differences cannot be assigned solely to runtime, parameter count, quantization or reasoning." }, { "id": "H06", "date_utc": "2026-10-08T18:48:10Z", "title": "GPT-OSS low-effort follow-up", "kind": "historical comparison follow-up", "status": "eight main responses completed; preliminary quality failed", "description": "GPT-OSS 20B completed eight low-effort requests. Preliminary totals were 24/40 and 25/40; neither round passed acceptance. The run compared against earlier Qwen/Strata observations rather than rerunning a randomized three-way benchmark.", "limits": "Reviewer variation was not calibrated. The first question included extra initialization, and the full timing aggregate retained it. A separate schema request is not folded into the main quality/timing total." }, { "id": "H07", "date_utc": "2026-10-08T18:51:30Z", "title": "GPT-OSS high-effort diagnostic", "kind": "reasoning diagnostic", "status": "four calls recorded; all final answers truncated", "description": "One four-case round used high reasoning and doubled output caps. All four responses ended at the length limit; zero complete final answers were delivered.", "limits": "Reasoning effort and cap changed together. No full high-effort quality score was assigned, and this track cannot replace the low-effort main track." }, { "id": "F01", "date_utc": "2026-10-08", "title": "Higher-quantization feasibility assessment", "kind": "read-only planning", "status": "assessment completed; predictions not measurements", "description": "Reviewed IQ3_XXS, IQ3_S, UD-IQ4_XS, UD-Q4_K_XL and UD-Q6_K_XL against historical RAM, commit, disk and documented runtime requirements. IQ3_XXS was the first bounded trial; higher formats were initially untested and Q6 required separate compatibility work.", "limits": "Download sizes and rough RAM estimates were planning figures, not measured candidate peaks. IQ4 and Q6 have no completed local evaluation in this evidence set. Later IQ3_S/Q4 work does not retroactively change this assessment." }, { "id": "H08", "date_utc": "2026-10-08T19:06:35Z", "title": "IQ3_XXS installation", "kind": "installation", "status": "installed and verified; not automatically activated", "description": "The higher-weight option was installed beside IQ2_XS, with recorded file verification and original weights preserved.", "limits": "Installation is not an answer-quality improvement, memory qualification or default-provider replacement." }, { "id": "H09", "date_utc": "2026-10-08T19:30:00Z", "title": "Twenty-call quantization, budget and grounding study", "kind": "controlled quality study", "status": "20 calls completed; proven quality upgrade not established", "description": "Separate tracks compared matched 32K IQ2/IQ3_XXS profiles, wider reasoning budgets, an evidence policy with and without supplied grounding, and two independent holdout cases. Matched original cases scored IQ2 28.5/40 versus IQ3 26/40; wide hard cases 12/20 versus 11/20; holdouts 19.5/20 versus 18/20. Only 5/20 answers passed the full preliminary content contract; all six structured calls passed schema validation.", "limits": "The new 32K resident IQ2 control was not the original 64K launcher. No rescore or winner-selected rerun. Unguarded cold loads sampled physical headroom as low as 0.02 GiB for IQ2 and 1.24 GiB for IQ3, so these comparison profiles were not daily-use recommendations." }, { "id": "H10", "date_utc": "2026-10-08T19:50:01Z", "title": "Guarded IQ3_XXS memory qualification", "kind": "resource-only qualification", "status": "qualified for two tested workloads; quality not scored", "description": "Two separate supplied synthetic holdout calls completed in 84.16 and 63.88 seconds. Sampled whole-system RAM peaked at approximately 33.10 and 33.14 GiB. A guarded optional profile reduced observed memory demand at a throughput cost; the existing default stayed unchanged.", "limits": "Two workloads do not establish broad reliability. Allocation headroom, expert budgets and file-release hints are not a production RAM cap; one-second sampling can miss transient peaks." }, { "id": "H11", "date_utc": "2026-10-08T22:21:18Z", "title": "First IQ3_S installation attempt", "kind": "installation attempt", "status": "partial/unsealed; superseded by later verification", "description": "The saved progress record showed approximately 46.0 GB of present final-or-partial data against an 83.6 GB weight total. No installation summary or verification terminal is present for this directory.", "limits": "Present/downloaded bytes are not a completed publisher-hash check. Saved artifacts do not establish the exact reason this attempt ended." }, { "id": "H12", "date_utc": "2026-10-08T22:44:08Z", "title": "Second IQ3_S setup attempt", "kind": "installation attempt", "status": "setup validator failed; later verification separate", "description": "The official setup returned success, but the wrapper recorded installation failure because tokenizer/template artifacts were not found by its check. Protected assets remained unchanged, and models were not run.", "limits": "This failure belongs to setup validation; it is not a model-quality or inference-speed failure." }, { "id": "H13", "date_utc": "2026-10-08T22:54:00Z", "title": "Existing IQ3_S pack verification", "kind": "installation recovery verification", "status": "verified existing files; no inference", "description": "Verified the already-present weights and native tokenizer/template pack without downloading or rerunning official setup. IQ3_S was not activated as default.", "limits": "Quality and resources were not tested by this verification. It supersedes the missing-pack installation diagnosis, not its historical record." }, { "id": "H14", "date_utc": "2026-10-08T22:55:24Z", "title": "UD-Q4_K_XL installation", "kind": "installation", "status": "installed and pack-verified; no inference", "description": "Installed and verified the Q4 files/native pack alongside existing options. Protected assets remained unchanged; the default profile was not activated or replaced.", "limits": "Different file-tier I/O and compatibility conversion make later Q4 trials configured-profile tests, not pure quantization isolation." }, { "id": "H15", "date_utc": "2026-10-08T23:21:39Z", "title": "Higher-precision coding and QA suite attempt", "kind": "aborted quality/resource study", "status": "halted after one IQ2 call by resource gate", "description": "A planned four-quant suite included four coding fixtures and four QA fixtures per model. Only one IQ2_XS generation call was entered before resource qualification failed; sampled system-commit headroom fell to 0.35 GiB. Later candidate calls were not run.", "limits": "Not a completed four-model coding benchmark. Unattempted cells are absent evidence, not failed answers; the stop does not prove OOM or a candidate-quality disadvantage." }, { "id": "H16", "date_utc": "2026-10-09T00:08:29Z", "title": "Buffered-cache performance qualification", "kind": "bounded throughput/resource probe", "status": "IQ3_S qualified; Q4 safety gate failed", "description": "Two 512-token, no-thinking calls were attempted. IQ3_S completed its phase; Q4 failed the minimum sampled physical-headroom policy, with a 3.74 GiB minimum below the 4 GiB gate. Later samples were stopped.", "limits": "Intentional length termination is a bounded throughput observation, not a completed high-quality developer answer. No quality evaluation." }, { "id": "H17", "date_utc": "2026-10-09T00:14:45Z", "title": "Mapped-page-release performance probe", "kind": "bounded memory-policy probe", "status": "IQ3_S qualified; Q4 safety gate failed", "description": "A distinct mapped-page-release tuning study attempted two calls. IQ3_S completed; Q4 again crossed the physical-headroom gate, at 3.91 GiB. This failed configuration remains recorded.", "limits": "Not pooled with the first probe. A page-release hint does not cap total process or system memory." }, { "id": "H18", "date_utc": "2026-10-09T00:20:13Z", "title": "Direct-NVMe pageable versus pinned expert budgets", "kind": "four-profile bounded speed study", "status": "four phases completed and resource-qualified", "description": "One no-thinking 512-token observation per profile: IQ3_S pageable24 18.2 tokens/s, pinned16 20.4 tokens/s; Q4 pageable24 5.9 tokens/s, pinned16 6.7 tokens/s. Actual expert residency was below requested budgets. The profiles used direct file reads rather than the earlier buffered-cache setup.", "limits": "Single observations, fixed order, shared OS cache, intentional cap termination. These are historical API decode rates for that exact profile/workload, not final-answer latency, statistical superiority or globally optimal settings." }, { "id": "H19", "date_utc": "2026-10-09T00:46:50Z", "title": "Long-reasoning IQ3_S validator exposure", "kind": "stability and bounded code contract", "status": "one long request passed its recorded exposure/resource/code gates", "description": "One high-reasoning request used the pinned16 profile, produced 5,700 generated tokens over 470.27 seconds, and exposed nonempty reasoning. The independent bounded logical-proposal validator passed 54/54 checks. The sampling coverage gate passed with 443 call-time samples.", "limits": "Reasoning characters are not an exact reasoning-token count. One validator request does not establish full developer quality, production stability or live Unreal automation." }, { "id": "H20", "date_utc": "2026-10-09T02:08:52Z", "title": "All-installed-options v1", "kind": "broad matrix attempt", "status": "failed before benchmark calls; superseded", "description": "The 179-request matrix planned ten artifact/configuration options. No generation or embedding calls were entered. Preloads failed, an installed-asset/protected-state gate failed, and the study halted before Strata phases.", "limits": "Zero benchmark responses: no quality or speed ranking can be drawn. The installed-asset check does not by itself establish the causal mechanism of the drift." }, { "id": "H21", "date_utc": "2026-10-09T02:30:10Z", "title": "Private-alias readiness isolation", "kind": "metadata readiness check", "status": "passed; zero inference", "description": "Verified isolated alias bindings using metadata GET/show checks only. No model loads, generations or embeddings were performed; protected state and released ports passed the recorded readiness checks.", "limits": "Readiness is not model-performance or capability evidence. GGML and llamacpp names identify installed artifacts; their native GGUF route is not evidence of two independently implemented engines." }, { "id": "H22", "date_utc": "2026-10-09T02:32:00Z", "title": "All-installed-options private v2", "kind": "broad matrix attempt", "status": "interrupted/unsealed; superseded by checkpoint v3", "description": "The saved summary counted three entered generation calls, no embeddings, and a pending call in the first GPT-OSS phase. It remained marked running and incomplete without a final terminal.", "limits": "A persisted pending call is not verified dispatch or successful delivery. No lifecycle success is invented from later idle-state observations." }, { "id": "H23", "date_utc": "2026-10-09T19:21:12Z", "title": "Detached-launch survival probe", "kind": "runner infrastructure probe", "status": "launch refused before inference", "description": "A newly created child remained inside a Windows Job, so the launcher refused detached execution and stopped its owned child. The evidence explicitly says inference had not started.", "limits": "An execution-lifetime infrastructure failure, not a model-quality or speed result. The directory timestamp differs from the actual recorded event time; the latter is used here." }, { "id": "H24", "date_utc": "2026-10-09T19:28:00Z", "title": "Checkpointed all-installed-options v3", "kind": "checkpointed engineering matrix", "status": "controller finished; successful full matrix remained incomplete", "description": "Canonical attempts reached 156/179: 151 generation calls and five embeddings. Eight candidate segments completed; original IQ2_XS failed after three records on malformed structured output/response contract, and original Q4 failed after eight records on an owned deadline. One earlier Q4 interruption attempt with two entered calls remains separately excluded. Synthetic tracks covered code, QA, context, workflow, text throughput, assisted vision for eligible models, and embeddings.", "limits": "execution_state complete does not mean benchmark_execution_complete. Failed original candidates are not promoted by later supplements. No pooled all-time ranking." }, { "id": "H25", "date_utc": "2026-10-10T04:06:00Z", "title": "Independent IQ2 missing-fourteen-cell supplement", "kind": "supplement attempt", "status": "interrupted/unfinalized; diagnostic only", "description": "Planned fourteen missing cells without altering v3. Nine complete response records were persisted; the tenth entered-call counter was a pending intent with no finalized response. The historical lifecycle/release remained unverified.", "limits": "All nine records are diagnostic only, never formal quality or speed samples. The pending cell is not confirmed dispatched. Four never-attempted tail targets were later tested separately." }, { "id": "H26", "date_utc": "2026-10-10T22:15:20Z", "title": "Independent Q4 nine-cell tail", "kind": "sealed supplement", "status": "nine requests attempted and terminal-sealed; six complete context/workflow responses", "description": "Completed the nine-cell tail without importing original Q4 measurements. Six complete answer deliveries and three intentional length-bounded speed samples were recorded. Its own three-repetition text-decode group had median 3.9 tokens/s, range 3.8–4.1.", "limits": "Original Q4 deadline/lifecycle failure remains a failure. This completed tail is not a completed full Q4 benchmark or a re-run of its failed original cells." }, { "id": "H27", "date_utc": "2026-10-10T22:42:00Z", "title": "Independent IQ2 four-cell recovery tail", "kind": "sealed recovery segment", "status": "four requests attempted and terminal-sealed; one complete workflow response", "description": "Tested only four never-attempted targets: one workflow case and three speed repetitions. It did not repeat the original failed cell, interrupted nine responses, or pending intent. Its own text-decode group had median 21.4 tokens/s, range 18.9–22.0.", "limits": "The original IQ2 malformed-response failure and interrupted lifecycle remain unchanged. Four scoped tail calls are not a repaired full IQ2 benchmark." } ], "public_additional_evidence": [ { "id": "A01", "title": "Initial deployment and lifecycle engineering", "description": "The historical deployment record reports five passing client failure unit tests and verified owned-server reuse, Windows PowerShell start/stop/restart, unload followed by owned-process-tree stop, and a retested PowerShell date/time compatibility fix. These are runtime engineering checks, not model expertise or live Unreal acceptance." }, { "id": "A02", "title": "IQ3 study tooling regression and evidence audit", "description": "A stored no-network/model mock regression reports 75 tests, zero failures/errors, and zero actual model or HTTP calls. The stored evidence audit checked identity, arithmetic and literal review excerpts; it did not validate the correctness of reviewer judgments. Report link validation passed, while visual render acceptance was explicitly not claimed." }, { "id": "A03", "title": "Blind-review and report snapshots", "description": "Four timestamped v3 review exports each retained 28 reviewed QA answers; they are evolving views of existing evidence, not four new benchmark runs. A later code-accounting correction preserves not-executed versus failed checks. Human acceptance remains pending; duplicate answers provide limited consistency checks, not calibrated inter-rater reliability." }, { "id": "A04", "title": "Interrupted IQ2 recovery byte seal", "description": "A new read-only recovery manifest preserved nine response records and one pending intent. It sealed captured bytes but did not fabricate an old successful terminal, historical release, or lifecycle qualification." }, { "id": "A05", "title": "Final interrupted-evidence snapshot", "description": "The October 10 snapshot reports 179/179 canonical attempted-or-intent targets, 178 recorded response targets, one unfinalized pending target, 110 formal complete deliveries and 128 observed complete deliveries including diagnostics. Provenance was verified in that snapshot; full-matrix completion, all-selected-lifecycle qualification and recorded termination verification remained false. The snapshot performed no new inference and no live-state check." } ], "public_coder_and_quantization_scope": { "coding_fixtures": [ "Minimal ring-buffer repair", "GPU interval union and percentile calculations", "Bounded logical-proposal validator", "Bilingual capture-status formatter" ], "v3_qualified_strata_results": [ { "study": "original checkpoint v3", "quantization": "IQ3_XXS", "whole_code_cases_passed": 3, "planned_code_cases": 4, "passed_executed_checks": 124, "executed_checks": 124, "not_executed_checks": 54, "note": "The unexecuted fourth-case checks are not converted into passing or failed checks." }, { "study": "original checkpoint v3", "quantization": "IQ3_S", "whole_code_cases_passed": 4, "planned_code_cases": 4, "passed_executed_checks": 178, "executed_checks": 178, "not_executed_checks": 0, "note": "A result for four authored synthetic fixtures; not general coding expertise or live UE editing." } ], "failed_or_diagnostic_strata_results": [ { "study": "original checkpoint v3", "quantization": "IQ2_XS", "observed_passed_code_cases": 2, "passed_executed_checks": 70, "not_executed_checks": 108, "formal_metrics_eligible": false }, { "study": "original checkpoint v3", "quantization": "UD-Q4_K_XL", "observed_passed_code_cases": 3, "passed_executed_checks": 131, "not_executed_checks": 47, "formal_metrics_eligible": false }, { "study": "interrupted IQ2 missing14", "quantization": "IQ2_XS", "observed_passed_code_cases": 1, "passed_executed_checks": 54, "formal_metrics_eligible": false } ], "quantization_decision": "The earlier matched 20-call study did not establish IQ3_XXS as a quality upgrade over IQ2_XS. The later v3 coding fixtures observed a stronger IQ3_S contract result than IQ3_XXS in that exact study. These are separate findings, not contradictory universal intelligence rankings. Installation, resource qualification, deterministic correctness and human acceptance remain separate gates.", "v3_speed_groups_separate": [ { "study": "original checkpoint v3", "quantization": "IQ3_XXS", "repetitions": 3, "median_api_decode_tokens_per_second": 12.7, "range_api_decode_tokens_per_second": [ 10.3, 12.9 ] }, { "study": "original checkpoint v3", "quantization": "IQ3_S", "repetitions": 3, "median_api_decode_tokens_per_second": 8.3, "range_api_decode_tokens_per_second": [ 8.1, 8.7 ] }, { "study": "independent IQ2 tail4", "quantization": "IQ2_XS", "repetitions": 3, "median_api_decode_tokens_per_second": 21.4, "range_api_decode_tokens_per_second": [ 18.9, 22 ] }, { "study": "independent Q4 tail9", "quantization": "UD-Q4_K_XL", "repetitions": 3, "median_api_decode_tokens_per_second": 3.9, "range_api_decode_tokens_per_second": [ 3.8, 4.1 ] } ], "speed_limit": "Each group is descriptive within one frozen study/candidate/case. Later tails were measured at different times and cannot be pooled with original lifecycle, cold-load, memory or quality evidence. Three repetitions do not establish p95, confidence intervals or a universal speed ranking." }, "public_limitations": [ "No live Unreal profiling, edits, rendering optimization, production automation or model training was established by these model-only fixture studies.", "Successfully stored, delivered, deterministic-correct, lifecycle-qualified and human-accepted are distinct evidence gates.", "179 targets means canonical attempted-or-intent coverage, not 179 successful responses. A pending intent is not confirmed model dispatch.", "Completed recovery segments do not repair historical malformed-output, deadline, interrupted or unsealed failures.", "Configured memory budgets are not system/process RAM caps. Whole-system sampling around one second is not model-exclusive allocation or an exact transient-peak measurement.", "Cold means model residency-cold, not disk/OS-cache cold. Fixed order, shared caches and first-request initialization affect timings.", "6K/24K-character context fixtures do not validate 32K/64K-token context occupancy. The original 65,536-token configuration was not full-context stress-tested.", "Vision fixtures used assisted PNG plus RenderSpec/host statistics and reused development labels; they are not independent pure-vision holdouts or rendering proof.", "Embedding retrieval used eight developer-authored queries, not full retrieval-service or RAG product acceptance.", "Agent quality grades are preliminary and not calibrated human acceptance. Schema validity does not prove correct profiler reasoning or a safe/correct UE plan.", "No full raw responses, reasoning traces, runtime commands, exact machine paths, process identifiers, local endpoints, model weights or secrets are needed in the public explanation." ] }