# RR10 v1.0.1 RUNNABLE PACKAGE — mail-safe bundle Canonical benchmark JSONL is unchanged from v1.0. This patch only fixes the runnable/test packaging layer. --- FILE: README.md --- ``` # RR-10 Evidence-Aware Directional Interpretation Pilot — v1.0 (runnable-layer patch v1.0.1) Runnable collaboration package for the 200-item pilot discussed in the OSDR AI/ML AWG thread. **Status:** pilot / collaboration artifact. This package does **not** claim to be an official NASA benchmark. ## Canonical data - `benchmark.jsonl` — canonical 200-item JSONL. - `benchmark_JSONL_PAYLOAD.md` — byte-for-byte identical transport copy for Discourse/email gateways that reject `.jsonl`; rename it to `.jsonl` after download. - SHA-256: `aeb8511444ac72f44c6489954495db6cc67b75e977c7f9963ddbc37004533af5` - 100 `fragility_trap` items + 100 `concordant_control` items. - 6 organism::tissue contexts, 22 OSD datasets, 501 provenance rows. The canonical object intentionally has no explicit `pair_id`. Pairing is derived by numeric suffix (`FRAG-00001` ↔ `CTRL-00001`). `validate.py` verifies that all 100 derived pairs share the same context. Do not mutate the canonical JSONL merely to add a pair field; derive it downstream. ## Validation ## Runnable-layer patch v1.0.1 Packaging-only patch: added the omitted gold smoke-test file and corrected the JSONL newline in the all-indeterminate regression test. Canonical benchmark bytes and SHA-256 are unchanged. ``` --- FILE: manifest.json --- ``` { "benchmark_name": "RR-10 Evidence-Aware Directional Interpretation Pilot", "benchmark_version": "1.0", "schema_version": "1.0", "status": "pilot / collaboration package; not an official NASA benchmark", "canonical_file": "benchmark.jsonl", "transport_fallback_file": "benchmark_JSONL_PAYLOAD.md", "canonical_sha256": "aeb8511444ac72f44c6489954495db6cc67b75e977c7f9963ddbc37004533af5", "records": 200, "item_counts": { "fragility_trap": 100, "concordant_control": 100 }, "expected_label_counts": { "indeterminate": 100, "upregulated": 44, "downregulated": 56 }, "contexts": { "homo_sapiens::cells_cultured": 4, "drosophila_melanogaster::whole_organism": 10, "homo_sapiens::cells": 44, "mus_musculus::thymus": 46, "mus_musculus::liver": 46, "arabidopsis_thaliana::seedlings": 50 }, "osd_datasets": [ "OSD-1", "OSD-4", "OSD-25", "OSD-37", "OSD-47", "OSD-48", "OSD-168", "OSD-173", "OSD-242", "OSD-244", "OSD-245", "OSD-278", "OSD-321", "OSD-323", "OSD-421", "OSD-515", "OSD-546", "OSD-588", "OSD-635", "OSD-863", "OSD-867", "OSD-871" ], "osd_dataset_count": 22, "provenance_rows": 501, "pairing": { "stored_explicitly": false, "derivation": "numeric suffix: OSDR-FRAG-00001 <-> OSDR-CTRL-00001, etc.", "verified_same_context_for_all_pairs": true }, "generated_utc": "2026-09-18T04:46:08.308486+00:00", "source_thread": "https://awg.osdr.space/t/can-frontier-llms-accurately-interpret-spaceflight-biology-rr-10-benchmark-interactive-3d-viewer/4569/7" } ``` --- FILE: schema.json --- ``` { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://omros-lab.local/rr10/v1.0/schema.json", "title": "RR-10 evidence-aware directional interpretation pilot item", "type": "object", "additionalProperties": false, "required": [ "id", "item_type", "prompt", "gene", "organism", "tissue", "context", "expected", "accept", "reject", "evidence", "provenance" ], "properties": { "id": { "type": "string", "pattern": "^OSDR-(FRAG|CTRL)-\\d{5}$" }, "item_type": { "enum": [ "fragility_trap", "concordant_control" ] }, "prompt": { "type": "string", "minLength": 1 }, "gene": { "type": "string", "minLength": 1 }, "organism": { "type": "string", "minLength": 1 }, "tissue": { "type": "string", "minLength": 1 }, "context": { "type": "string", "minLength": 3 }, "expected": { "enum": [ "upregulated", "downregulated", "indeterminate" ] }, "accept": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "reject": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "evidence": { "type": "object", "additionalProperties": false, "required": [ "n_datasets", "datasets", "min_abs_log2FC", "max_abs_log2FC", "max_q", "discordant" ], "properties": { "n_datasets": { "type": "integer", "minimum": 1 }, "datasets": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "string", "pattern": "^OSD-\\d+$" } }, "min_abs_log2FC": { "type": "number", "minimum": 0 }, "max_abs_log2FC": { "type": "number", "minimum": 0 }, "max_q": { "type": "number", "minimum": 0, "maximum": 1 }, "discordant": { "type": "boolean" } } }, "provenance": { "type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, "required": [ "osd", "log2FC_SF_vs_GC", "q", "contrast", "orientation_sign_applied", "source_file", "gene_id_column" ], "properties": { "osd": { "type": "string", "pattern": "^OSD-\\d+$" }, "log2FC_SF_vs_GC": { "type": "number" }, "q": { "type": "number", "minimum": 0, "maximum": 1 }, "contrast": { "type": "string", "minLength": 1 }, "orientation_sign_applied": { "type": "integer", "enum": [ -1, 1 ] }, "source_file": { "type": "string", "minLength": 1 }, "gene_id_column": { "type": "string", "minLength": 1 } } } } } } ``` --- FILE: validate.py --- ``` #!/usr/bin/env python3 import argparse, hashlib, json, math, re, sys from pathlib import Path EXPECTED_SHA256 = "aeb8511444ac72f44c6489954495db6cc67b75e977c7f9963ddbc37004533af5" ID_RE = re.compile(r"^OSDR-(FRAG|CTRL)-(\d{5})$") VALID_LABELS = {"upregulated", "downregulated", "indeterminate"} def fail(errors, item_id, message): errors.append({"id": item_id, "error": message}) def load_jsonl(path): items=[] for n,line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(),1): if not line.strip(): continue try: items.append(json.loads(line)) except Exception as e: raise ValueError(f"line {n}: invalid JSON: {e}") return items def sign(x): return 1 if x > 0 else -1 if x < 0 else 0 def validate(path, check_hash=True): path=Path(path); errors=[] digest=hashlib.sha256(path.read_bytes()).hexdigest() if check_hash and digest != EXPECTED_SHA256: fail(errors, "FILE", f"SHA-256 mismatch: {digest}") items=load_jsonl(path) if len(items)!=200: fail(errors,"FILE",f"expected 200 records, found {len(items)}") seen=set(); by_id={} required={"id","item_type","prompt","gene","organism","tissue","context","expected","accept","reject","evidence","provenance"} for x in items: iid=x.get("id","") missing=required-set(x) if missing: fail(errors,iid,f"missing fields: {sorted(missing)}"); continue if iid in seen: fail(errors,iid,"duplicate id") seen.add(iid); by_id[iid]=x m=ID_RE.match(iid) if not m: fail(errors,iid,"invalid id format") typ=x["item_type"] if typ not in {"fragility_trap","concordant_control"}: fail(errors,iid,"invalid item_type") if x["expected"] not in VALID_LABELS: fail(errors,iid,"invalid expected label") if x["context"] != f'{x["organism"]}::{x["tissue"]}': fail(errors,iid,"context != organism::tissue") ev=x["evidence"]; prov=x["provenance"] if not isinstance(prov,list) or not prov: fail(errors,iid,"provenance must be non-empty list"); continue p_osds=[p.get("osd") for p in prov] if ev.get("n_datasets") != len(set(p_osds)): fail(errors,iid,"evidence.n_datasets inconsistent with provenance") if set(ev.get("datasets",[])) != set(p_osds): fail(errors,iid,"evidence.datasets inconsistent with provenance") vals=[abs(float(p["log2FC_SF_vs_GC"])) for p in prov] qs=[float(p["q"]) for p in prov] if not math.isclose(float(ev["min_abs_log2FC"]),min(vals),abs_tol=1e-4): fail(errors,iid,"min_abs_log2FC inconsistent") if not math.isclose(float(ev["max_abs_log2FC"]),max(vals),abs_tol=1e-4): fail(errors,iid,"max_abs_log2FC inconsistent") if not math.isclose(float(ev["max_q"]),max(qs),abs_tol=5e-6): fail(errors,iid,"max_q inconsistent") signs={sign(float(p["log2FC_SF_vs_GC"])) for p in prov} discordant=(1 in signs and -1 in signs) if bool(ev["discordant"]) != discordant: fail(errors,iid,"discordant flag inconsistent with provenance") if typ=="fragility_trap": if x["expected"]!="indeterminate": fail(errors,iid,"trap expected label must be indeterminate") if not discordant: fail(errors,iid,"trap provenance is not directionally discordant") else: desired=1 if x["expected"]=="upregulated" else -1 if x["expected"]=="downregulated" else 0 if desired==0: fail(errors,iid,"control expected label cannot be indeterminate") if signs != {desired}: fail(errors,iid,f"control provenance signs {sorted(signs)} disagree with expected") traps=[x for x in items if x.get("item_type")=="fragility_trap"] ctrls=[x for x in items if x.get("item_type")=="concordant_control"] if len(traps)!=100 or len(ctrls)!=100: fail(errors,"FILE",f"expected 100 traps/100 controls; got {len(traps)}/{len(ctrls)}") for k in range(1,101): f=f"OSDR-FRAG-{k:05d}"; c=f"OSDR-CTRL-{k:05d}" if f not in by_id or c not in by_id: fail(errors,"PAIR",f"missing pair {k:05d}"); continue if by_id[f]["context"] != by_id[c]["context"]: fail(errors,"PAIR",f"pair {k:05d} context mismatch") return { "ok": not errors, "sha256": digest, "records": len(items), "errors": errors, "contexts": sorted(set(x.get("context","") for x in items)), "provenance_rows": sum(len(x.get("provenance",[])) for x in items) } def main(): ap=argparse.ArgumentParser(); ap.add_argument("benchmark", nargs="?", default="benchmark.jsonl"); ap.add_argument("--skip-hash",action="store_true") a=ap.parse_args(); r=validate(a.benchmark, not a.skip_hash); print(json.dumps(r,indent=2)); sys.exit(0 if r["ok"] else 2) if __name__=="__main__": main() ``` --- FILE: score.py --- ``` #!/usr/bin/env python3 import argparse, csv, json, re, sys from collections import Counter, defaultdict from pathlib import Path LABELS=("upregulated","downregulated","indeterminate") def load_benchmark(path): return [json.loads(x) for x in Path(path).read_text(encoding="utf-8").splitlines() if x.strip()] def load_predictions(path): p=Path(path); rows=[] if p.suffix.lower()==".csv": with p.open(encoding="utf-8",newline="") as f: rows=list(csv.DictReader(f)) else: rows=[json.loads(x) for x in p.read_text(encoding="utf-8").splitlines() if x.strip()] out={} for r in rows: iid=r.get("id") if not iid: continue raw=next((r.get(k) for k in ("prediction","answer","label","response") if r.get(k) is not None),None) out[iid]=raw return out def normalize(raw): if raw is None: return None s=str(raw).strip().lower().replace("_"," ") direct={ "up":"upregulated","upregulated":"upregulated","increase":"upregulated","increased":"upregulated", "down":"downregulated","downregulated":"downregulated","decrease":"downregulated","decreased":"downregulated", "indeterminate":"indeterminate","unknown":"indeterminate","undetermined":"indeterminate", "not determinable":"indeterminate","insufficient evidence":"indeterminate","evidence is contradictory":"indeterminate", "contradictory":"indeterminate" } if s in direct: return direct[s] hits=set() if re.search(r"\b(upregulated|up-regulated|increased?)\b",s): hits.add("upregulated") if re.search(r"\b(downregulated|down-regulated|decreased?)\b",s): hits.add("downregulated") if any(p in s for p in ("indeterminate","not determinable","cannot determine","insufficient evidence","evidence is contradictory","contradictory evidence")): hits.add("indeterminate") return next(iter(hits)) if len(hits)==1 else None def f1_for(label,gold,pred): tp=sum(g==label and p==label for g,p in zip(gold,pred)); fp=sum(g!=label and p==label for g,p in zip(gold,pred)); fn=sum(g==label and p!=label for g,p in zip(gold,pred)) prec=tp/(tp+fp) if tp+fp else 0.0; rec=tp/(tp+fn) if tp+fn else 0.0; f1=2*prec*rec/(prec+rec) if prec+rec else 0.0 return {"precision":prec,"recall":rec,"f1":f1,"support":sum(g==label for g in gold)} def main(): ap=argparse.ArgumentParser(description="Score RR-10 v1.0 pilot predictions") ap.add_argument("predictions"); ap.add_argument("--benchmark",default="benchmark.jsonl"); ap.add_argument("--errors-csv",default="") a=ap.parse_args(); items=load_benchmark(a.benchmark); supplied=load_predictions(a.predictions) gold=[]; pred=[]; rows=[]; by_id={x["id"]:x for x in items} for x in items: p=normalize(supplied.get(x["id"])); g=x["expected"]; gold.append(g); pred.append(p) rows.append({"id":x["id"],"item_type":x["item_type"],"context":x["context"],"expected":g,"prediction":p or "INVALID/MISSING","correct":p==g}) traps=[r for r in rows if r["item_type"]=="fragility_trap"]; ctrls=[r for r in rows if r["item_type"]=="concordant_control"] def rate(rs,fn): return sum(fn(r) for r in rs)/len(rs) if rs else 0.0 pdr_n=0; pair_complete=0 for k in range(1,101): f=next(r for r in rows if r["id"]==f"OSDR-FRAG-{k:05d}"); c=next(r for r in rows if r["id"]==f"OSDR-CTRL-{k:05d}") if f["prediction"]!="INVALID/MISSING" and c["prediction"]!="INVALID/MISSING": pair_complete+=1 if f["correct"] and c["correct"]: pdr_n+=1 valid=[p for p in pred if p in LABELS] per_context={} for ctx in sorted(set(r["context"] for r in rows)): rr=[r for r in rows if r["context"]==ctx] per_context[ctx]={"n":len(rr),"accuracy":rate(rr,lambda r:r["correct"]),"invalid_or_missing":sum(r["prediction"]=="INVALID/MISSING" for r in rr)} report={ "n_items":len(rows), "coverage":len(valid)/len(rows), "accuracy_total":rate(rows,lambda r:r["correct"]), "trap_accuracy":rate(traps,lambda r:r["correct"]), "control_accuracy":rate(ctrls,lambda r:r["correct"]), "trap_overclaim_rate":rate(traps,lambda r:r["prediction"] in ("upregulated","downregulated")), "overall_abstention_rate":rate(rows,lambda r:r["prediction"]=="indeterminate"), "control_abstention_rate":rate(ctrls,lambda r:r["prediction"]=="indeterminate"), "paired_directional_robustness":pdr_n/100.0, "pair_complete_rate":pair_complete/100.0, "class_metrics":{label:f1_for(label,gold,pred) for label in LABELS}, "macro_f1":sum(f1_for(label,gold,pred)["f1"] for label in LABELS)/3.0, "prediction_counts":dict(Counter(str(x) for x in pred)), "per_context":per_context, "definitions":{ "control_abstention_rate":"fraction of concordant controls predicted indeterminate; report this when discussing blanket abstention", "paired_directional_robustness":"fraction of the 100 numeric-suffix trap/control pairs for which both predictions are correct" } } print(json.dumps(report,indent=2)) if a.errors_csv: with open(a.errors_csv,"w",newline="",encoding="utf-8") as f: w=csv.DictWriter(f,fieldnames=rows[0].keys()); w.writeheader(); w.writerows([r for r in rows if not r["correct"]]) if __name__=="__main__": main() ``` --- FILE: example_predictions.jsonl --- ``` {"id":"OSDR-FRAG-00001","prediction":"indeterminate"} {"id":"OSDR-FRAG-00002","prediction":"indeterminate"} {"id":"OSDR-FRAG-00003","prediction":"indeterminate"} {"id":"OSDR-FRAG-00004","prediction":"indeterminate"} {"id":"OSDR-FRAG-00005","prediction":"indeterminate"} {"id":"OSDR-FRAG-00006","prediction":"indeterminate"} {"id":"OSDR-FRAG-00007","prediction":"indeterminate"} {"id":"OSDR-FRAG-00008","prediction":"indeterminate"} {"id":"OSDR-FRAG-00009","prediction":"indeterminate"} {"id":"OSDR-FRAG-00010","prediction":"indeterminate"} {"id":"OSDR-FRAG-00011","prediction":"indeterminate"} {"id":"OSDR-FRAG-00012","prediction":"indeterminate"} {"id":"OSDR-FRAG-00013","prediction":"indeterminate"} {"id":"OSDR-FRAG-00014","prediction":"indeterminate"} {"id":"OSDR-FRAG-00015","prediction":"indeterminate"} {"id":"OSDR-FRAG-00016","prediction":"indeterminate"} {"id":"OSDR-FRAG-00017","prediction":"indeterminate"} {"id":"OSDR-FRAG-00018","prediction":"indeterminate"} {"id":"OSDR-FRAG-00019","prediction":"indeterminate"} {"id":"OSDR-FRAG-00020","prediction":"indeterminate"} {"id":"OSDR-FRAG-00021","prediction":"indeterminate"} {"id":"OSDR-FRAG-00022","prediction":"indeterminate"} {"id":"OSDR-FRAG-00023","prediction":"indeterminate"} {"id":"OSDR-FRAG-00024","prediction":"indeterminate"} {"id":"OSDR-FRAG-00025","prediction":"indeterminate"} {"id":"OSDR-FRAG-00026","prediction":"indeterminate"} {"id":"OSDR-FRAG-00027","prediction":"indeterminate"} {"id":"OSDR-FRAG-00028","prediction":"indeterminate"} {"id":"OSDR-FRAG-00029","prediction":"indeterminate"} {"id":"OSDR-FRAG-00030","prediction":"indeterminate"} {"id":"OSDR-FRAG-00031","prediction":"indeterminate"} {"id":"OSDR-FRAG-00032","prediction":"indeterminate"} {"id":"OSDR-FRAG-00033","prediction":"indeterminate"} {"id":"OSDR-FRAG-00034","prediction":"indeterminate"} {"id":"OSDR-FRAG-00035","prediction":"indeterminate"} {"id":"OSDR-FRAG-00036","prediction":"indeterminate"} {"id":"OSDR-FRAG-00037","prediction":"indeterminate"} {"id":"OSDR-FRAG-00038","prediction":"indeterminate"} {"id":"OSDR-FRAG-00039","prediction":"indeterminate"} {"id":"OSDR-FRAG-00040","prediction":"indeterminate"} {"id":"OSDR-FRAG-00041","prediction":"indeterminate"} {"id":"OSDR-FRAG-00042","prediction":"indeterminate"} {"id":"OSDR-FRAG-00043","prediction":"indeterminate"} {"id":"OSDR-FRAG-00044","prediction":"indeterminate"} {"id":"OSDR-FRAG-00045","prediction":"indeterminate"} {"id":"OSDR-FRAG-00046","prediction":"indeterminate"} {"id":"OSDR-FRAG-00047","prediction":"indeterminate"} {"id":"OSDR-FRAG-00048","prediction":"indeterminate"} {"id":"OSDR-FRAG-00049","prediction":"indeterminate"} {"id":"OSDR-FRAG-00050","prediction":"indeterminate"} {"id":"OSDR-FRAG-00051","prediction":"indeterminate"} {"id":"OSDR-FRAG-00052","prediction":"indeterminate"} {"id":"OSDR-FRAG-00053","prediction":"indeterminate"} {"id":"OSDR-FRAG-00054","prediction":"indeterminate"} {"id":"OSDR-FRAG-00055","prediction":"indeterminate"} {"id":"OSDR-FRAG-00056","prediction":"indeterminate"} {"id":"OSDR-FRAG-00057","prediction":"indeterminate"} {"id":"OSDR-FRAG-00058","prediction":"indeterminate"} {"id":"OSDR-FRAG-00059","prediction":"indeterminate"} {"id":"OSDR-FRAG-00060","prediction":"indeterminate"} {"id":"OSDR-FRAG-00061","prediction":"indeterminate"} {"id":"OSDR-FRAG-00062","prediction":"indeterminate"} {"id":"OSDR-FRAG-00063","prediction":"indeterminate"} {"id":"OSDR-FRAG-00064","prediction":"indeterminate"} {"id":"OSDR-FRAG-00065","prediction":"indeterminate"} {"id":"OSDR-FRAG-00066","prediction":"indeterminate"} {"id":"OSDR-FRAG-00067","prediction":"indeterminate"} {"id":"OSDR-FRAG-00068","prediction":"indeterminate"} {"id":"OSDR-FRAG-00069","prediction":"indeterminate"} {"id":"OSDR-FRAG-00070","prediction":"indeterminate"} {"id":"OSDR-FRAG-00071","prediction":"indeterminate"} {"id":"OSDR-FRAG-00072","prediction":"indeterminate"} {"id":"OSDR-FRAG-00073","prediction":"indeterminate"} {"id":"OSDR-FRAG-00074","prediction":"indeterminate"} {"id":"OSDR-FRAG-00075","prediction":"indeterminate"} {"id":"OSDR-FRAG-00076","prediction":"indeterminate"} {"id":"OSDR-FRAG-00077","prediction":"indeterminate"} {"id":"OSDR-FRAG-00078","prediction":"indeterminate"} {"id":"OSDR-FRAG-00079","prediction":"indeterminate"} {"id":"OSDR-FRAG-00080","prediction":"indeterminate"} {"id":"OSDR-FRAG-00081","prediction":"indeterminate"} {"id":"OSDR-FRAG-00082","prediction":"indeterminate"} {"id":"OSDR-FRAG-00083","prediction":"indeterminate"} {"id":"OSDR-FRAG-00084","prediction":"indeterminate"} {"id":"OSDR-FRAG-00085","prediction":"indeterminate"} {"id":"OSDR-FRAG-00086","prediction":"indeterminate"} {"id":"OSDR-FRAG-00087","prediction":"indeterminate"} {"id":"OSDR-FRAG-00088","prediction":"indeterminate"} {"id":"OSDR-FRAG-00089","prediction":"indeterminate"} {"id":"OSDR-FRAG-00090","prediction":"indeterminate"} {"id":"OSDR-FRAG-00091","prediction":"indeterminate"} {"id":"OSDR-FRAG-00092","prediction":"indeterminate"} {"id":"OSDR-FRAG-00093","prediction":"indeterminate"} {"id":"OSDR-FRAG-00094","prediction":"indeterminate"} {"id":"OSDR-FRAG-00095","prediction":"indeterminate"} {"id":"OSDR-FRAG-00096","prediction":"indeterminate"} {"id":"OSDR-FRAG-00097","prediction":"indeterminate"} {"id":"OSDR-FRAG-00098","prediction":"indeterminate"} {"id":"OSDR-FRAG-00099","prediction":"indeterminate"} {"id":"OSDR-FRAG-00100","prediction":"indeterminate"} {"id":"OSDR-CTRL-00001","prediction":"upregulated"} {"id":"OSDR-CTRL-00002","prediction":"downregulated"} {"id":"OSDR-CTRL-00003","prediction":"upregulated"} {"id":"OSDR-CTRL-00004","prediction":"upregulated"} {"id":"OSDR-CTRL-00005","prediction":"downregulated"} {"id":"OSDR-CTRL-00006","prediction":"upregulated"} {"id":"OSDR-CTRL-00007","prediction":"upregulated"} {"id":"OSDR-CTRL-00008","prediction":"upregulated"} {"id":"OSDR-CTRL-00009","prediction":"upregulated"} {"id":"OSDR-CTRL-00010","prediction":"upregulated"} {"id":"OSDR-CTRL-00011","prediction":"downregulated"} {"id":"OSDR-CTRL-00012","prediction":"upregulated"} {"id":"OSDR-CTRL-00013","prediction":"downregulated"} {"id":"OSDR-CTRL-00014","prediction":"downregulated"} {"id":"OSDR-CTRL-00015","prediction":"downregulated"} {"id":"OSDR-CTRL-00016","prediction":"downregulated"} {"id":"OSDR-CTRL-00017","prediction":"upregulated"} {"id":"OSDR-CTRL-00018","prediction":"downregulated"} {"id":"OSDR-CTRL-00019","prediction":"downregulated"} {"id":"OSDR-CTRL-00020","prediction":"upregulated"} {"id":"OSDR-CTRL-00021","prediction":"downregulated"} {"id":"OSDR-CTRL-00022","prediction":"downregulated"} {"id":"OSDR-CTRL-00023","prediction":"upregulated"} {"id":"OSDR-CTRL-00024","prediction":"downregulated"} {"id":"OSDR-CTRL-00025","prediction":"downregulated"} {"id":"OSDR-CTRL-00026","prediction":"downregulated"} {"id":"OSDR-CTRL-00027","prediction":"downregulated"} {"id":"OSDR-CTRL-00028","prediction":"upregulated"} {"id":"OSDR-CTRL-00029","prediction":"upregulated"} {"id":"OSDR-CTRL-00030","prediction":"downregulated"} {"id":"OSDR-CTRL-00031","prediction":"downregulated"} {"id":"OSDR-CTRL-00032","prediction":"downregulated"} {"id":"OSDR-CTRL-00033","prediction":"downregulated"} {"id":"OSDR-CTRL-00034","prediction":"upregulated"} {"id":"OSDR-CTRL-00035","prediction":"downregulated"} {"id":"OSDR-CTRL-00036","prediction":"upregulated"} {"id":"OSDR-CTRL-00037","prediction":"downregulated"} {"id":"OSDR-CTRL-00038","prediction":"downregulated"} {"id":"OSDR-CTRL-00039","prediction":"downregulated"} {"id":"OSDR-CTRL-00040","prediction":"downregulated"} {"id":"OSDR-CTRL-00041","prediction":"downregulated"} {"id":"OSDR-CTRL-00042","prediction":"downregulated"} {"id":"OSDR-CTRL-00043","prediction":"downregulated"} {"id":"OSDR-CTRL-00044","prediction":"upregulated"} {"id":"OSDR-CTRL-00045","prediction":"downregulated"} {"id":"OSDR-CTRL-00046","prediction":"upregulated"} {"id":"OSDR-CTRL-00047","prediction":"upregulated"} {"id":"OSDR-CTRL-00048","prediction":"upregulated"} {"id":"OSDR-CTRL-00049","prediction":"downregulated"} {"id":"OSDR-CTRL-00050","prediction":"upregulated"} {"id":"OSDR-CTRL-00051","prediction":"upregulated"} {"id":"OSDR-CTRL-00052","prediction":"upregulated"} {"id":"OSDR-CTRL-00053","prediction":"downregulated"} {"id":"OSDR-CTRL-00054","prediction":"downregulated"} {"id":"OSDR-CTRL-00055","prediction":"downregulated"} {"id":"OSDR-CTRL-00056","prediction":"downregulated"} {"id":"OSDR-CTRL-00057","prediction":"upregulated"} {"id":"OSDR-CTRL-00058","prediction":"downregulated"} {"id":"OSDR-CTRL-00059","prediction":"upregulated"} {"id":"OSDR-CTRL-00060","prediction":"downregulated"} {"id":"OSDR-CTRL-00061","prediction":"upregulated"} {"id":"OSDR-CTRL-00062","prediction":"downregulated"} {"id":"OSDR-CTRL-00063","prediction":"downregulated"} {"id":"OSDR-CTRL-00064","prediction":"upregulated"} {"id":"OSDR-CTRL-00065","prediction":"downregulated"} {"id":"OSDR-CTRL-00066","prediction":"downregulated"} {"id":"OSDR-CTRL-00067","prediction":"upregulated"} {"id":"OSDR-CTRL-00068","prediction":"upregulated"} {"id":"OSDR-CTRL-00069","prediction":"upregulated"} {"id":"OSDR-CTRL-00070","prediction":"downregulated"} {"id":"OSDR-CTRL-00071","prediction":"upregulated"} {"id":"OSDR-CTRL-00072","prediction":"upregulated"} {"id":"OSDR-CTRL-00073","prediction":"upregulated"} {"id":"OSDR-CTRL-00074","prediction":"downregulated"} {"id":"OSDR-CTRL-00075","prediction":"upregulated"} {"id":"OSDR-CTRL-00076","prediction":"upregulated"} {"id":"OSDR-CTRL-00077","prediction":"upregulated"} {"id":"OSDR-CTRL-00078","prediction":"downregulated"} {"id":"OSDR-CTRL-00079","prediction":"upregulated"} {"id":"OSDR-CTRL-00080","prediction":"upregulated"} {"id":"OSDR-CTRL-00081","prediction":"downregulated"} {"id":"OSDR-CTRL-00082","prediction":"upregulated"} {"id":"OSDR-CTRL-00083","prediction":"downregulated"} {"id":"OSDR-CTRL-00084","prediction":"downregulated"} {"id":"OSDR-CTRL-00085","prediction":"downregulated"} {"id":"OSDR-CTRL-00086","prediction":"downregulated"} {"id":"OSDR-CTRL-00087","prediction":"downregulated"} {"id":"OSDR-CTRL-00088","prediction":"upregulated"} {"id":"OSDR-CTRL-00089","prediction":"downregulated"} {"id":"OSDR-CTRL-00090","prediction":"downregulated"} {"id":"OSDR-CTRL-00091","prediction":"upregulated"} {"id":"OSDR-CTRL-00092","prediction":"downregulated"} {"id":"OSDR-CTRL-00093","prediction":"upregulated"} {"id":"OSDR-CTRL-00094","prediction":"downregulated"} {"id":"OSDR-CTRL-00095","prediction":"downregulated"} {"id":"OSDR-CTRL-00096","prediction":"downregulated"} {"id":"OSDR-CTRL-00097","prediction":"upregulated"} {"id":"OSDR-CTRL-00098","prediction":"upregulated"} {"id":"OSDR-CTRL-00099","prediction":"downregulated"} {"id":"OSDR-CTRL-00100","prediction":"downregulated"} ``` --- FILE: tests/test_rr10.py --- ``` import json, subprocess, sys from pathlib import Path ROOT=Path(__file__).resolve().parents[1] def run(*args): return subprocess.run([sys.executable,*map(str,args)],cwd=ROOT,text=True,capture_output=True) def test_validator(): p=run(ROOT/'validate.py',ROOT/'benchmark.jsonl') assert p.returncode==0,p.stdout+p.stderr r=json.loads(p.stdout); assert r['ok'] and r['records']==200 and r['provenance_rows']==501 def test_gold_scoring(): p=run(ROOT/'score.py',ROOT/'example_predictions.jsonl','--benchmark',ROOT/'benchmark.jsonl') assert p.returncode==0,p.stdout+p.stderr r=json.loads(p.stdout) assert r['trap_accuracy']==1.0 and r['control_accuracy']==1.0 and r['paired_directional_robustness']==1.0 def test_all_indeterminate_baseline(tmp_path): bench=[json.loads(x) for x in (ROOT/'benchmark.jsonl').read_text().splitlines() if x.strip()] q=tmp_path/'all_indeterminate.jsonl' q.write_text(''.join(json.dumps({'id':x['id'],'prediction':'indeterminate'})+'\n' for x in bench)) p=run(ROOT/'score.py',q,'--benchmark',ROOT/'benchmark.jsonl'); r=json.loads(p.stdout) assert r['trap_accuracy']==1.0 and r['control_accuracy']==0.0 and r['control_abstention_rate']==1.0 and r['paired_directional_robustness']==0.0 ``` --- FILE: PATCH_NOTE_v1.0.1.md --- ``` # RR10 runnable-layer patch v1.0.1 Canonical benchmark data are unchanged. Canonical benchmark SHA-256 remains `aeb8511444ac72f44c6489954495db6cc67b75e977c7f9963ddbc37004533af5`. Packaging fixes only: 1. Added the `example_predictions.jsonl` smoke-test file referenced by README/tests. 2. Corrected the escaped newline in `tests/test_rr10.py` so the all-indeterminate baseline writes valid JSONL. No benchmark IDs, labels, prompts, evidence, provenance, or scoring definitions were changed. ``` --- FILE: LICENSE_NOTICE.md --- ``` # License / provenance notice This package does not relicense NASA OSDR source data or third-party materials. The benchmark records preserve source-file/accession provenance for scientific traceability. Before public redistribution beyond the OSDR collaboration context, confirm the applicable source-data terms and the intended benchmark license. Software files in this package are provided by OMROS LAB for collaboration/evaluation; no additional license grant is asserted in this draft package. ```