close the RECAP critic-policy loop

tests/test_experience_data.py

@@ -71,6 +71,8 @@ def test_experience_dataset_contract_and_intervention_label(tmp_path: Path) -> N
     assert intervention["returns"].shape == (16,)
     assert intervention["return_target"].shape == ()
     assert intervention["value_target"].shape == ()
+    assert intervention["value_histories"].shape == (17, 8, 2)
+    assert intervention["value_targets"].shape == (17,)
     assert intervention["advantages"].shape == (16,)
     assert intervention["advantage_label"] == "positive"
     assert intervention["reasoning"] == "Steer left."
@@ -78,3 +80,21 @@ def test_experience_dataset_contract_and_intervention_label(tmp_path: Path) -> N
         intervention["actions"], intervention["history_xy"]
     )
     torch.testing.assert_close(decoded, intervention["future_xy"], atol=1e-5, rtol=1e-5)
+
+
+def test_learned_value_predictions_drive_advantage_relabeling(tmp_path: Path) -> None:
+    manifest = _make_source_project(tmp_path / "source")
+    dataset = Pi06ExperienceDataset(manifest, image_size=32)
+    before = dataset.advantages.clone()
+    predicted = torch.zeros(len(dataset.episodes), dataset.action_horizon + 1)
+    predicted[:, :-1] = torch.linspace(-0.9, -0.1, dataset.action_horizon)
+
+    dataset.relabel_from_value_predictions(predicted)
+
+    assert not torch.allclose(dataset.advantages, before)
+    intervention_index = next(
+        index
+        for index, episode in enumerate(dataset.episodes)
+        if episode.source_type == "intervention_correction"
+    )
+    assert bool(dataset.positive_episodes[intervention_index])

tests/test_tokenizer.py

@@ -37,6 +37,28 @@ def test_omitted_advantage_is_absent_not_a_third_condition_token() -> None:
     assert tokenizer.ADVANTAGE_NEGATIVE not in sequence.input_ids
 
 
+def test_value_input_does_not_encode_experience_outcome() -> None:
+    tokenizer = MiniTokenizer()
+    history = torch.zeros(8, 2)
+    value_input = tokenizer.build_value_input(history)
+
+    assert tokenizer.METADATA_DEMO not in value_input
+    assert tokenizer.METADATA_UNDERSTEER not in value_input
+    assert tokenizer.METADATA_STOP not in value_input
+    assert tokenizer.METADATA_INTERVENTION not in value_input
+
+
+def test_predicted_subtask_can_be_connected_to_action_condition() -> None:
+    tokenizer = MiniTokenizer()
+    history = torch.zeros(8, 2)
+    predicted = tokenizer.subtasks.encode("steer_left")
+    condition = tokenizer.build_action_condition(history, predicted, "positive")
+
+    assert condition[-2].item() == predicted
+    assert condition[-1].item() == tokenizer.ADVANTAGE_POSITIVE
+    assert tokenizer.ACTION_START not in condition
+
+
 def test_labels_predict_subtask_and_fast_but_not_external_advantage() -> None:
     tokenizer = MiniTokenizer()
     positive = tokenizer.build_policy_sequence(