pika-git-1

Evidence

@@ -35,6 +35,10 @@ pub fn run() -> anyhow::Result<()> {
+        PhCommand::FailLane(args) => cmd_fail_lane(&cli, args),
+        PhCommand::RequeueLane(args) => cmd_requeue_lane(&cli, args),
+        PhCommand::RecoverRun(args) => cmd_recover_run(&cli, args),
+        PhCommand::WakeCi => cmd_wake_ci(&cli),

@@ -82,6 +86,10 @@ enum PhCommand {
+    FailLane(LaneActionArgs),
+    RequeueLane(LaneActionArgs),
+    RecoverRun(RecoverRunArgs),
+    WakeCi,

Evidence

@@ -92,6 +100,30 @@ struct LoginArgs {
+#[derive(Debug, Clone, clap::Args)]
+struct LaneActionArgs {
+    branch_or_id: Option<String>,
+    #[arg(long, conflicts_with = "branch_or_id")]
+    nightly_run_id: Option<i64>,
+    #[arg(
+        long,
+        conflicts_with = "lane_run_id",
+        required_unless_present = "lane_run_id"
+    )]
+    lane: Option<String>,
+    #[arg(long, conflicts_with = "lane", required_unless_present = "lane")]
+    lane_run_id: Option<i64>,
+}
+
+#[derive(Debug, Clone, clap::Args)]
+struct RecoverRunArgs {
+    branch_or_id: Option<String>,
+    #[arg(long, conflicts_with = "branch_or_id")]
+    nightly_run_id: Option<i64>,
+    #[arg(long, conflicts_with = "nightly_run_id")]
+    run_id: Option<i64>,
+}

Evidence

@@ -191,6 +223,46 @@ struct BranchLogsResponse {
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+struct NightlyDetailResponse {
+    nightly_run_id: i64,
+    ...
+    lanes: Vec<CiLane>,
+}
+
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+struct LaneMutationResponse {
+    status: String,
+    branch_id: Option<i64>,
+    nightly_run_id: Option<i64>,
+    lane_run_id: i64,
+    lane_status: String,
+}
+
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+struct RecoverRunResponse {
+    status: String,
+    ...
+    recovered_lane_count: usize,
+}
+
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+struct WakeCiResponse {
+    status: String,
+    message: String,
+}

Evidence

@@ -366,6 +438,53 @@ fn cmd_url(cli: &Cli, branch_or_id: Option<&str>) -> anyhow::Result<()> {
+fn cmd_fail_lane(cli: &Cli, args: &LaneActionArgs) -> anyhow::Result<()> {
+    let session = load_session(&cli.state_dir)?;
+    ...
+    execute_lane_action(&api, args, LaneActionKind::Fail)
+}
+
+fn cmd_requeue_lane(cli: &Cli, args: &LaneActionArgs) -> anyhow::Result<()> {
+    ...
+    execute_lane_action(&api, args, LaneActionKind::Requeue)
+}
+
+fn cmd_recover_run(cli: &Cli, args: &RecoverRunArgs) -> anyhow::Result<()> {
+    ...
+    let response = api.recover_branch_ci_run(resolved.branch_id, run_id)?;
+    println!("recovered branch #{} run #{} lanes={}", ...);
+}
+
+fn cmd_wake_ci(cli: &Cli) -> anyhow::Result<()> {
+    ...
+    let response = api.wake_ci()?;
+    println!("{}", response.message);
+}

Evidence

@@ -431,6 +565,112 @@ fn infer_current_branch() -> anyhow::Result<String> {
+fn execute_lane_action(
+    api: &ApiClient,
+    args: &LaneActionArgs,
+    action: LaneActionKind,
+) -> anyhow::Result<()> {
+    if let Some(nightly_run_id) = args.nightly_run_id {
+        let nightly = api.nightly_detail(nightly_run_id)?;
+        let lane = resolve_lane_selector(&nightly.lanes, ...);
+        ...
+    }
+    let resolved = resolve_branch_ref(api, args.branch_or_id.as_deref())?;
+    let branch = api.branch_detail(resolved.branch_id)?;
+    let lane = resolve_branch_lane(&branch, ...);
+    ...
+}

@@ +fn resolve_branch_lane<'a>(
+    branch: &'a BranchDetailResponse,
+    lane: Option<&str>,
+    lane_run_id: Option<i64>,
+) -> anyhow::Result<&'a CiLane> {
+    let selector = lane_selector(lane, lane_run_id)?;
+    for run in &branch.ci_runs {
+        if let Ok(found) = resolve_lane_selector(&run.lanes, ...) {
+            return Ok(found);
+        }
+    }
+    ...
+}

@@ +fn resolve_lane_selector<'a>(
+    lanes: &'a [CiLane],
+    lane: Option<&str>,
+    lane_run_id: Option<i64>,
+) -> anyhow::Result<&'a CiLane> {

Evidence

@@ -710,6 +950,15 @@ impl ApiClient {
+    fn nightly_detail(&self, nightly_run_id: i64) -> anyhow::Result<NightlyDetailResponse> {
+        self.send(Method::GET, &format!("/news/api/forge/nightly/{nightly_run_id}"), None::<&()>, true)
+    }

@@ -728,6 +977,89 @@ impl ApiClient {
+    fn fail_branch_ci_lane(&self, branch_id: i64, lane_run_id: i64) -> anyhow::Result<LaneMutationResponse> {
+        self.send(Method::POST, &format!("/news/branch/{branch_id}/ci/fail/{lane_run_id}"), Some(&serde_json::json!({})), true)
+    }
+    fn requeue_branch_ci_lane(...)
+    fn recover_branch_ci_run(...)
+    fn fail_nightly_lane(...)
+    fn requeue_nightly_lane(...)
+    fn recover_nightly_run(...)
+    fn wake_ci(&self) -> anyhow::Result<WakeCiResponse> {
+        self.send(Method::POST, "/news/api/forge/ci/wake", Some(&serde_json::json!({})), true)
+    }

Evidence

@@ -115,6 +115,8 @@ pub struct BranchCiLaneRecord {
+    pub last_heartbeat_at: Option<String>,
+    pub lease_expires_at: Option<String>,

@@ -174,6 +176,8 @@ pub struct NightlyLaneRecord {
+    pub last_heartbeat_at: Option<String>,
+    pub lease_expires_at: Option<String>,

@@ -2169,7 +2529,7 @@ fn list_branch_ci_run_lanes(
-            "SELECT id, lane_id, title, entrypoint, status, pikaci_run_id, pikaci_target_id, log_text, retry_count, rerun_of_lane_run_id, created_at, started_at, finished_at
+            "SELECT id, lane_id, title, entrypoint, status, pikaci_run_id, pikaci_target_id, log_text, retry_count, rerun_of_lane_run_id, created_at, started_at, finished_at, last_heartbeat_at, lease_expires_at

Evidence

@@ -1141,6 +1163,344 @@ impl Store {
+    pub fn fail_branch_ci_lane(
+        &self,
+        branch_id: i64,
+        lane_run_id: i64,
+        actor_npub: &str,
+    ) -> anyhow::Result<Option<()>> {
+        ...
+        if !matches!(row.status.as_str(), "queued" | "running") {
+            bail!("lane {} is already {}", row.lane_id, row.status);
+        }
+        ...
+        SET status = 'failed', log_text = ?1, finished_at = CURRENT_TIMESTAMP,
+            lease_expires_at = NULL, claim_token = ?2
+        ...
+    }

@@ +    pub fn requeue_branch_ci_lane(
+        ...
+        if !matches!(row.status.as_str(), "queued" | "running" | "failed") {
+            bail!("lane {} cannot be requeued from {}", ...);
+        }
+        ...
+        SET status = 'queued', log_text = NULL, ... claim_token = ?1
+        ...
+    }

@@ +    pub fn recover_branch_ci_run(
+        ...
+        UPDATE branch_ci_run_lanes
+        SET status = 'queued', ... claim_token = claim_token + 1
+        WHERE branch_ci_run_id = ?1 AND status IN ('queued', 'running', 'failed')
+        ...
+    }

Evidence

@@ -1002,10 +1028,34 @@ pub async fn serve(
+        .route("/news/branch/:branch_id/ci/fail/:lane_run_id", post(fail_branch_ci_lane_handler))
+        .route("/news/branch/:branch_id/ci/requeue/:lane_run_id", post(requeue_branch_ci_lane_handler))
+        .route("/news/branch/:branch_id/ci/recover/:run_id", post(recover_branch_ci_run_handler))
+        .route("/news/nightly/:nightly_run_id/fail/:lane_run_id", post(fail_nightly_lane_handler))
+        .route("/news/nightly/:nightly_run_id/requeue/:lane_run_id", post(requeue_nightly_lane_handler))
+        .route("/news/nightly/:nightly_run_id/recover", post(recover_nightly_run_handler))

@@ +        .route("/news/api/forge/nightly/:nightly_run_id", get(api_forge_nightly_detail_handler))
+        .route("/news/api/forge/ci/wake", post(wake_ci_handler))

Evidence

@@ -405,6 +405,9 @@ struct CiLaneView {
+    last_heartbeat_at: Option<String>,
+    lease_expires_at: Option<String>,
+    operator_hint: Option<String>,

@@ -1848,6 +1931,66 @@ fn lane_status_badge_class(status: &str) -> &'static str {
+fn parse_ci_timestamp(raw: &str) -> Option<DateTime<Utc>> {
+    DateTime::parse_from_rfc3339(raw)
+        .map(|value| value.with_timezone(&Utc))
+        .ok()
+        .or_else(|| {
+            NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M:%S")
+                ...
+        })
+}
+
+fn lane_operator_hint(
+    status: &str,
+    created_at: &str,
+    started_at: Option<&str>,
+    finished_at: Option<&str>,
+    last_heartbeat_at: Option<&str>,
+    lease_expires_at: Option<&str>,
+) -> Option<String> {

Evidence

@@ -439,6 +448,7 @@ struct BranchCiLiveTemplate {
+    branch_id: i64,

@@ -495,6 +505,22 @@ struct NightlyLiveTemplate {
+    nightly_run_id: i64,

@@ -1741,6 +1796,7 @@ fn render_branch_ci_live_html(
+        branch_id: record.branch_id,

@@ -1763,6 +1819,7 @@ fn render_nightly_live_html(
+        nightly_run_id: run.nightly_run_id,

Evidence

@@ branch_ci_live.html - action buttons for branch CI lanes

@@ nightly_live.html - action buttons for nightly lanes

@@ detail.html - operator hints and actions on branch detail page

@@ nightly.html - operator hints and actions on nightly detail page

Evidence

@@ -3367,4 +3742,181 @@ mod tests {
+    #[test]
+    fn manual_requeue_rejects_old_workers_for_branch_and_nightly_lanes() {
+        ...
+        store.requeue_branch_ci_lane(branch.branch_id, first_branch.lane_run_id)...
+        assert!(store.heartbeat_branch_ci_lane_run(first_branch.lane_run_id, first_branch.claim_token, 120)
+            .expect_err("stale branch heartbeat should fail")
+            .to_string().contains(CI_LANE_LEASE_LOST));
+        ...
+        assert!(second_branch.claim_token > first_branch.claim_token);
+    }

@@ +    #[test]
+    fn recover_run_requeues_only_nonterminal_lanes() {
+        ...
+        store.finish_branch_ci_lane_run(jobs[0].lane_run_id, jobs[0].claim_token, "success", "ok")...
+        store.finish_branch_ci_lane_run(jobs[1].lane_run_id, jobs[1].claim_token, "failed", "boom")...
+        let recovered = store.recover_branch_ci_run(branch.branch_id, jobs[2].suite_id)...
+        assert_eq!(recovered, 2);
+        assert_eq!(lanes[0].status, "success");
+        assert_eq!(lanes[1].status, "queued");
+        assert_eq!(lanes[2].status, "queued");
+    }

@@ ph/src/lib.rs tests
+    #[test]
+    fn fail_lane_resolves_branch_lane_name_against_latest_run() {
+    ...
+    #[test]
+    fn requeue_lane_resolves_nightly_lane_name() {
+    ...
+    #[test]
+    fn wake_ci_hits_scheduler_wake_endpoint() {