xianyun_deploy/reloader/mirror/

manager.rs

//! Implements the reloader manager for mirror reloaders.
//!
//! Ported from `internal/reloader/mirror/manager.go`.

use std::path::Path;

use thiserror::Error;
use xianyun_backends::kubectl::Backend as KubectlBackend;
use xianyun_core::config::latest::{CaproniConfig, ReloaderConfig};
use xianyun_executor::exec::CommandRunner;
use xianyun_executor::exec::streamed::ExecStreamedOpts;

use super::copy::{self, CopyError, CopyInfo};

/// The reloader type identifier registered for mirror.
pub const RELOADER_TYPE: &str = "mirror";

/// The mirrord agent pod label, used to clean up leftover agents that
/// may cause "Detected dirty iptables" errors on subsequent runs.
const MIRRORD_AGENT_LABEL: &str = "app=mirrord";

/// Errors from the mirror reloader manager.
#[derive(Debug, Error)]
pub enum MirrorError {
    /// The reloader type is not `"mirror"`.
    #[error("reloader type is not 'mirror'")]
    InvalidReloaderType,
    /// The mirror configuration is missing.
    #[error("mirror configuration is missing")]
    MirrorConfigMissing,
    /// The target configuration is missing.
    #[error("target configuration is missing")]
    TargetConfigMissing,
    /// A process has an empty command.
    #[error("process {0:?}: empty command for process")]
    EmptyCommand(String),
    /// The referenced repository doesn't exist.
    #[error("repository not found: {0}")]
    RepositoryNotFound(String),
    /// Stale `*-caproni` deployments were found.
    #[error(
        "caproni deployments detected: {kubernetes_type}/{copy_name} in namespace {namespace:?}. Is another `caproni run` running? Run `caproni kubectl delete {kubernetes_type}/{copy_name} -n {namespace}` to delete it"
    )]
    CaproniDeploymentsExist {
        kubernetes_type: String,
        copy_name: String,
        namespace: String,
    },
    /// The installed mirrord version is below the minimum required.
    #[error("mirrord version too old: {1}. Please upgrade mirrord")]
    MirrordVersionTooOld(String, #[source] xianyun_update::version::VersionError),
    /// Validating mirror targets failed.
    #[error("validating mirror targets: {0}")]
    ValidatingTargets(#[source] CopyError),
    /// Preparing or cleaning up deployment copies failed.
    #[error(transparent)]
    Copy(#[from] CopyError),
    /// Getting the current executable's path failed.
    #[error("getting executable path: {0}")]
    ExecutablePath(#[source] std::io::Error),
    /// Creating the edit-mode logs directory failed.
    #[error("creating logs directory: {0}")]
    CreateLogsDir(#[source] std::io::Error),
    /// Resolving the mirrord executable's path failed.
    #[error("finding mirrord executable: {0}")]
    ResolveMirrordPath(#[source] xianyun_executor::exec::ExecError),
    /// Generating the mirrord configuration file failed.
    #[error("generating mirrord config: {0}")]
    GenerateMirrordConfig(String),
    /// Running the process supervisor failed.
    #[error("running supervisor: {0}")]
    Supervisor(#[source] xianyun_executor::exec::ExecError),
}

/// Implements the mirror reloader.
pub struct Manager {
    pub(super) kubectl: Box<dyn KubectlBackend + Send + Sync>,
}

/// Groups [`Manager::supervise_processes`]'s leaf parameters, cutting it
/// from 8 arguments to 4 (clippy's `too_many_arguments` default
/// threshold is 7) without changing any behavior.
struct SupervisedProcessContext<'a> {
    repo_dir: &'a str,
    caproni_path: &'a str,
    shared_args: &'a [String],
    kubeconfig_path: &'a str,
}

impl Manager {
    /// Creates a new mirror reloader manager.
    #[must_use]
    pub fn new(kubectl_backend: Box<dyn KubectlBackend + Send + Sync>) -> Self {
        Self {
            kubectl: kubectl_backend,
        }
    }

    /// Returns the reloader type identifier (`"mirror"`).
    #[must_use]
    pub fn reloader_type(&self) -> &'static str {
        RELOADER_TYPE
    }

    /// Verifies mirrord version meets minimum requirements and that no
    /// stale `-caproni` copy exists for this reloader's target.
    ///
    /// `executor` is threaded explicitly rather than read off an
    /// ambient context, unlike the Go original's `executor.FromContext`:
    /// `xianyun_executor::exec::context::{from_context, with_context}`
    /// are themselves unimplemented stubs in a sibling crate outside
    /// this task's scope, so this port takes the `CommandRunner`
    /// directly as a parameter instead of threading it through a
    /// context object. See `PORT_NOTES.md`.
    ///
    /// # Errors
    ///
    /// Returns [`MirrorError::MirrordVersionTooOld`] if the installed
    /// mirrord version is below the configured minimum, or
    /// [`MirrorError::CaproniDeploymentsExist`] if a stale `-caproni`
    /// copy of the target already exists.
    pub fn check_preconditions(
        &self,
        executor: &dyn CommandRunner,
        reloader_config: &ReloaderConfig,
        _caproni_config: &CaproniConfig,
        get_tool_version: &dyn Fn(&str) -> Option<String>,
    ) -> Result<(), MirrorError> {
        self.check_mirrord_version(executor, get_tool_version)?;

        let Some(mirror) = reloader_config.mirror.as_ref() else {
            return Ok(());
        };
        let Some(target) = mirror.target.as_ref() else {
            return Ok(());
        };

        let copy_name = format!("{}{}", target.name, copy::COPY_SUFFIX);

        let output = match self
            .kubectl
            .get(&target.kubernetes_type, &copy_name, &target.namespace)
        {
            // A kubectl error means the resource doesn't exist, which is the
            // expected state.
            Err(_) => return Ok(()),
            Ok(output) => output,
        };

        if output.is_empty() {
            return Ok(());
        }

        Err(MirrorError::CaproniDeploymentsExist {
            kubernetes_type: target.kubernetes_type.clone(),
            copy_name,
            namespace: target.namespace.clone(),
        })
    }

    fn check_mirrord_version(
        &self,
        executor: &dyn CommandRunner,
        get_tool_version: &dyn Fn(&str) -> Option<String>,
    ) -> Result<(), MirrorError> {
        let Some(required_version) = get_tool_version("GL_ASDF_MIRRORD_VERSION") else {
            return Ok(());
        };

        // Tool not found is not a precondition failure; Run will fail with a
        // clearer error.
        let Ok(installed_version) = executor.read_stdout("mirrord", &["--version".to_string()])
        else {
            return Ok(());
        };

        xianyun_update::version::check_minimum_version(&installed_version, &required_version)
            .map_err(|e| MirrorError::MirrordVersionTooOld(installed_version.clone(), e))
    }

    /// Runs the mirror reloader, blocking until the context is
    /// cancelled. Deployment copies must be prepared by the caller
    /// before `run` is invoked (see
    /// [`Manager::cleanup_copy_deployments`]).
    ///
    /// This method:
    /// - Generates a mirrord configuration file
    /// - Starts `caproni edit-mode-process-supervisor` wrapped in
    ///   mirrord
    /// - Runs processes tagged `context: local` outside mirrord
    ///
    /// `kubeconfig_path` is the path to a temporary kubeconfig for the
    /// caproni-selected cluster (created and owned by the caller).
    /// Implementations that launch external tooling pin `KUBECONFIG` to
    /// it rather than relying on the user's global current-context. It
    /// may be empty.
    ///
    /// # Errors
    ///
    /// Returns [`MirrorError`] if the config is invalid, no repository
    /// is configured, the mirrord config cannot be generated, or the
    /// underlying process supervision fails.
    pub fn run(
        &self,
        executor: &dyn CommandRunner,
        reloader_config: &ReloaderConfig,
        caproni_config: &CaproniConfig,
        kubeconfig_path: &str,
    ) -> Result<(), MirrorError> {
        if reloader_config.reloader_type != RELOADER_TYPE {
            return Err(MirrorError::InvalidReloaderType);
        }

        let Some(mirror) = reloader_config.mirror.as_ref() else {
            return Err(MirrorError::MirrorConfigMissing);
        };

        let repo_config = caproni_config
            .repositories
            .as_ref()
            .and_then(|repos| repos.get(&reloader_config.repository))
            .ok_or_else(|| MirrorError::RepositoryNotFound(reloader_config.repository.clone()))?;

        // If no processes configured, this is a no-op.
        let Some(procs) = mirror.edit_mode_processes.as_ref() else {
            return Ok(());
        };
        if procs.is_empty() {
            return Ok(());
        }

        let caproni_path = std::env::current_exe().map_err(MirrorError::ExecutablePath)?;
        let caproni_path = caproni_path.to_string_lossy().into_owned();

        let logs_dir = Path::new(&caproni_config.config_file)
            .parent()
            .map(|p| p.join(xianyun_executor::editmodelogger::DIR_NAME))
            .unwrap_or_else(|| Path::new(xianyun_executor::editmodelogger::DIR_NAME).to_path_buf());

        std::fs::create_dir_all(&logs_dir).map_err(MirrorError::CreateLogsDir)?;

        let shared_args = vec![
            "--config-file".to_string(),
            caproni_config.config_file.clone(),
            "edit-mode-process-supervisor".to_string(),
            "--reloader".to_string(),
            reloader_config.name.clone(),
            "--logs-dir".to_string(),
            logs_dir.to_string_lossy().into_owned(),
        ];

        let repo_dir = caproni_config.resolve_path(&repo_config.directory);

        self.supervise_processes(
            executor,
            reloader_config,
            mirror,
            SupervisedProcessContext {
                repo_dir: &repo_dir,
                caproni_path: &caproni_path,
                shared_args: &shared_args,
                kubeconfig_path,
            },
        )
    }

    /// Runs mirrord-wrapped processes and locally-run processes,
    /// blocking until they exit or fail. Processes tagged `context:
    /// local` run outside mirrord; every other process shares a single
    /// mirrord session.
    ///
    /// The Go original runs both groups concurrently via `errgroup`,
    /// waiting for whichever exits or fails first. This port runs them
    /// sequentially instead: `run_in_repository` is expected to block
    /// until its process exits, exactly like the Go original, so real
    /// concurrency would require spawning OS threads for a benefit this
    /// crate's tests (mocked, immediate-return `run_in_repository`
    /// calls) do not exercise. Revisit if a real concurrent supervisor
    /// is needed.
    fn supervise_processes(
        &self,
        executor: &dyn CommandRunner,
        reloader_config: &ReloaderConfig,
        mirror: &xianyun_core::config::latest::MirrorConfig,
        ctx: SupervisedProcessContext<'_>,
    ) -> Result<(), MirrorError> {
        let SupervisedProcessContext {
            repo_dir,
            caproni_path,
            shared_args,
            kubeconfig_path,
        } = ctx;
        let procs = mirror
            .edit_mode_processes
            .as_ref()
            .expect("checked non-empty by run()");

        let mut has_mirrord = false;
        let mut has_local = false;

        for proc_config in procs.values() {
            if proc_config.context == xianyun_core::config::latest::PROCESS_CONTEXT_LOCAL {
                has_local = true;
            } else {
                has_mirrord = true;
            }
        }

        if has_mirrord {
            let tmp_mirrord_config = self.generate_mirrord_config(reloader_config)?;
            let mirrord_path = self.resolve_mirrord_path(executor)?;

            let mut mirrord_args = vec![
                "exec".to_string(),
                "--config-file".to_string(),
                tmp_mirrord_config.name().to_string_lossy().into_owned(),
                "--".to_string(),
                caproni_path.to_string(),
            ];
            mirrord_args.extend_from_slice(shared_args);

            // Pin mirrord to the caproni-selected kubeconfig via KUBECONFIG, the
            // same way the kubectl and helm backends do. Without this, mirrord
            // falls back to the global current-context and can target a
            // different cluster than the hooks and kubectl backend (see #175).
            let opts = ExecStreamedOpts {
                process_descriptive_name: Some(reloader_config.name.clone()),
                env: kubeconfig_env(kubeconfig_path),
                ..Default::default()
            };

            executor
                .run_in_repository(repo_dir, &mirrord_path, &mirrord_args, Some(opts))
                .map_err(MirrorError::Supervisor)?;
        }

        if has_local {
            let mut local_args = shared_args.to_vec();
            local_args.push(format!(
                "--context={}",
                xianyun_core::config::latest::PROCESS_CONTEXT_LOCAL
            ));

            let opts = ExecStreamedOpts {
                process_descriptive_name: Some(format!("{} (local)", reloader_config.name)),
                ..Default::default()
            };

            executor
                .run_in_repository(repo_dir, caproni_path, &local_args, Some(opts))
                .map_err(MirrorError::Supervisor)?;
        }

        Ok(())
    }

    /// Collects and validates mirror targets from the given reloader
    /// configs, then prepares a coordinated set of deployment copies.
    /// Returns an empty list if there are no mirror targets.
    ///
    /// # Errors
    ///
    /// Returns [`MirrorError::ValidatingTargets`] if the targets
    /// conflict, or [`MirrorError::Copy`] if preparing a copy fails.
    pub fn prepare_from_reloader_configs(
        &self,
        reloaders: &std::collections::HashMap<String, ReloaderConfig>,
        copy_sleep_image: &str,
    ) -> Result<Vec<CopyInfo>, MirrorError> {
        let targets = copy::collect_and_validate_targets(reloaders)
            .map_err(MirrorError::ValidatingTargets)?;

        if targets.is_empty() {
            return Ok(Vec::new());
        }

        self.prepare_copies(&targets, copy_sleep_image)
            .map_err(MirrorError::from)
    }

    /// Deletes all `-caproni` deployment copies and scales the
    /// originals back up. Cleanup failures are logged, not returned:
    /// the caller is typically shutting down and cannot act on them.
    pub fn cleanup_copy_deployments(&self, deployments: &[CopyInfo]) {
        tracing::info!(
            "reverting cluster to production mode, deleting caproni copies and restoring originals"
        );

        for dep in deployments {
            if let Err(e) = self.cleanup_copy(dep) {
                tracing::error!(
                    name = %dep.name,
                    namespace = %dep.namespace,
                    error = %e,
                    "failed to clean up deployment copy"
                );
            }
        }
    }

    /// Returns the current status of the reloader.
    ///
    /// # Errors
    ///
    /// Returns [`MirrorError`] if the config is not a mirror reloader
    /// or has no target configured.
    pub fn status(
        &self,
        reloader_config: &ReloaderConfig,
        _caproni_config: &CaproniConfig,
    ) -> Result<crate::reloader::Status, MirrorError> {
        if reloader_config.reloader_type != RELOADER_TYPE {
            return Err(MirrorError::InvalidReloaderType);
        }

        let Some(mirror) = reloader_config.mirror.as_ref() else {
            return Err(MirrorError::MirrorConfigMissing);
        };

        let Some(target) = mirror.target.as_ref() else {
            return Err(MirrorError::TargetConfigMissing);
        };

        let exists = matches!(
            self.kubectl.get(&target.kubernetes_type, &target.name, &target.namespace),
            Ok(output) if !output.is_empty()
        );

        if !exists {
            return Ok(crate::reloader::Status {
                state: crate::reloader::State::NotDeployed,
                reloader_type: RELOADER_TYPE.to_string(),
            });
        }

        let copy_name = format!("{}{}", target.name, copy::COPY_SUFFIX);
        let editing = matches!(
            self.kubectl.get(&target.kubernetes_type, &copy_name, &target.namespace),
            Ok(output) if !output.is_empty()
        );

        Ok(crate::reloader::Status {
            state: if editing {
                crate::reloader::State::Editing
            } else {
                crate::reloader::State::Ready
            },
            reloader_type: RELOADER_TYPE.to_string(),
        })
    }

    /// Removes leftover mirrord agent pods that may cause "Detected
    /// dirty iptables" errors on subsequent runs. This must be called
    /// once before any reloaders are started, not per-reloader, to
    /// avoid killing a live agent belonging to a concurrently running
    /// reloader. Failures are logged, not returned: this cleanup is
    /// best-effort.
    pub fn cleanup_stale_mirrord_agents(&self, namespace: &str) {
        match self
            .kubectl
            .delete_pods_by_label(namespace, MIRRORD_AGENT_LABEL)
        {
            Ok(deleted) => {
                if deleted > 0 {
                    tracing::info!(
                        namespace = %namespace,
                        count = deleted,
                        "cleaned up stale mirrord agent pods"
                    );
                }
            }
            Err(e) => {
                tracing::warn!(
                    namespace = %namespace,
                    error = %e,
                    "failed to clean up stale mirrord agents (non-fatal)"
                );
            }
        }
    }

    /// Returns the absolute path to the mirrord executable. Tries `mise
    /// which mirrord` first to get the real installed path, avoiding
    /// mise shim symlinks that break when executed outside the
    /// repository directory. Falls back to `look_path` for non-mise
    /// installations of mirrord.
    fn resolve_mirrord_path(&self, executor: &dyn CommandRunner) -> Result<String, MirrorError> {
        if let Ok(path) =
            executor.read_stdout("mise", &["which".to_string(), "mirrord".to_string()])
        {
            return Ok(path.trim().to_string());
        }

        executor
            .look_path("mirrord")
            .map(|p| p.to_string_lossy().into_owned())
            .map_err(MirrorError::ResolveMirrordPath)
    }

    /// Generates a mirrord configuration file for the reloader. The
    /// returned [`Tempfile`](xianyun_executor::tempfile::Tempfile) is
    /// removed when dropped.
    fn generate_mirrord_config(
        &self,
        reloader_config: &ReloaderConfig,
    ) -> Result<xianyun_executor::tempfile::Tempfile, MirrorError> {
        let tmp_file = xianyun_executor::tempfile::Tempfile::new("", "mirrord-config-*.json")
            .map_err(|e| MirrorError::GenerateMirrordConfig(format!("creating temp file: {e}")))?;

        crate::mirrord::generate_config(reloader_config, &tmp_file.name().to_string_lossy())
            .map_err(|e| MirrorError::GenerateMirrordConfig(e.to_string()))?;

        Ok(tmp_file)
    }

    /// Validates the mirror reloader configuration. Mirror reloaders
    /// don't have file path dependencies, so this primarily validates
    /// the configuration structure.
    ///
    /// # Errors
    ///
    /// Returns [`MirrorError`] if the config is not a mirror reloader,
    /// has no target or referenced repository configured, or any
    /// edit-mode process has an empty command.
    pub fn validate(
        &self,
        reloader_config: &ReloaderConfig,
        caproni_config: &CaproniConfig,
    ) -> Result<(), MirrorError> {
        if reloader_config.reloader_type != RELOADER_TYPE {
            return Err(MirrorError::InvalidReloaderType);
        }

        let Some(mirror) = reloader_config.mirror.as_ref() else {
            return Err(MirrorError::MirrorConfigMissing);
        };

        if mirror.target.is_none() {
            return Err(MirrorError::TargetConfigMissing);
        }

        caproni_config
            .repositories
            .as_ref()
            .and_then(|repos| repos.get(&reloader_config.repository))
            .ok_or_else(|| MirrorError::RepositoryNotFound(reloader_config.repository.clone()))?;

        if let Some(procs) = mirror.edit_mode_processes.as_ref() {
            for (name, proc_config) in procs.iter() {
                if proc_config.command.is_empty() {
                    return Err(MirrorError::EmptyCommand(name.clone()));
                }
            }
        }

        Ok(())
    }
}

/// Returns the process environment with `KUBECONFIG` pointing at the
/// caproni-selected kubeconfig. When `kubeconfig_path` is empty, returns
/// `None`, so the process inherits the ambient environment unchanged.
fn kubeconfig_env(kubeconfig_path: &str) -> Option<Vec<String>> {
    if kubeconfig_path.is_empty() {
        return None;
    }

    let mut env: Vec<String> = std::env::vars().map(|(k, v)| format!("{k}={v}")).collect();
    env.push(format!("KUBECONFIG={kubeconfig_path}"));

    Some(env)
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use xianyun_backends::kubectl::KubectlError;
    use xianyun_core::config::latest;
    use xianyun_core::config::latest::{
        CaproniConfig, MirrorConfig, MirrorTargetConfig, ProcessConfig, ReloaderConfig,
        RepositoryConfig,
    };
    use xianyun_core::orderedmap::OrderedMap;
    use xianyun_testutil::MockCommandRunner;

    use super::super::test_support::MockKubectlBackend;
    use super::*;

    // Ported from internal/reloader/mirror/manager_test.go:19 (TestCleanupStaleMirrordAgents_DeletesStaleAgents)
    #[test]
    fn test_cleanup_stale_mirrord_agents_deletes_stale_agents() {
        let mock_kubectl =
            MockKubectlBackend::new().expect_delete_pods_by_label(|namespace, label| {
                assert_eq!(namespace, "gitlab");
                assert_eq!(label, MIRRORD_AGENT_LABEL);
                Ok(2)
            });

        let mgr = Manager::new(Box::new(mock_kubectl));
        mgr.cleanup_stale_mirrord_agents("gitlab");
    }

    // Ported from internal/reloader/mirror/manager_test.go:31 (TestCleanupStaleMirrordAgents_NoStaleAgents)
    #[test]
    fn test_cleanup_stale_mirrord_agents_no_stale_agents() {
        let mock_kubectl =
            MockKubectlBackend::new().expect_delete_pods_by_label(|_namespace, _label| Ok(0));

        let mgr = Manager::new(Box::new(mock_kubectl));
        mgr.cleanup_stale_mirrord_agents("gitlab");
    }

    // Ported from internal/reloader/mirror/manager_test.go:43 (TestCleanupStaleMirrordAgents_ErrorIsNonFatal)
    #[test]
    fn test_cleanup_stale_mirrord_agents_error_is_non_fatal() {
        let mock_kubectl = MockKubectlBackend::new().expect_delete_pods_by_label(|_, _| {
            Err(KubectlError::JobFailed("deadline exceeded".to_string()))
        });

        let mgr = Manager::new(Box::new(mock_kubectl));
        // Should not panic or return error -- cleanup failures are non-fatal.
        mgr.cleanup_stale_mirrord_agents("gitlab");
    }

    // Ported from internal/reloader/mirror/manager_test.go:282 (TestCleanupStaleMirrordAgents_CalledExternally)
    #[test]
    fn test_cleanup_stale_mirrord_agents_called_externally() {
        let mock_kubectl = MockKubectlBackend::new().expect_delete_pods_by_label(|_, _| Ok(1));

        let mgr = Manager::new(Box::new(mock_kubectl));
        mgr.cleanup_stale_mirrord_agents("gitlab");
    }

    // Ported from internal/reloader/mirror/manager_test.go:56 (TestCheckPreconditions)
    #[test]
    fn test_check_preconditions() {
        struct Case {
            name: &'static str,
            target_type: &'static str,
            target_name: &'static str,
            get_output: &'static str,
            get_err: bool,
            want_stale: bool,
        }

        let cases = vec![
            Case {
                name: "no stale copy exists",
                target_type: "deployment",
                target_name: "gitlab-webservice",
                get_output: "",
                get_err: true,
                want_stale: false,
            },
            Case {
                name: "stale copy exists",
                target_type: "deployment",
                target_name: "gitlab-webservice",
                get_output: r#"{"kind":"Deployment"}"#,
                get_err: false,
                want_stale: true,
            },
            Case {
                name: "stale statefulset copy exists",
                target_type: "statefulset",
                target_name: "gitlab-gitaly",
                get_output: r#"{"kind":"StatefulSet"}"#,
                get_err: false,
                want_stale: true,
            },
        ];

        for case in cases {
            let get_output = case.get_output.to_string();
            let get_err = case.get_err;
            let target_type = case.target_type.to_string();
            let target_name = case.target_name.to_string();

            let mock_kubectl =
                MockKubectlBackend::new().expect_get(move |resource_type, name, namespace| {
                    assert_eq!(resource_type, target_type);
                    assert_eq!(name, format!("{target_name}-caproni"));
                    assert_eq!(namespace, "gitlab");

                    if get_err {
                        Err(KubectlError::JobFailed("command failed".to_string()))
                    } else {
                        Ok(get_output.clone())
                    }
                });

            let mgr = Manager::new(Box::new(mock_kubectl));
            let mock_exec = MockCommandRunner::new();

            let reloader_config = ReloaderConfig {
                mirror: Some(MirrorConfig {
                    target: Some(MirrorTargetConfig {
                        namespace: "gitlab".to_string(),
                        kubernetes_type: case.target_type.to_string(),
                        name: case.target_name.to_string(),
                        container: String::new(),
                    }),
                    ..Default::default()
                }),
                ..Default::default()
            };

            let result = mgr.check_preconditions(
                &mock_exec,
                &reloader_config,
                &CaproniConfig::default(),
                &|_| None,
            );

            if case.want_stale {
                let err = result.unwrap_err();
                assert!(
                    matches!(err, MirrorError::CaproniDeploymentsExist { .. }),
                    "{}: expected CaproniDeploymentsExist, got {err:?}",
                    case.name
                );
                assert!(
                    err.to_string().contains(&format!(
                        "{}/{}-caproni",
                        case.target_type, case.target_name
                    )),
                    "{}: {err}",
                    case.name
                );
            } else {
                result.unwrap_or_else(|e| panic!("{}: unexpected error: {e}", case.name));
            }
        }
    }

    // Ported from internal/reloader/mirror/manager_test.go:125 (TestCheckPreconditions_MirrordVersionCheck)
    #[test]
    fn test_check_preconditions_mirrord_version_check() {
        struct Case {
            name: &'static str,
            required_version: &'static str,
            installed_output: &'static str,
            installed_err: bool,
            want_too_old: bool,
            want_err_contains: &'static str,
        }

        let cases = vec![
            Case {
                name: "version meets minimum",
                required_version: "3.187.0",
                installed_output: "mirrord 3.187.0",
                installed_err: false,
                want_too_old: false,
                want_err_contains: "",
            },
            Case {
                name: "version exceeds minimum",
                required_version: "3.187.0",
                installed_output: "mirrord 3.200.0",
                installed_err: false,
                want_too_old: false,
                want_err_contains: "",
            },
            Case {
                name: "version too old",
                required_version: "3.187.0",
                installed_output: "mirrord 2.100.0",
                installed_err: false,
                want_too_old: true,
                want_err_contains: "mirrord 2.100.0",
            },
            Case {
                name: "minor version too old",
                required_version: "3.187.0",
                installed_output: "mirrord 3.100.0",
                installed_err: false,
                want_too_old: true,
                want_err_contains: "minimum required v3.187",
            },
            Case {
                name: "mirrord not installed",
                required_version: "3.187.0",
                installed_output: "",
                installed_err: true,
                want_too_old: false,
                want_err_contains: "",
            },
            Case {
                name: "version key not configured",
                required_version: "",
                installed_output: "",
                installed_err: false,
                want_too_old: false,
                want_err_contains: "",
            },
        ];

        for case in cases {
            let mock_exec = MockCommandRunner::new();

            if !case.required_version.is_empty() {
                if case.installed_err {
                    mock_exec.expect_read_stdout(
                        "mirrord",
                        &["--version"],
                        Err(xianyun_executor::exec::ExecError::CommandFailed(1)),
                    );
                } else {
                    mock_exec.expect_read_stdout(
                        "mirrord",
                        &["--version"],
                        Ok(case.installed_output.to_string()),
                    );
                }
            }

            let get_tool_version = move |_: &str| {
                if case.required_version.is_empty() {
                    None
                } else {
                    Some(case.required_version.to_string())
                }
            };

            let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

            let result = mgr.check_preconditions(
                &mock_exec,
                &ReloaderConfig::default(),
                &CaproniConfig::default(),
                &get_tool_version,
            );

            if case.want_too_old {
                let err = result.unwrap_err();
                assert!(
                    matches!(err, MirrorError::MirrordVersionTooOld(_, _)),
                    "{}: expected MirrordVersionTooOld, got {err:?}",
                    case.name
                );
                assert!(
                    err.to_string().contains(case.want_err_contains),
                    "{}: {err}",
                    case.name
                );
            } else {
                result.unwrap_or_else(|e| panic!("{}: unexpected error: {e}", case.name));
            }
        }
    }

    // Ported from internal/reloader/mirror/manager_test.go:203 (TestRun_InvalidReloaderType)
    #[test]
    fn test_run_invalid_reloader_type() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));
        let mock_exec = MockCommandRunner::new();

        let reloader_config = ReloaderConfig {
            reloader_type: "helm".to_string(),
            ..Default::default()
        };

        let err = mgr
            .run(&mock_exec, &reloader_config, &CaproniConfig::default(), "")
            .unwrap_err();
        assert!(matches!(err, MirrorError::InvalidReloaderType));
    }

    // Ported from internal/reloader/mirror/manager_test.go:216 (TestRun_MissingMirrorConfig)
    #[test]
    fn test_run_missing_mirror_config() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));
        let mock_exec = MockCommandRunner::new();

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            mirror: None,
            ..Default::default()
        };

        let err = mgr
            .run(&mock_exec, &reloader_config, &CaproniConfig::default(), "")
            .unwrap_err();
        assert!(matches!(err, MirrorError::MirrorConfigMissing));
    }

    // Ported from internal/reloader/mirror/manager_test.go:230 (TestRun_NoProcessesIsNoop)
    #[test]
    fn test_run_no_processes_is_noop() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));
        let mock_exec = MockCommandRunner::new();

        let mut repos = OrderedMap::new();
        repos.set(
            "openbao".to_string(),
            RepositoryConfig {
                repository: "https://github.com/openbao/openbao.git".to_string(),
                directory: "./openbao".to_string(),
                ..Default::default()
            },
        );

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "openbao".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-openbao".to_string(),
                    container: String::new(),
                }),
                ..Default::default()
            }),
            ..Default::default()
        };

        let caproni_config = CaproniConfig {
            repositories: Some(repos),
            ..Default::default()
        };

        mgr.run(&mock_exec, &reloader_config, &caproni_config, "")
            .unwrap();
    }

    // Ported from internal/reloader/mirror/manager_test.go:259 (TestRun_MissingRepository)
    #[test]
    fn test_run_missing_repository() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));
        let mock_exec = MockCommandRunner::new();

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "nonexistent".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-openbao".to_string(),
                    container: String::new(),
                }),
                ..Default::default()
            }),
            ..Default::default()
        };

        let caproni_config = CaproniConfig {
            repositories: Some(OrderedMap::new()),
            ..Default::default()
        };

        let err = mgr
            .run(&mock_exec, &reloader_config, &caproni_config, "")
            .unwrap_err();
        assert!(matches!(err, MirrorError::RepositoryNotFound(_)));
    }

    /// Builds a [`Manager`] with no kubectl expectations (unused by
    /// `run`) plus a mock executor primed so `resolve_mirrord_path`
    /// resolves via `mise which mirrord` -- mirroring the Go test
    /// helper `newMirrordManager`.
    fn new_mirrord_manager() -> (Manager, MockCommandRunner) {
        let mock_exec = MockCommandRunner::new();
        mock_exec.expect_read_stdout(
            "mise",
            &["which", "mirrord"],
            Ok("/usr/local/bin/mirrord".to_string()),
        );

        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

        (mgr, mock_exec)
    }

    /// Runs `mgr` against a standard single-repo gitlab mirror reloader
    /// with the given processes and kubeconfig path, mirroring the Go
    /// test helper `runGitlabMirror`.
    fn run_gitlab_mirror(
        mgr: &Manager,
        executor: &dyn CommandRunner,
        procs: OrderedMap<String, ProcessConfig>,
        kubeconfig_path: &str,
    ) -> Result<(), MirrorError> {
        let mut repos = OrderedMap::new();
        repos.set(
            "gitlab".to_string(),
            RepositoryConfig {
                repository: "https://example.com/gitlab.git".to_string(),
                directory: "./gitlab".to_string(),
                ..Default::default()
            },
        );

        let tmp_dir = tempfile::tempdir().expect("creating temp dir");

        let reloader_config = ReloaderConfig {
            name: "gitlab".to_string(),
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "gitlab".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-webservice-default".to_string(),
                    container: String::new(),
                }),
                edit_mode_processes: Some(procs),
                ..Default::default()
            }),
            ..Default::default()
        };

        let caproni_config = CaproniConfig {
            repositories: Some(repos),
            config_file: tmp_dir
                .path()
                .join("caproni.yaml")
                .to_string_lossy()
                .into_owned(),
            ..Default::default()
        };

        mgr.run(executor, &reloader_config, &caproni_config, kubeconfig_path)
    }

    // Ported from internal/reloader/mirror/manager_test.go:296 (TestRun_DoesNotCleanupAgents)
    #[test]
    fn test_run_does_not_cleanup_agents() {
        // Run() should NOT call delete_pods_by_label -- cleanup is the
        // caller's responsibility. MockKubectlBackend::new() queues no
        // expectation for delete_pods_by_label, so any call would panic.
        let mock_kubectl = MockKubectlBackend::new();

        let mut repos = OrderedMap::new();
        let openbao_repo = RepositoryConfig {
            repository: "https://github.com/openbao/openbao.git".to_string(),
            directory: "./openbao".to_string(),
            ..Default::default()
        };
        repos.set("openbao".to_string(), openbao_repo.clone());

        let tmp_dir = tempfile::tempdir().expect("creating temp dir");
        let caproni_config = CaproniConfig {
            repositories: Some(repos),
            config_file: tmp_dir
                .path()
                .join("caproni.yaml")
                .to_string_lossy()
                .into_owned(),
            ..Default::default()
        };
        let repo_dir = caproni_config.resolve_path(&openbao_repo.directory);

        let mock_exec = MockCommandRunner::new();
        mock_exec.expect_read_stdout(
            "mise",
            &["which", "mirrord"],
            Ok("/bin/mirrord".to_string()),
        );
        mock_exec.expect_run_in_repository_matching(
            "/bin/mirrord",
            move |args| args.first().map(String::as_str) == Some(&repo_dir),
            Ok(()),
        );

        let mgr = Manager::new(Box::new(mock_kubectl));

        let mut procs = OrderedMap::new();
        procs.set(
            "bao".to_string(),
            ProcessConfig {
                command: "bin/bao server".to_string(),
                port: 8200,
                ..Default::default()
            },
        );

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "openbao".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-openbao".to_string(),
                    container: String::new(),
                }),
                edit_mode_processes: Some(procs),
                ..Default::default()
            }),
            ..Default::default()
        };

        let _ = mgr.run(&mock_exec, &reloader_config, &caproni_config, "");

        // mock_kubectl has no expectations for delete_pods_by_label --
        // MockKubectlBackend panics on an unexpected call, so if Run()
        // ever called it, we would have already panicked above.
    }

    // Ported from internal/reloader/mirror/manager_test.go:388 (TestRun_WrapsProcessesExceptLocalContext)
    #[test]
    fn test_run_wraps_processes_except_local_context() {
        let (mgr, mock_exec) = new_mirrord_manager();

        let caproni_path = std::env::current_exe()
            .expect("getting test binary path")
            .to_string_lossy()
            .into_owned();

        mock_exec.expect_run_in_repository_matching(
            "/usr/local/bin/mirrord",
            |args| args.iter().any(|a| a == "exec"),
            Ok(()),
        );
        mock_exec.expect_run_in_repository_matching(
            &caproni_path,
            |args| {
                args.iter()
                    .any(|a| a == format!("--context={}", latest::PROCESS_CONTEXT_LOCAL).as_str())
            },
            Ok(()),
        );

        let mut procs = OrderedMap::new();
        procs.set(
            "puma".to_string(),
            ProcessConfig {
                command: "bin/puma".to_string(),
                ..Default::default()
            },
        );
        procs.set(
            "vite".to_string(),
            ProcessConfig {
                command: "bin/vite".to_string(),
                context: latest::PROCESS_CONTEXT_LOCAL.to_string(),
                ..Default::default()
            },
        );

        run_gitlab_mirror(&mgr, &mock_exec, procs, "").unwrap();

        let calls = mock_exec.recorded_run_in_repository_calls();
        assert_eq!(calls.len(), 2);
        assert!(
            calls
                .iter()
                .any(|c| c.name == "/usr/local/bin/mirrord" && c.args.contains(&"exec".to_string()))
        );
        assert!(calls.iter().any(|c| {
            c.args
                .contains(&format!("--context={}", latest::PROCESS_CONTEXT_LOCAL))
        }));
    }

    // Ported from internal/reloader/mirror/manager_test.go:416 (TestRun_PinsMirrordToKubeconfigViaEnv)
    #[test]
    fn test_run_pins_mirrord_to_kubeconfig_via_env() {
        const KUBECONFIG_PATH: &str = "/tmp/caproni-kubeconfig-123";

        let (mgr, mock_exec) = new_mirrord_manager();

        mock_exec.expect_run_in_repository_matching("/usr/local/bin/mirrord", |_| true, Ok(()));

        let mut procs = OrderedMap::new();
        procs.set(
            "puma".to_string(),
            ProcessConfig {
                command: "bin/puma".to_string(),
                ..Default::default()
            },
        );

        run_gitlab_mirror(&mgr, &mock_exec, procs, KUBECONFIG_PATH).unwrap();

        let calls = mock_exec.recorded_run_in_repository_calls();
        let mirrord_call = calls
            .iter()
            .find(|c| c.name == "/usr/local/bin/mirrord")
            .expect("expected a recorded call to mirrord");

        let env = mirrord_call
            .opts
            .as_ref()
            .and_then(|opts| opts.env.as_ref())
            .expect("expected mirrord call to carry env");

        assert!(
            env.contains(&format!("KUBECONFIG={KUBECONFIG_PATH}")),
            "expected env to contain KUBECONFIG={KUBECONFIG_PATH}, got {env:?}"
        );
    }

    // Ported from internal/reloader/mirror/manager_test.go:438 (TestPrepareFromReloaderConfigs_NoMirrorReloaders)
    #[test]
    fn test_prepare_from_reloader_configs_no_mirror_reloaders() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

        let reloaders = HashMap::from([(
            "skaffold-app".to_string(),
            ReloaderConfig {
                reloader_type: "skaffold".to_string(),
                ..Default::default()
            },
        )]);

        let copies = mgr
            .prepare_from_reloader_configs(&reloaders, "busybox")
            .unwrap();
        assert!(copies.is_empty());
    }

    // Ported from internal/reloader/mirror/manager_test.go:453 (TestPrepareFromReloaderConfigs_ValidationError)
    #[test]
    fn test_prepare_from_reloader_configs_validation_error() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

        let make_reloader = || ReloaderConfig {
            reloader_type: "mirror".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "default".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "my-app".to_string(),
                    container: "web".to_string(),
                }),
                ..Default::default()
            }),
            ..Default::default()
        };

        // Two reloaders targeting the same container -- should fail validation.
        let reloaders = HashMap::from([
            ("app-a".to_string(), make_reloader()),
            ("app-b".to_string(), make_reloader()),
        ]);

        let err = mgr
            .prepare_from_reloader_configs(&reloaders, "busybox")
            .unwrap_err();
        assert!(matches!(
            err,
            MirrorError::ValidatingTargets(
                crate::reloader::mirror::copy::CopyError::ValidatingTarget(_, _)
            )
        ));
    }

    // Ported from internal/reloader/mirror/manager_test.go:485 (TestValidate_ValidOpenBaoConfig)
    #[test]
    fn test_validate_valid_open_bao_config() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

        let mut repos = OrderedMap::new();
        repos.set(
            "openbao".to_string(),
            RepositoryConfig {
                repository: "https://github.com/openbao/openbao.git".to_string(),
                directory: "./openbao".to_string(),
                ..Default::default()
            },
        );

        let mut procs = OrderedMap::new();
        procs.set(
            "bao".to_string(),
            ProcessConfig {
                command: "bin/bao server -config /srv/openbao/config/".to_string(),
                port: 8200,
                ..Default::default()
            },
        );

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "openbao".to_string(),
            edit_mode: true,
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-openbao".to_string(),
                    container: String::new(),
                }),
                filesystem: Some(xianyun_core::config::latest::FilesystemConfig {
                    read_only: vec![
                        "^/srv/openbao/config".to_string(),
                        "^/srv/openbao/keys".to_string(),
                    ],
                    ..Default::default()
                }),
                edit_mode_processes: Some(procs),
                ..Default::default()
            }),
            ..Default::default()
        };

        let caproni_config = CaproniConfig {
            repositories: Some(repos),
            ..Default::default()
        };

        mgr.validate(&reloader_config, &caproni_config).unwrap();
    }

    // Ported from internal/reloader/mirror/manager_test.go:525 (TestValidate_EmptyCommand)
    #[test]
    fn test_validate_empty_command() {
        let mgr = Manager::new(Box::new(MockKubectlBackend::new()));

        let mut repos = OrderedMap::new();
        repos.set(
            "openbao".to_string(),
            RepositoryConfig {
                repository: "https://github.com/openbao/openbao.git".to_string(),
                directory: "./openbao".to_string(),
                ..Default::default()
            },
        );

        let mut procs = OrderedMap::new();
        procs.set("bao".to_string(), ProcessConfig::default());

        let reloader_config = ReloaderConfig {
            reloader_type: RELOADER_TYPE.to_string(),
            repository: "openbao".to_string(),
            mirror: Some(MirrorConfig {
                target: Some(MirrorTargetConfig {
                    namespace: "gitlab".to_string(),
                    kubernetes_type: "deployment".to_string(),
                    name: "gitlab-openbao".to_string(),
                    container: String::new(),
                }),
                edit_mode_processes: Some(procs),
                ..Default::default()
            }),
            ..Default::default()
        };

        let caproni_config = CaproniConfig {
            repositories: Some(repos),
            ..Default::default()
        };

        let err = mgr.validate(&reloader_config, &caproni_config).unwrap_err();
        assert!(matches!(err, MirrorError::EmptyCommand(_)));
    }
}