From cc9eb8e756e84cd3fd59baf5b80efab0ffd5757d Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Mon, 6 May 2024 12:28:40 -0700 Subject: tools/unitctl: enable passing IP addresses to the 'instances new' command * use path seperator constant from rust std package * pass a ControlSocket into deploy_new_container instead of a string * parse and validate a ControlSocket from argument to instances new * conditionally mount control socket only if its a unix socket * use create_image in a way that actually pulls nonpresent images * possibly override container command if TCP socket passed in * handle more weird error cases * add a ton of validation cases in the CLI command handler * add a nice little progress bar :) Signed-off-by: Ava Hahn --- tools/unitctl/Cargo.lock | 28 +++- tools/unitctl/unit-client-rs/Cargo.toml | 1 + tools/unitctl/unit-client-rs/src/unit_client.rs | 4 +- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 177 +++++++++++++-------- tools/unitctl/unit-client-rs/src/unitd_instance.rs | 1 + tools/unitctl/unit-client-rs/src/unitd_process.rs | 1 + tools/unitctl/unitctl/src/cmd/instances.rs | 104 +++++++++++- 7 files changed, 235 insertions(+), 81 deletions(-) (limited to 'tools') diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 2acbfb9a..16241296 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -390,6 +390,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.2" @@ -416,12 +425,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -1226,6 +1232,17 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "pbr" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5827dfa0d69b6c92493d6c38e633bbaa5937c153d0d7c28bf12313f8c6d514" +dependencies = [ + "crossbeam-channel", + "libc", + "winapi", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2001,6 +2018,7 @@ dependencies = [ "hyper 0.14.27", "hyper-tls", "hyperlocal", + "pbr", "rand", "regex", "rustls", diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index 3e48ee23..b7b8b496 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -29,6 +29,7 @@ unit-openapi = { path = "../unit-openapi" } rustls = "0.23.5" bollard = "0.16.1" regex = "1.10.4" +pbr = "1.1.1" [dev-dependencies] rand = "0.8.5" diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index f76004cd..b8c73ec0 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -250,7 +250,7 @@ impl UnitClient { Err(Box::new(UnitClientError::new( hyper_error, self.control_socket.to_string(), - "".to_string(), + "/listeners".to_string(), ))) } else { Err(Box::new(UnitClientError::OpenAPIError { source: err })) @@ -268,7 +268,7 @@ impl UnitClient { Err(Box::new(UnitClientError::new( hyper_error, self.control_socket.to_string(), - "".to_string(), + "/status".to_string(), ))) } else { Err(Box::new(UnitClientError::OpenAPIError { source: err })) diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 4c86c870..b9199e40 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -1,19 +1,29 @@ use std::collections::HashMap; use std::fs::read_to_string; -use std::path::PathBuf; +use std::path::{PathBuf, MAIN_SEPARATOR}; +use std::io::stderr; use crate::futures::StreamExt; use crate::unit_client::UnitClientError; use crate::unitd_process::UnitdProcess; +use crate::control_socket_address::ControlSocket; + use bollard::container::{Config, ListContainersOptions, StartContainerOptions}; use bollard::image::CreateImageOptions; -use bollard::models::{ContainerCreateResponse, HostConfig, Mount, MountTypeEnum}; +use bollard::models::{ + ContainerCreateResponse, HostConfig, Mount, + MountTypeEnum, ContainerSummary, +}; use bollard::secret::ContainerInspectResponse; -use bollard::{models::ContainerSummary, Docker}; +use bollard::Docker; + use regex::Regex; + use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; +use pbr::ProgressBar; + #[derive(Clone, Debug)] pub struct UnitdContainer { pub container_id: Option, @@ -121,6 +131,7 @@ impl Serialize for UnitdContainer { where S: Serializer, { + // 5 = fields to serialize let mut state = serializer.serialize_map(Some(5))?; state.serialize_entry("container_id", &self.container_id)?; state.serialize_entry("container_image", &self.container_image)?; @@ -143,18 +154,23 @@ impl UnitdContainer { // cant do this functionally because of the async call let mut mapped = vec![]; for ctr in summary { - if ctr.clone().image.or(Some(String::new())).unwrap().contains("unit") { - let mut c = UnitdContainer::from(&ctr); - if let Some(names) = ctr.names { - if names.len() > 0 { - let name = names[0].strip_prefix("/").or(Some(names[0].as_str())).unwrap(); - if let Ok(cir) = docker.inspect_container(name, None).await { - c.details = Some(cir); + if ctr.clone().image + .or(Some(String::new())) + .unwrap() + .contains("unit") { + let mut c = UnitdContainer::from(&ctr); + if let Some(names) = ctr.names { + if names.len() > 0 { + let name = names[0].strip_prefix("/") + .or(Some(names[0].as_str())).unwrap(); + if let Ok(cir) = docker + .inspect_container(name, None).await { + c.details = Some(cir); + } } } + mapped.push(c); } - mapped.push(c); - } } mapped } @@ -190,7 +206,7 @@ impl UnitdContainer { * that doesnt actually exist */ if cfg!(target_os = "macos") { - let mut abs = PathBuf::from("/"); + let mut abs = PathBuf::from(String::from(MAIN_SEPARATOR)); let m = matches.strip_prefix("/host_mnt/private") .unwrap_or(matches.strip_prefix("/host_mnt") .unwrap_or(matches.as_path())); @@ -238,19 +254,27 @@ impl UnitdContainer { * ON FAILURE returns wrapped error from Docker API */ pub async fn deploy_new_container( - socket: &String, + socket: ControlSocket, application: &String, image: &String, ) -> Result, UnitClientError> { match Docker::connect_with_local_defaults() { Ok(docker) => { let mut mounts = vec![]; - mounts.push(Mount { - typ: Some(MountTypeEnum::BIND), - source: Some(socket.clone()), - target: Some("/var/run".to_string()), - ..Default::default() - }); + // if a unix socket is specified, mounts its directory + if socket.is_local_socket() { + let mount_path = PathBuf::from(socket.clone()) + .as_path() + .to_string_lossy() + .to_string(); + mounts.push(Mount { + typ: Some(MountTypeEnum::BIND), + source: Some(mount_path), + target: Some("/var/run".to_string()), + ..Default::default() + }); + } + // mount application dir mounts.push(Mount { typ: Some(MountTypeEnum::BIND), source: Some(application.clone()), @@ -259,40 +283,57 @@ pub async fn deploy_new_container( ..Default::default() }); - let _ = docker - .create_image( - Some(CreateImageOptions { - from_image: image.as_str(), - ..Default::default() - }), - None, - None, - ) - .next() - .await - .unwrap() - .or_else(|err| { - Err(UnitClientError::UnitdDockerError { - message: err.to_string(), - }) - }); + let mut pb = ProgressBar::on(stderr(), 10); + let mut totals = HashMap::new(); + let mut stream = docker.create_image( + Some(CreateImageOptions { + from_image: image.as_str(), + ..Default::default() + }), None, None + ); + while let Some(res) = stream.next().await { + if let Ok(info) = res { + if let Some(id) = info.id { + if let Some(_) = totals.get_mut(&id) { + if let Some(delta) = info.progress_detail + .and_then(|detail| detail.current) { + pb.add(delta as u64); + } + } else { + if let Some(total) = info.progress_detail + .and_then(|detail| detail.total) { + totals.insert(id, total); + pb.total += total as u64; + } + } + } + } + } + pb.finish(); + // create the new unit container let resp: ContainerCreateResponse; - match docker - .create_container::( - None, - Config { - image: Some(image.clone()), - host_config: Some(HostConfig { - network_mode: Some("host".to_string()), - mounts: Some(mounts), - ..Default::default() - }), - ..Default::default() - }, - ) - .await - { + let host_conf = HostConfig { + mounts: Some(mounts), + network_mode: Some("host".to_string()), + ..Default::default() + }; + let mut container_conf = Config { + image: Some(image.clone()), + ..Default::default() + }; + if let ControlSocket::TcpSocket(ref uri) = socket { + let port = uri.port_u16().or(Some(80)).unwrap(); + // override port + container_conf.cmd = Some(vec![ + "unitd".to_string(), + "--no-daemon".to_string(), + "--control".to_string(), + format!("{}:{}", uri.host().unwrap(), port), + ]); + } + container_conf.host_config = Some(host_conf); + match docker.create_container::(None, container_conf).await { Err(err) => { return Err(UnitClientError::UnitdDockerError { message: err.to_string(), @@ -301,6 +342,8 @@ pub async fn deploy_new_container( Ok(response) => resp = response, } + // create container gives us an ID + // but start container requires a name let mut list_container_filters = HashMap::new(); list_container_filters.insert("id".to_string(), vec![resp.id]); match docker @@ -312,26 +355,28 @@ pub async fn deploy_new_container( })) .await { - Err(e) => Err(UnitClientError::UnitdDockerError { message: e.to_string() }), + // somehow our container doesnt exist + Err(e) => Err(UnitClientError::UnitdDockerError{ + message: e.to_string() + }), + // here it is! Ok(info) => { if info.len() < 1 { return Err(UnitClientError::UnitdDockerError { message: "couldnt find new container".to_string(), }); - } - if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { - return Err(UnitClientError::UnitdDockerError { - message: "new container has no name".to_string(), - }); - } + } else if info[0].names.is_none() || + info[0].names.clone().unwrap().len() < 1 { + return Err(UnitClientError::UnitdDockerError { + message: "new container has no name".to_string(), + }); + } - match docker - .start_container( - info[0].names.clone().unwrap()[0].strip_prefix("/").unwrap(), - None::>, - ) - .await - { + // start our container + match docker.start_container( + info[0].names.clone().unwrap()[0].strip_prefix(MAIN_SEPARATOR).unwrap(), + None::>, + ).await { Err(err) => Err(UnitClientError::UnitdDockerError { message: err.to_string(), }), diff --git a/tools/unitctl/unit-client-rs/src/unitd_instance.rs b/tools/unitctl/unit-client-rs/src/unitd_instance.rs index a7fb1bdc..ace8e858 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_instance.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_instance.rs @@ -26,6 +26,7 @@ impl Serialize for UnitdInstance { where S: Serializer, { + // 11 = fields to serialize let mut state = serializer.serialize_map(Some(11))?; let runtime_flags = self .process diff --git a/tools/unitctl/unit-client-rs/src/unitd_process.rs b/tools/unitctl/unit-client-rs/src/unitd_process.rs index 47ffcb5d..3dc0c3af 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_process.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_process.rs @@ -27,6 +27,7 @@ impl Serialize for UnitdProcess { where S: Serializer, { + // 6 = fields to serialize let mut state = serializer.serialize_map(Some(6))?; state.serialize_entry("pid", &self.process_id)?; state.serialize_entry("user", &self.user)?; diff --git a/tools/unitctl/unitctl/src/cmd/instances.rs b/tools/unitctl/unitctl/src/cmd/instances.rs index b9af75f6..a030f7d3 100644 --- a/tools/unitctl/unitctl/src/cmd/instances.rs +++ b/tools/unitctl/unitctl/src/cmd/instances.rs @@ -1,8 +1,11 @@ use crate::unitctl::{InstanceArgs, InstanceCommands}; use crate::{OutputFormat, UnitctlError}; +use crate::unitctl_error::ControlSocketErrorKind; + use std::path::PathBuf; use unit_client_rs::unitd_docker::deploy_new_container; use unit_client_rs::unitd_instance::UnitdInstance; +use unit_client_rs::control_socket_address::ControlSocket; pub(crate) async fn cmd(args: InstanceArgs) -> Result<(), UnitctlError> { if let Some(cmd) = args.command { @@ -12,19 +15,104 @@ pub(crate) async fn cmd(args: InstanceArgs) -> Result<(), UnitctlError> { ref application, ref image, } => { - println!("Pulling and starting a container from {}", image); - println!("Will mount {} to /var/run for socket access", socket); - println!("Will READ ONLY mount {} to /www for application access", application); - println!("Note: Container will be on host network"); - if !PathBuf::from(socket).is_dir() || !PathBuf::from(application).is_dir() { - eprintln!("application and socket paths must be directories"); + // validation for application dir + if !PathBuf::from(application).is_dir() { + eprintln!("application path must be a directory"); + Err(UnitctlError::NoFilesImported) + } else if !PathBuf::from(application).as_path().exists() { + eprintln!("application path must exist"); Err(UnitctlError::NoFilesImported) + } else { - deploy_new_container(socket, application, image).await.map_or_else( + let addr = ControlSocket::parse_address(socket); + if let Err(e) = addr { + return Err(UnitctlError::UnitClientError{source: e}); + } + + // validate we arent processing an abstract socket + if let ControlSocket::UnixLocalAbstractSocket(_) = addr.as_ref().unwrap() { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: "cannot pass abstract socket to docker container".to_string(), + }) + } + + // warn user of OSX docker limitations + if let ControlSocket::UnixLocalSocket(ref sock_path) = addr.as_ref().unwrap() { + if cfg!(target_os = "macos") { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: format!("Docker on OSX will break unix sockets mounted {} {}", + "in containers, see the following link for more information", + "https://github.com/docker/for-mac/issues/483"), + }) + } + + if !sock_path.is_dir() { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: format!("user must specify a directory of UNIX socket directory"), + }) + } + } + + // validate a TCP URI + if let ControlSocket::TcpSocket(uri) = addr.as_ref().unwrap() { + if let Some(host) = uri.host() { + if host != "127.0.0.1" { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: "TCP URI must point to 127.0.0.1".to_string(), + }) + } + } else { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: "TCP URI must point to a host".to_string(), + }) + } + + if let Some(port) = uri.port_u16() { + if port < 1025 { + eprintln!("warning! you are asking docker to forward a privileged port. {}", + "please make sure docker has access to it"); + } + } else { + return Err(UnitctlError::ControlSocketError{ + kind: ControlSocketErrorKind::General, + message: "TCP URI must specify a port".to_string(), + }) + } + + if uri.path() != "/" { + eprintln!("warning! path {} will be ignored", uri.path()) + } + } + + // reflect changes to user + // print this to STDERR to avoid polluting deserialized data output + eprintln!("> Pulling and starting a container from {}", image); + eprintln!("> Will READ ONLY mount {} to /www for application access", application); + eprintln!("> Container will be on host network"); + match addr.as_ref().unwrap() { + ControlSocket::UnixLocalSocket(path) => + eprintln!("> Will mount directory containing {} to /var/www for control API", + path.as_path().to_string_lossy()), + ControlSocket::TcpSocket(uri) => + eprintln!("> Will forward port {} for control API", uri.port_u16().unwrap()), + _ => unimplemented!(), // abstract socket case ruled out previously + } + + if cfg!(target_os = "macos") { + eprintln!("> mac users: enable host networking in docker desktop"); + } + + // do the actual deployment + deploy_new_container(addr.unwrap(), application, image).await.map_or_else( |e| Err(UnitctlError::UnitClientError { source: e }), |warn| { for i in warn { - println!("warning from docker: {}", i); + eprintln!("warning! from docker: {}", i); } Ok(()) }, -- cgit