src/s3.rs
//! Minimal S3 object store client: presigned requests sent with a blocking agent.
use std::{
env,
io::Read,
process::{Command, Stdio},
time::Duration,
};
use rusty_s3::actions::{list_objects_v2::ListObjectsContent, ListObjectsV2};
use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle};
use ureq::{Agent, AsSendBody, Error::StatusCode};
use crate::Result;
/// Lifetime of each presigned URL.
const TTL: Duration = Duration::from_secs(300);
/// Runs the AWS CLI with `args` and returns its trimmed standard output.
fn aws(args: &[&str]) -> Result<String> {
let out = Command::new("aws")
.args(args)
.stderr(Stdio::inherit())
.output()
.map_err(|e| {
format!("set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or install the AWS CLI ({e})")
})?;
if !out.status.success() {
return Err(format!("aws {} failed", args.join(" ")).into());
}
Ok(String::from_utf8(out.stdout)?.trim().to_owned())
}
/// Reads credentials from `NAME=value` lines, the `env-no-export` format of the AWS CLI.
fn parse_creds(text: &str) -> Option<Credentials> {
let var = |name| {
text.lines()
.find_map(|l| l.strip_prefix::<&str>(name)?.strip_prefix('='))
};
let (key, secret) = (var("AWS_ACCESS_KEY_ID")?, var("AWS_SECRET_ACCESS_KEY")?);
Some(match var("AWS_SESSION_TOKEN") {
Some(token) => Credentials::new_with_token(key, secret, token),
None => Credentials::new(key, secret),
})
}
pub struct Store {
bucket: Bucket,
creds: Credentials,
agent: Agent,
}
impl Store {
/// Builds a store for `bucket` from the standard `AWS_*` environment variables.
/// Without credentials in the environment, the AWS CLI supplies them and the region.
pub fn from_env(bucket: &str) -> Result<Self> {
let env_creds = Credentials::from_env();
let cli_region = || aws(&["configure", "get", "region"]).ok();
let region = env::var("AWS_REGION")
.or_else(|_| env::var("AWS_DEFAULT_REGION"))
.ok()
.or_else(|| env_creds.is_none().then(cli_region).flatten())
.unwrap_or_else(|| "us-east-1".into());
let creds = match env_creds {
Some(creds) => creds,
None => {
let args = [
"configure",
"export-credentials",
"--format",
"env-no-export",
];
parse_creds(&aws(&args)?).ok_or("the AWS CLI returned no credentials")?
}
};
let (endpoint, style) = match env::var("AWS_ENDPOINT_URL") {
Ok(url) => (url, UrlStyle::Path),
Err(_) => (
format!("https://s3.{region}.amazonaws.com"),
UrlStyle::VirtualHost,
),
};
Ok(Self {
bucket: Bucket::new(endpoint.parse()?, style, bucket.to_owned(), region)?,
creds,
agent: Agent::new_with_defaults(),
})
}
/// Lists all objects whose key starts with `prefix`.
pub fn list(&self, prefix: &str) -> Result<Vec<ListObjectsContent>> {
let mut objects = Vec::new();
let mut token: Option<String> = None;
loop {
let url = {
let mut action = self.bucket.list_objects_v2(Some(&self.creds));
// Ask for raw keys, not percent-encoded ones.
action.query_mut().remove("encoding-type");
action.with_prefix(prefix);
if let Some(token) = &token {
action.with_continuation_token(token.as_str());
}
action.sign(TTL)
};
let body = self
.agent
.get(url.as_str())
.call()?
.body_mut()
.read_to_string()?;
let page = ListObjectsV2::parse_response(&body)?;
objects.extend(page.contents);
token = page.next_continuation_token;
if token.is_none() {
return Ok(objects);
}
}
}
/// Opens the object at `key` for reading, or returns `None` if it does not exist.
pub fn get(&self, key: &str) -> Result<Option<impl Read>> {
let url = self.bucket.get_object(Some(&self.creds), key).sign(TTL);
match self.agent.get(url.as_str()).call() {
Ok(res) => Ok(Some(res.into_body().into_reader())),
Err(StatusCode(404)) => Ok(None),
Err(e) => Err(e.into()),
}
}
/// Writes `body` to `key`, replacing any existing object.
pub fn put(&self, key: &str, body: impl AsSendBody) -> Result<()> {
let url = self.bucket.put_object(Some(&self.creds), key).sign(TTL);
self.agent.put(url.as_str()).send(body)?;
Ok(())
}
/// Writes `body` to `key` only if no object exists there. Returns whether it wrote.
pub fn put_if_absent(&self, key: &str, body: &str) -> Result<bool> {
let mut action = self.bucket.put_object(Some(&self.creds), key);
action.headers_mut().insert("if-none-match", "*");
let url = action.sign(TTL);
match self
.agent
.put(url.as_str())
.header("if-none-match", "*")
.send(body)
{
Ok(_) => Ok(true),
Err(StatusCode(409 | 412)) => Ok(false),
Err(e) => Err(e.into()),
}
}
pub fn delete(&self, key: &str) -> Result<()> {
let url = self.bucket.delete_object(Some(&self.creds), key).sign(TTL);
self.agent.delete(url.as_str()).call()?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn creds() {
let cases = [
("AWS_ACCESS_KEY_ID=k\nAWS_SECRET_ACCESS_KEY=s", Some(("k", "s", None))),
(
"AWS_ACCESS_KEY_ID=k\nAWS_SECRET_ACCESS_KEY=s=\nAWS_SESSION_TOKEN=t\nAWS_CREDENTIAL_EXPIRATION=x",
Some(("k", "s=", Some("t"))),
),
("AWS_ACCESS_KEY_ID=k", None),
("", None),
];
for (input, want) in cases {
let got = parse_creds(input);
let got = got.as_ref().map(|c| (c.key(), c.secret(), c.token()));
assert_eq!(got, want, "{input}");
}
}
}