src/main.rs
//! Git remote helper that stores each ref as a bundle in an S3 bucket.
//!
//! Git runs it as `git-remote-s3 <remote> s3://<bucket>/<prefix>`.
mod s3;
use std::{
collections::HashSet,
env,
fs::{self, File},
io::{self, BufRead, Read},
path::PathBuf,
process::{self, Command, Stdio},
time::{SystemTime, UNIX_EPOCH},
};
use s3::Store;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
/// Age in seconds after which a push lock left by another client is ignored.
const LOCK_TTL: u64 = 60;
struct Remote {
store: Store,
prefix: String,
/// Scratch file for the bundle in transit.
tmp: PathBuf,
}
impl Remote {
/// Returns the `(sha, ref)` of every bundle below `<prefix>/<dir>`, newest first.
///
/// A ref has more than one bundle only after a push stopped before it deleted the old one.
/// The newest bundle is then the tip of the ref.
fn refs(&self, dir: &str) -> Result<Vec<(String, String)>> {
let root = format!("{}/", self.prefix);
let mut objects = self.store.list(&format!("{root}{dir}"))?;
objects.sort_by(|a, b| b.last_modified.cmp(&a.last_modified));
Ok(objects
.iter()
.filter_map(|o| parse_key(o.key.strip_prefix(&root)?))
.map(|(sha, name)| (sha.to_owned(), name.to_owned()))
.collect())
}
fn list(&self) -> Result<()> {
let mut refs = self.refs("refs/")?;
let mut seen = HashSet::new();
refs.retain(|(_, name)| seen.insert(name.clone()));
for (sha, name) in &refs {
println!("{sha} {name}");
}
if let Some(mut body) = self.store.get(&format!("{}/HEAD", self.prefix))? {
let mut head = String::new();
body.read_to_string(&mut head)?;
if refs.iter().any(|(_, name)| name == head.trim()) {
println!("@{} HEAD", head.trim());
}
}
Ok(())
}
fn fetch(&self, sha: &str, name: &str) -> Result<()> {
// An earlier bundle or fetch can already contain this object.
if git(&["cat-file", "-e", sha]).is_ok() {
return Ok(());
}
let dir = format!("{}/{name}/", self.prefix);
let mut body = match self.store.get(&format!("{dir}{sha}.bundle"))? {
Some(body) => body,
// A push replaced the bundle after the list. Unless that push was forced,
// the bundle that is there now contains this commit.
None => {
let gone = || format!("{name} changed on the remote, fetch again");
let refs = self.refs(&format!("{name}/"))?;
let (tip, _) = refs.iter().find(|(_, n)| n == name).ok_or_else(gone)?;
let key = format!("{dir}{tip}.bundle");
self.store.get(&key)?.ok_or_else(gone)?
}
};
io::copy(&mut body, &mut File::create(&self.tmp)?)?;
let res = git(&["bundle", "unbundle", &self.tmp.to_string_lossy()]);
fs::remove_file(&self.tmp)?;
res.map(drop)
}
fn push(&self, src: &str, dst: &str, force: bool) -> Result<()> {
let dir = format!("{}/{dst}/", self.prefix);
if src.is_empty() {
let objects = self.store.list(&dir)?;
// Skip keys of refs nested below this one.
let mut own = objects.iter().filter(|o| !o.key[dir.len()..].contains('/'));
return own.try_for_each(|o| self.store.delete(&o.key));
}
let sha = git(&["rev-parse", src])?;
git(&["bundle", "create", &self.tmp.to_string_lossy(), src])?;
let lock = format!("{dir}LOCK#.lock");
let res = self.lock(&lock).and_then(|()| {
let res = self.upload(&sha, dst, &dir, force);
self.store.delete(&lock)?;
res
});
fs::remove_file(&self.tmp)?;
res
}
/// Replaces the bundle of `dst` with the one in `self.tmp`. The caller holds the lock.
fn upload(&self, sha: &str, dst: &str, dir: &str, force: bool) -> Result<()> {
let old = self.refs(&format!("{dst}/"))?;
let old: Vec<&str> = old
.iter()
.filter(|(_, name)| name == dst)
.map(|(sha, _)| sha.as_str())
.collect();
if let Some(&tip) = old.first() {
let ancestor = || git(&["merge-base", "--is-ancestor", tip, sha]).is_ok();
if tip != sha && !force && !ancestor() {
return Err("fetch first".into());
}
}
if !old.contains(&sha) {
self.store
.put(&format!("{dir}{sha}.bundle"), &File::open(&self.tmp)?)?;
}
for old in old.iter().filter(|old| **old != sha) {
self.store.delete(&format!("{dir}{old}.bundle"))?;
}
if dst.starts_with("refs/heads/") {
self.store
.put_if_absent(&format!("{}/HEAD", self.prefix), dst)?;
}
Ok(())
}
/// Creates the lock object at `key`, replacing a lock older than [`LOCK_TTL`].
fn lock(&self, key: &str) -> Result<()> {
if self.store.put_if_absent(key, "")? {
return Ok(());
}
let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
let held = self.store.list(key)?;
let stale = |ts: &str| epoch(ts).is_some_and(|t| now > t + LOCK_TTL);
if held.iter().all(|o| stale(&o.last_modified)) {
self.store.delete(key)?;
if self.store.put_if_absent(key, "")? {
return Ok(());
}
}
Err("locked by another push".into())
}
}
/// Runs `git` with `args` and returns its trimmed standard output.
fn git(args: &[&str]) -> Result<String> {
let out = Command::new("git")
.args(args)
.stderr(Stdio::inherit())
.output()?;
if !out.status.success() {
return Err(format!("git {} failed", args[0]).into());
}
Ok(String::from_utf8(out.stdout)?.trim().to_owned())
}
/// Splits `s3://<bucket>/<prefix>` into bucket and prefix.
fn parse_url(url: &str) -> Option<(&str, &str)> {
let (bucket, prefix) = url.strip_prefix("s3://")?.split_once('/')?;
let prefix = prefix.trim_end_matches('/');
(!bucket.is_empty() && !prefix.is_empty()).then_some((bucket, prefix))
}
/// Splits a bundle key `<ref>/<sha>.bundle` into `(sha, ref)`.
fn parse_key(key: &str) -> Option<(&str, &str)> {
let (name, file) = key.rsplit_once('/')?;
Some((file.strip_suffix(".bundle")?, name))
}
/// Splits a push refspec `[+]<src>:<dst>` into `(src, dst, force)`.
fn parse_refspec(spec: &str) -> Option<(&str, &str, bool)> {
let (src, dst) = spec.split_once(':')?;
let plain = src.strip_prefix('+');
Some((plain.unwrap_or(src), dst, plain.is_some()))
}
/// Converts an RFC 3339 UTC time such as `2024-02-29T12:34:56.000Z` to Unix seconds.
fn epoch(ts: &str) -> Option<u64> {
let num = |range| ts.get::<std::ops::Range<usize>>(range)?.parse::<u64>().ok();
let (y, m, d) = (num(0..4)?, num(5..7)?, num(8..10)?);
// Days from civil date, with the year starting in March.
let (y, m) = if m > 2 { (y, m - 3) } else { (y - 1, m + 9) };
let days =
y / 400 * 146_097 + y % 400 * 365 + y % 400 / 4 - y % 400 / 100 + (153 * m + 2) / 5 + d
- 719_469;
Some(days * 86_400 + num(11..13)? * 3_600 + num(14..16)? * 60 + num(17..19)?)
}
fn run() -> Result<()> {
let url = env::args()
.nth(2)
.ok_or("usage: git-remote-s3 <remote> <url>")?;
let (bucket, prefix) = parse_url(&url).ok_or("url must be s3://<bucket>/<prefix>")?;
let remote = Remote {
store: Store::from_env(bucket)?,
prefix: prefix.to_owned(),
// Git does not set GIT_DIR outside a repository, for example for `git ls-remote <url>`.
tmp: env::var_os("GIT_DIR")
.map_or_else(env::temp_dir, PathBuf::from)
.join(format!("s3-{}.bundle", process::id())),
};
// A blank line ends a batch of fetch or push commands; outside a batch it ends the session.
let mut batch = false;
for line in io::stdin().lock().lines() {
let line = line?;
let words: Vec<&str> = line.split(' ').collect();
match words[..] {
["capabilities"] => println!("fetch\npush\n"),
["list"] | ["list", "for-push"] => {
remote.list()?;
println!();
}
["fetch", sha, name] => {
remote.fetch(sha, name)?;
batch = true;
}
["push", spec] => {
let (src, dst, force) = parse_refspec(spec).ok_or("bad refspec")?;
match remote.push(src, dst, force) {
Ok(()) => println!("ok {dst}"),
Err(e) => println!("error {dst} {}", e.to_string().replace('\n', " ")),
}
batch = true;
}
[""] if batch => {
println!();
batch = false;
}
[""] => break,
_ => return Err(format!("unknown command: {line}").into()),
}
}
Ok(())
}
fn main() {
if let Err(e) = run() {
eprintln!("git-remote-s3: {e}");
process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn url() {
let cases = [
("s3://bucket/repo", Some(("bucket", "repo"))),
("s3://bucket/a/b/", Some(("bucket", "a/b"))),
("s3://bucket", None),
("s3://bucket/", None),
("s3:///repo", None),
("https://bucket/repo", None),
];
for (input, want) in cases {
assert_eq!(parse_url(input), want, "{input}");
}
}
#[test]
fn key() {
let cases = [
(
"refs/heads/main/abc.bundle",
Some(("abc", "refs/heads/main")),
),
("refs/heads/a/b/abc.bundle", Some(("abc", "refs/heads/a/b"))),
("refs/heads/main/LOCK#.lock", None),
("HEAD", None),
];
for (input, want) in cases {
assert_eq!(parse_key(input), want, "{input}");
}
}
#[test]
fn refspec() {
let cases = [
(
"refs/heads/a:refs/heads/b",
Some(("refs/heads/a", "refs/heads/b", false)),
),
(
"+refs/heads/a:refs/heads/b",
Some(("refs/heads/a", "refs/heads/b", true)),
),
(":refs/heads/b", Some(("", "refs/heads/b", false))),
("refs/heads/a", None),
];
for (input, want) in cases {
assert_eq!(parse_refspec(input), want, "{input}");
}
}
#[test]
fn time() {
let cases = [
("1970-01-01T00:00:00Z", Some(0)),
("2000-03-01T00:00:00.000Z", Some(951_868_800)),
("2024-02-29T12:34:56.000Z", Some(1_709_210_096)),
("2024-12-31T23:59:59Z", Some(1_735_689_599)),
("yesterday", None),
];
for (input, want) in cases {
assert_eq!(epoch(input), want, "{input}");
}
}
}