Commit d8de5f9f authored by Jeremy Soller's avatar Jeremy Soller
Browse files

Merge branch 'transaction' into 'master'

Transactions

See merge request !9
parents e7130b2b 2312d23b
......@@ -8,6 +8,7 @@ repository = "https://gitlab.redox-os.org/redox-os/pkgar"
edition = "2018"
[dependencies]
bitflags = "1.2.1"
blake3 = { version = "0.3.6", default_features = false, features = ["rayon"] }
plain = "0.2.3"
sodiumoxide = { version = "0.2.6", default_features = false }
......
//! The packed structs represent the on-disk format of pkgar
use blake3::Hash;
use plain::Plain;
use crate::{Error, Mode};
#[derive(Clone, Copy, Debug)]
#[repr(packed)]
pub struct Entry {
......@@ -17,8 +20,8 @@ pub struct Entry {
}
impl Entry {
pub fn blake3(&self) -> [u8; 32] {
self.blake3
pub fn blake3(&self) -> Hash {
Hash::from(self.blake3)
}
pub fn offset(&self) -> u64 {
......@@ -29,12 +32,13 @@ impl Entry {
self.size
}
pub fn mode(&self) -> u32 {
self.mode
pub fn mode(&self) -> Result<Mode, Error> {
Mode::from_bits(self.mode)
.ok_or(Error::InvalidMode(self.mode))
}
/// Retrieve the path, ending at the first NUL
pub fn path(&self) -> &[u8] {
pub fn path_bytes(&self) -> &[u8] {
let mut i = 0;
while i < self.path.len() {
if self.path[i] == 0 {
......
......@@ -4,9 +4,10 @@ use core::fmt::{Display, Formatter, Result};
#[derive(Debug)]
pub enum Error {
InvalidBlake3,
InvalidData,
InvalidKey,
InvalidBlake3,
InvalidMode(u32),
InvalidSignature,
Plain(plain::Error),
Overflow,
......@@ -19,10 +20,11 @@ impl Display for Error {
use Error::*;
let msg = match self {
InvalidData => "DataInvalid".to_string(),
InvalidKey => "KeyInvalid".to_string(),
InvalidBlake3 => "InvalidBlake3".to_string(),
InvalidSignature => "InvalidSignature".to_string(),
InvalidBlake3 => "Invalid Blake3".to_string(),
InvalidData => "Data Invalid".to_string(),
InvalidKey => "Key Invalid".to_string(),
InvalidMode(mode) => format!("Invalid Mode: {:o}", mode),
InvalidSignature => "Invalid Signature".to_string(),
Plain(err) => format!("Plain: {:?}", err),
Overflow => "Overflow".to_string(),
TryFromInt(err) => format!("TryFromInt: {}", err),
......
......@@ -4,6 +4,8 @@ extern crate alloc;
use core::mem;
use bitflags::bitflags;
pub use crate::entry::Entry;
pub use crate::error::Error;
pub use crate::header::Header;
......@@ -17,6 +19,28 @@ mod package;
pub const HEADER_SIZE: usize = mem::size_of::<Header>();
pub const ENTRY_SIZE: usize = mem::size_of::<Entry>();
bitflags! {
/// Ensures that all platforms use the same mode defines.
pub struct Mode: u32 {
const PERM = 0o007777;
const KIND = 0o170000;
const FILE = 0o100000;
const SYMLINK = 0o120000;
}
}
impl Mode {
/// Only any kind bits
pub fn kind(self) -> Mode {
self & Mode::KIND
}
/// Only any permissions bits
pub fn perm(self) -> Mode {
self & Mode::PERM
}
}
#[cfg(test)]
mod tests {
use core::mem;
......
......@@ -37,12 +37,12 @@ pub trait PackageSrc {
}
/// Read from this src at a given entry's data with a given offset within that entry
fn read_entry(&mut self, entry: Entry, offset: u64, buf: &mut [u8]) -> Result<usize, Self::Err> {
if offset > entry.size {
fn read_entry(&mut self, entry: Entry, offset: usize, buf: &mut [u8]) -> Result<usize, Self::Err> {
if offset as u64 > entry.size {
return Ok(0);
}
let mut end = usize::try_from(entry.size - offset)
let mut end = usize::try_from(entry.size - offset as u64)
.map_err(Error::TryFromInt)?;
if end > buf.len() {
......@@ -52,9 +52,9 @@ pub trait PackageSrc {
let offset =
HEADER_SIZE as u64 +
self.header().entries_size()? +
entry.offset + offset;
entry.offset + offset as u64;
self.read_at(offset, &mut buf[..end])
self.read_at(offset as u64, &mut buf[..end])
}
}
......
......@@ -82,16 +82,16 @@ pub struct PublicKeyFile {
impl PublicKeyFile {
/// Parse a `PublicKeyFile` from `file` (in toml format).
pub fn open(file: &Path) -> Result<PublicKeyFile, Error> {
let content = fs::read_to_string(file)
pub fn open(file: impl AsRef<Path>) -> Result<PublicKeyFile, Error> {
let content = fs::read_to_string(&file)
.map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::from(src),
})?;
toml::from_str(&content)
.map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::from(src),
})
}
......@@ -103,15 +103,15 @@ impl PublicKeyFile {
}
/// Shortcut to write the public key to `file`
pub fn save(&self, file: &Path) -> Result<(), Error> {
pub fn save(&self, file: impl AsRef<Path>) -> Result<(), Error> {
self.write(
File::create(file)
File::create(&file)
.map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::from(src)
})?
).map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src,
})
}
......@@ -215,16 +215,16 @@ impl SecretKeyFile {
}
/// Parse a `SecretKeyFile` from `file` (in toml format).
pub fn open(file: &Path) -> Result<SecretKeyFile, Error> {
let content = fs::read_to_string(file)
pub fn open(file: impl AsRef<Path>) -> Result<SecretKeyFile, Error> {
let content = fs::read_to_string(&file)
.map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::Io(src),
})?;
toml::from_str(&content)
.map_err(|src| Error {
path:file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::Deser(src),
})
}
......@@ -239,19 +239,19 @@ impl SecretKeyFile {
///
/// Make sure to call `encrypt()` in order to encrypt
/// the private key, otherwise it will be stored as plain text.
pub fn save(&self, file: &Path) -> Result<(), Error> {
pub fn save(&self, file: impl AsRef<Path>) -> Result<(), Error> {
self.write(
OpenOptions::new()
.write(true)
.create(true)
.mode(0o600)
.open(file)
.open(&file)
.map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src: ErrorKind::from(src),
})?
).map_err(|src| Error {
path: file.to_path_buf(),
path: file.as_ref().to_path_buf(),
src,
})
}
......
......@@ -86,7 +86,7 @@ fn cli() -> Result<i32, Error> {
.expect("Secret key was encrypted after being decrypted");
if let Some(file) = submatches.value_of("file") {
pkey.save(file.as_ref())?;
pkey.save(file)?;
} else {
pkey.write(io::stdout().lock())
.map_err(|src| Error {
......
......@@ -24,6 +24,10 @@ version = "0.3.6"
default-features = false
features = ["rayon"]
[dev-dependencies]
copy_dir = "0.1.2"
tempfile = "3.1.0"
[features]
default = ["clap", "std"]
std = []
......
use std::ffi::OsStr;
use std::fs;
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::fs::{self, File};
use std::io::{self, Seek, SeekFrom, Write};
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::{symlink, OpenOptionsExt, PermissionsExt};
use std::path::{Component, Path};
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use blake3::{Hash, Hasher};
use pkgar_core::{Entry, Header, PackageSrc};
use pkgar_core::{Entry, Header, Mode, PackageSrc};
use pkgar_keys::PublicKeyFile;
use sodiumoxide::crypto::sign;
use crate::Error;
use crate::{Error, ErrorKind, READ_WRITE_HASH_BUF_SIZE};
use crate::ext::{copy_and_hash, EntryExt};
use crate::package::PackageFile;
// This ensures that all platforms use the same mode defines
const MODE_PERM: u32 = 0o7777;
const MODE_KIND: u32 = 0o170000;
const MODE_FILE: u32 = 0o100000;
const MODE_SYMLINK: u32 = 0o120000;
//TODO: Refactor to reduce duplication between these functions
fn copy_and_hash<R: Read, W: Write>(mut read: R, mut write: W, buf: &mut [u8]) -> Result<(u64, Hash), Error> {
let mut hasher = Hasher::new();
let mut total = 0;
loop {
let count = read.read(buf)?;
if count == 0 {
break;
}
total += count as u64;
//TODO: Progress
write.write_all(&buf[..count])?;
hasher.update_with_join::<blake3::join::RayonJoin>(&buf[..count]);
}
Ok((total, hasher.finalize()))
}
fn copy_entry_and_hash<W: Write>(
src: &mut PackageFile,
entry: Entry,
mut write: W,
buf: &mut [u8]
) -> Result<(u64, Hash), Error> {
let mut hasher = Hasher::new();
let mut total = 0;
loop {
let count = src.read_entry(entry, total, buf)?;
if count == 0 {
break;
}
total += count as u64;
write.write_all(&buf[..count])?;
hasher.update_with_join::<blake3::join::RayonJoin>(&buf[..count]);
}
Ok((total, hasher.finalize()))
}
use crate::transaction::Transaction;
fn folder_entries<P, Q>(base: P, path: Q, entries: &mut Vec<Entry>) -> io::Result<()>
where P: AsRef<Path>, Q: AsRef<Path>
......@@ -93,11 +50,15 @@ fn folder_entries<P, Q>(base: P, path: Q, entries: &mut Vec<Entry>) -> io::Resul
path_bytes[..relative_bytes.len()].copy_from_slice(relative_bytes);
let file_type = metadata.file_type();
let mut mode = metadata.permissions().mode() & MODE_PERM;
let file_mode = metadata.permissions().mode();
//TODO: Use pkgar_core::Mode for all ops. This is waiting on error
// handling.
let mut mode = file_mode & Mode::PERM.bits();
if file_type.is_file() {
mode |= MODE_FILE;
mode |= Mode::FILE.bits();
} else if file_type.is_symlink() {
mode |= MODE_SYMLINK;
mode |= Mode::SYMLINK.bits();
} else {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
......@@ -117,10 +78,14 @@ fn folder_entries<P, Q>(base: P, path: Q, entries: &mut Vec<Entry>) -> io::Resul
Ok(())
}
pub fn create(secret_path: &str, archive_path: &str, folder: &str) -> Result<(), Error> {
pub fn create(
secret_path: impl AsRef<Path>,
archive_path: impl AsRef<Path>,
folder: impl AsRef<Path>,
) -> Result<(), Error> {
let secret_key = pkgar_keys::get_skey(&secret_path.as_ref())?
.key()
.expect(&format!("{} was encrypted?", secret_path));
.expect(&format!("{} was encrypted?", secret_path.as_ref().display()));
//TODO: move functions to library
......@@ -128,11 +93,16 @@ pub fn create(secret_path: &str, archive_path: &str, folder: &str) -> Result<(),
.write(true)
.create(true)
.truncate(true)
.open(archive_path)?;
.open(&archive_path)
.map_err(|e| Error::from(e).path(&archive_path) )?;
// Create a list of entries
let mut entries = Vec::new();
folder_entries(folder, folder, &mut entries)?;
folder_entries(&folder, &folder, &mut entries)
.map_err(|e| Error::from(e)
.reason("Recursing buildroot")
.path(&folder)
)?;
// Create initial header
let mut header = Header {
......@@ -149,46 +119,67 @@ pub fn create(secret_path: &str, archive_path: &str, folder: &str) -> Result<(),
for entry in &mut entries {
entry.offset = data_size;
data_size = data_size.checked_add(entry.size)
.ok_or(Error::Core(pkgar_core::Error::Overflow))?;
.ok_or(pkgar_core::Error::Overflow)?;
}
// Seek to data offset
let data_offset = header.total_size()?;
archive_file.seek(SeekFrom::Start(data_offset as u64))?;
archive_file.seek(SeekFrom::Start(data_offset as u64))
.map_err(|e| Error::from(e)
.reason(format!("Seek to {} (data offset)", data_offset))
.path(&archive_path)
)?;
//TODO: fallocate data_offset + data_size
// Stream each file, writing data and calculating b3sums
let mut header_hasher = blake3::Hasher::new();
let mut buf = vec![0; 4 * 1024 * 1024];
for entry in &mut entries {
let relative = Path::new(OsStr::from_bytes(entry.path()));
let path = Path::new(folder).join(relative);
let relative = entry.check_path()?;
let path = folder.as_ref().join(relative);
let mode = entry.mode()
.map_err(|e| Error::from(e)
.entry(*entry)
)?;
let mode_kind = entry.mode & MODE_KIND;
let (total, hash) = match mode_kind {
MODE_FILE => {
let (total, hash) = match mode.kind() {
Mode::FILE => {
let mut entry_file = fs::OpenOptions::new()
.read(true)
.open(path)?;
copy_and_hash(&mut entry_file, &mut archive_file, &mut buf)?
.open(&path)
.map_err(|e| Error::from(e).path(&path) )?;
copy_and_hash(&mut entry_file, &mut archive_file, &mut buf)
.map_err(|e| Error::from(e)
.reason(format!("Writing entry to archive: '{}'", relative.display()))
.path(&path)
)?
},
MODE_SYMLINK => {
let destination = fs::read_link(path)?;
Mode::SYMLINK => {
let destination = fs::read_link(&path)
.map_err(|e| Error::from(e).path(&path) )?;
let mut data = destination.as_os_str().as_bytes();
copy_and_hash(&mut data, &mut archive_file, &mut buf)?
copy_and_hash(&mut data, &mut archive_file, &mut buf)
.map_err(|e| Error::from(e)
.reason(format!("Writing entry to archive: '{}'", relative.display()))
.path(&path)
)?
},
_ => {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("Unsupported mode {:#o}", { entry.mode })
)));
}
_ => return Err(Error::from(
pkgar_core::Error::InvalidMode(mode.bits())
)
.entry(*entry)),
};
if total != { entry.size } {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("Copied {} instead of {}", total, { entry.size })
)));
if total != entry.size() {
return Err(ErrorKind::LengthMismatch {
actual: total,
expected: entry.size(),
}
.as_error()
.entry(*entry)
);
}
entry.blake3.copy_from_slice(hash.as_bytes());
......@@ -203,131 +194,95 @@ pub fn create(secret_path: &str, archive_path: &str, folder: &str) -> Result<(),
header.signature = sign::sign_detached(unsafe { &plain::as_bytes(&header)[64..] }, &secret_key).0;
// Write archive header
archive_file.seek(SeekFrom::Start(0))?;
archive_file.seek(SeekFrom::Start(0))
.map_err(|e| Error::from(e).path(&archive_path) )?;
archive_file.write_all(unsafe {
plain::as_bytes(&header)
})?;
})
.map_err(|e| Error::from(e).path(&archive_path) )?;
// Write each entry header
for entry in &entries {
let checked_path = entry.check_path()?;
archive_file.write_all(unsafe {
plain::as_bytes(entry)
})?;
})
.map_err(|e| Error::from(e)
.reason(format!("Write entry {}", checked_path.display()))
.path(&archive_path)
)?;
}
Ok(())
}
pub fn extract(public_path: &str, archive_path: &str, folder: &str) -> Result<(), Error> {
let public_key = PublicKeyFile::open(&public_path.as_ref())?.pkey;
pub fn extract(
pkey_path: impl AsRef<Path>,
archive_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> Result<(), Error> {
let pkey = PublicKeyFile::open(&pkey_path.as_ref())?.pkey;
let mut package = PackageFile::new(archive_path, &public_key)?;
let entries = package.read_entries()?;
let mut package = PackageFile::new(archive_path, &pkey)?;
// TODO: Validate that all entries can be installed, before installing
Transaction::install(&mut package, base_dir)?
.commit()?;
let folder_path = Path::new(folder);
let mut buf = vec![0; 4 * 1024 * 1024];
let mut renames = Vec::new();
for entry in entries {
let relative = Path::new(OsStr::from_bytes(entry.path()));
for component in relative.components() {
match component {
Component::Normal(_) => (),
invalid => {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("entry path contains invalid component: {:?}", invalid)
)));
}
}
}
Ok(())
}
let entry_path = folder_path.join(relative);
if ! entry_path.starts_with(&folder_path) {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("entry path escapes from folder: {:?}", relative)
)));
}
pub fn remove(
pkey_path: impl AsRef<Path>,
archive_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
) -> Result<(), Error> {
let pkey = PublicKeyFile::open(&pkey_path.as_ref())?.pkey;
let entry_hash = Hash::from(entry.blake3);
let temp_name = if let Some(file_name) = entry_path.file_name().and_then(|x| x.to_str())
{
format!(".pkgar.{}", file_name)
} else {
format!(".pkgar.{}", entry_hash.to_hex())
};
let temp_path = if let Some(parent) = entry_path.parent() {
fs::create_dir_all(parent)?;
parent.join(temp_name)
} else {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("entry path has no parent: {:?}", entry_path)
)));
};
let mut package = PackageFile::new(archive_path, &pkey)?;
let mode = entry.mode;
let mode_kind = mode & MODE_KIND;
let mode_perm = mode & MODE_PERM;
let (total, hash) = match mode_kind {
MODE_FILE => {
//TODO: decide what to do when temp files are left over
let mut temp_file = fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(mode_perm)
.open(&temp_path)?;
copy_entry_and_hash(&mut package, entry, &mut temp_file, &mut buf)?
},
MODE_SYMLINK => {
let mut data = Vec::new();
let (total, hash) = copy_entry_and_hash(&mut package, entry, &mut data, &mut buf)?;
let os_str: &OsStr = OsStrExt::from_bytes(data.as_slice());
symlink(os_str, &temp_path)?;
(total, hash)
},
_ => {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("Unsupported mode {:#o}", mode)
)));
}
};
if total != entry.size {
return Err(Error::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("Copied {} instead of {}", total, entry.size())
)));
}
if entry_hash != hash {
let _ = fs::remove_file(temp_path);
return Err(Error::Core(pkgar_core::Error::InvalidBlake3));
}
Transaction::remove(&mut package, base_dir)?
.commit()?;
renames.push((temp_path, entry_path));
}
Ok(())
}