Merge branch 'keys-lib-cheap-header' into 'master'

Keys lib

See merge request !6

Cargo.toml

-[package]
-name = "pkgar"
-version = "0.1.6"
-description = "Redox Package Archive"
-license = "MIT"
-authors = ["Jeremy Soller <jackpot51@gmail.com>", "Wesley Hershberger <mggmugginsmc@gmail.com>"]
-repository = "https://gitlab.redox-os.org/redox-os/pkgar"
-edition = "2018"
-
-[[bin]]
-name = "pkgar"
-required-features = ["clap", "rand", "std"]
-
-[dependencies]
-plain = "0.2.3"
-sodiumoxide = { version = "0.2.5", default_features = false }
-
-[dependencies.clap]
-version = "2.33.0"
-optional = true
-
-[dependencies.rand]
-version = "0.7.2"
-optional = true
-
-[dependencies.rand_core]
-version = "0.5.1"
-default-features = false
-
-[dependencies.blake3]
-version = "0.2.1"
-default-features = false
-features = ["rayon"]
-
-[features]
-default = ["clap", "rand", "std"]
-std = []
+[workspace]
+members = [
+    "pkgar",
+    "pkgar-core",
+    "pkgar-keys",
+]
 

README.md

 # pkgar - Package Archive
 
-pkgar refers to three related items - the file format, the library, and the
-command line executable.
+Pkgar is the package archive format for Redox OS.
 
-The pkgar format is not designed to be the best format for all archive uses,
-only the best default format for packages on Redox OS. It is reproducible,
-meaning archiving a directory will produce the same results every time. It
-provides cryptographic signatures and integrity checking for package files. It
-also allows this functionality to be used without storing the entire package
-archive, by only storing the package header. Large files, compression,
-encryption, and random access are not optimized for. Little endian is currently
-assumed, as well as Unix mode flags.
+## Project Layout
+There are currently two crates in this repo. See their READMEs for more specific
+docs:
+- `pkgar`: The implementation of the pkgar file format as a library, and a cli
+  tool for manpulating pkgar packages.
+- `pkgar-keys`: Key management tool/library for pkgar.
 
-***This specification is currently a work in progress***
-
-## File Format - .pkgar
-
-pkgar is a format for packages may be delivered in a single file (.pkgar), or as
-a header file (.pkgar_head) with an associated data file (.pkgar_data). The
-purpose of this is to allow downloading a header only and verifying local files
-before downloading file data. Concatenating the header and data files creates a
-valid single file: `cat example.pkgar_head example.pkgar_data > example.pkgar`
-
-### Header Portion
-
-The header portion is designed to contain the data required to verify files
-already installed on disk. It is signed using NaCl (or a compatible
-implementation such as libsodium), and contains the blake3, offset, size, mode,
-and name of each file. The user and group IDs are left out intentionally, to
-support the installation of a package either as root or as a user, for example,
-in the user's home directory.
-
-#### Header Struct
-
-The size of the header struct is 136 bytes. All fields are packed.
-
-- signature - 512-bit (64 byte) NaCl signature of header data
-- public_key - 256-bit (32 byte) NaCl public key used to generate signature
-- blake3 - 256-bit (32 byte) blake3 sum of the entry data
-- count - 64-bit count of entry structs, which immediately follow
-
-#### Entry Struct
-
-The size of the entry struct is 308 bytes. All fields are packed.
-
-- blake3 - 256-bit (32 byte) blake3 sum of the file data
-- offset - 64-bit little endian offset of file data in the data portion
-- size - 64-bit little endian size in bytes of the file data in the data portion
-- mode - 32-bit Unix permissions (user, group, other with read, write, execute)
-- path - 256 byte NUL-terminated relative path from extract directory
-
-### Data Portion
-
-The data portion is used to look up file data only. It could be compressed to
-produce a .pkgar_data.gz file, for example. It can be removed after the install
-is completed. It is possible for it to contain holes, invalid data, or
-unreferenced data - so long as the blake3 of files identified in the header are
-still valid. This data should be removed when an archive is rebuilt.
-
-### Operation
-
-A reader should first verify the header portion's signature matches that of a
-valid package source. Then, they should locate the entry for the file of
-interest. If desired, they can check if a locally cached file matches the
-referenced blake3. If this is not the case, they may access the data portion and
-verify that the data at the offset and length in the header entry matches the
-blake3. In that case, the data may be retrieved.

pkgar-core/Cargo.toml

+[package]
+name = "pkgar-core"
+version = "0.1.0"
+description = "Core Data Types for the Redox Package Archive"
+license = "MIT"
+authors = ["Jeremy Soller <jackpot51@gmail.com>", "Wesley Hershberger <mggmugginsmc@gmail.com>"]
+repository = "https://gitlab.redox-os.org/redox-os/pkgar"
+edition = "2018"
+
+[dependencies]
+blake3 = { version = "0.3.5", default_features = false, features = ["rayon"] }
+plain = "0.2.3"
+sodiumoxide = { version = "0.2.5", default_features = false }
+

src/entry.rs → pkgar-core/src/entry.rs

 //! The packed structs represent the on-disk format of pkgar
+use core::fmt;
 
 use plain::Plain;
 
@@ -18,6 +19,22 @@ pub struct Entry {
 }
 
 impl Entry {
+    pub fn blake3(&self) -> [u8; 32] {
+        self.blake3
+    }
+    
+    pub fn offset(&self) -> u64 {
+        self.offset
+    }
+    
+    pub fn size(&self) -> u64 {
+        self.size
+    }
+    
+    pub fn mode(&self) -> u32 {
+        self.mode
+    }
+    
     /// Retrieve the path, ending at the first NUL
     pub fn path(&self) -> &[u8] {
         let mut i = 0;
@@ -29,6 +46,46 @@ impl Entry {
         }
         &self.path[..i]
     }
+    
+    /*
+    pub fn read_at(&self, package: &mut Package, offset: u64, buf: &mut [u8]) -> Result<usize, Error> {
+        if offset >= self.size {
+            return Ok(0);
+        }
+        let mut end = offset.checked_add(buf.len() as u64)
+            .ok_or(Error::Overflow)?;
+        
+        if end > self.size {
+            end = self.size;
+        }
+        let buf_len = usize::try_from(end.checked_sub(offset).unwrap())
+            .map_err(Error::TryFromInt)?;
+        
+        package.src.read_at(
+            // Offset to first entry data
+            package.header.total_size()?
+                // Add offset to provided entry data
+                .checked_add(self.offset)
+                .ok_or(Error::Overflow)?
+                
+                // Offset into entry data
+                .checked_add(offset)
+                .ok_or(Error::Overflow)?,
+            &mut buf[..buf_len])
+    }*/
 }
 
 unsafe impl Plain for Entry {}
+
+impl fmt::Debug for Entry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Entry {{\n\tblake3: {:?},\n\toffset: {:?},\n\tsize: {:?},\n\tmode: {:#o},\n\tpath: {:?}\n}}",
+            self.blake3(),
+            self.offset(),
+            self.size(),
+            self.mode(),
+            &self.path[..],
+        )
+    }
+}
+

pkgar-core/src/error.rs

+use alloc::format;
+use alloc::string::ToString;
+use core::fmt::{Display, Formatter, Result};
+
+#[derive(Debug)]
+pub enum Error {
+    InvalidData,
+    InvalidKey,
+    InvalidBlake3,
+    InvalidSignature,
+    Plain(plain::Error),
+    Overflow,
+    TryFromInt(core::num::TryFromIntError),
+}
+
+//TODO: Improve Error messages
+impl Display for Error {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        use Error::*;
+        
+        let msg = match self {
+            InvalidData => "DataInvalid".to_string(),
+            InvalidKey => "KeyInvalid".to_string(),
+            InvalidBlake3 => "InvalidBlake3".to_string(),
+            InvalidSignature => "InvalidSignature".to_string(),
+            Plain(err) => format!("Plain: {:?}", err),
+            Overflow => "Overflow".to_string(),
+            TryFromInt(err) => format!("TryFromInt: {}", err),
+        };
+        write!(f, "{}", msg)
+    }
+}
+
+impl From<plain::Error> for Error {
+    fn from(err: plain::Error) -> Error {
+        Error::Plain(err)
+    }
+}
+
+impl From<core::num::TryFromIntError> for Error {
+    fn from(err: core::num::TryFromIntError) -> Error {
+        Error::TryFromInt(err)
+    }
+}
+

src/header.rs → pkgar-core/src/header.rs

 //! The packed structs represent the on-disk format of pkgar
 
 use core::convert::TryFrom;
+use core::fmt;
 use core::mem;
 use plain::Plain;
 use sodiumoxide::crypto::sign::{self, PublicKey};
@@ -48,14 +49,16 @@ impl Header {
 
     /// Parse header from raw header data without verification
     pub unsafe fn new_unchecked<'a>(data: &'a [u8]) -> Result<&'a Header, Error> {
-        plain::from_bytes(data)
-            .map_err(Error::Plain)
+        Ok(plain::from_bytes(data)?)
+    }
+
+    pub fn count(&self) -> u64 {
+        self.count
     }
 
     /// Retrieve the size of the entries
     pub fn entries_size(&self) -> Result<u64, Error> {
-        let entry_size = u64::try_from(mem::size_of::<Entry>())
-            .map_err(Error::TryFromInt)?;
+        let entry_size = u64::try_from(mem::size_of::<Entry>())?;
         self.count
             .checked_mul(entry_size)
             .ok_or(Error::Overflow)
@@ -63,8 +66,7 @@ impl Header {
 
     /// Retrieve the size of the Header and its entries
     pub fn total_size(&self) -> Result<u64, Error> {
-        let header_size = u64::try_from(mem::size_of::<Header>())
-            .map_err(Error::TryFromInt)?;
+        let header_size = u64::try_from(mem::size_of::<Header>())?;
         self.entries_size()?
             .checked_add(header_size)
             .ok_or(Error::Overflow)
@@ -72,8 +74,7 @@ impl Header {
 
     /// Parse entries from raw entries data and verify using blake3
     pub fn entries<'a>(&self, data: &'a [u8]) -> Result<&'a [Entry], Error> {
-        let entries_size = usize::try_from(self.entries_size()?)
-            .map_err(Error::TryFromInt)?;
+        let entries_size = usize::try_from(self.entries_size()?)?;
 
         let entries_data = data.get(..entries_size)
             .ok_or(Error::Plain(plain::Error::TooShort))?;
@@ -94,8 +95,18 @@ impl Header {
 
     /// Parse entries from raw entries data without verification
     pub unsafe fn entries_unchecked<'a>(data: &'a [u8]) -> Result<&'a [Entry], Error> {
-        plain::slice_from_bytes(data)
-            .map_err(Error::Plain)
+        Ok(plain::slice_from_bytes(data)?)
+    }
+}
+
+impl fmt::Debug for Header {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Header {{\n\tsignature: {:?},\n\tpublic_key: {:?},\n\tblake3: {:?},count: {:?},\n}}",
+            &self.signature[..],
+            self.public_key,
+            self.blake3,
+            self.count(),
+        )
     }
 }
 

src/lib.rs → pkgar-core/src/lib.rs

-#![cfg_attr(not(feature = "std"), no_std)]
+#![no_std]
+extern crate alloc;
+
+use core::mem;
 
 pub use crate::entry::Entry;
 pub use crate::error::Error;
 pub use crate::header::Header;
-pub use crate::package::{Package, PackageSrc};
+pub use crate::package::{PackageBuf, PackageSrc};
 
 mod entry;
 mod error;
 mod header;
 mod package;
 
-#[cfg(feature = "std")]
-pub mod bin;
+pub const HEADER_SIZE: usize = mem::size_of::<Header>();
+pub const ENTRY_SIZE: usize = mem::size_of::<Entry>();
 
 #[cfg(test)]
 mod tests {
     use core::mem;
 
-    use crate::{Entry, Header};
+    use crate::{Entry, ENTRY_SIZE, Header, HEADER_SIZE};
 
     #[test]
     fn header_size() {
         assert_eq!(mem::size_of::<Header>(), 136);
+        assert_eq!(HEADER_SIZE, 136);
     }
 
     #[test]
     fn entry_size() {
         assert_eq!(mem::size_of::<Entry>(), 308);
+        assert_eq!(ENTRY_SIZE, 308);
     }
 }
 

pkgar-core/src/package.rs

+use alloc::vec;
+use alloc::vec::Vec;
+use core::convert::TryFrom;
+
+use sodiumoxide::crypto::sign::PublicKey;
+
+use crate::{Entry, Error, HEADER_SIZE, Header};
+
+pub trait PackageSrc {
+    type Err: From<Error>;
+    
+    fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result<usize, Self::Err>;
+    
+    fn header(&self) -> Header;
+    
+    /// Users of implementors of `PackageSrc` should use `header` instead of `read_header` for
+    /// cheap header access.
+    /// Implementors of `PackageSrc` should call this function during initialization and store
+    /// the result to pass out with `header`.
+    fn read_header(&mut self, public_key: &PublicKey) -> Result<Header, Self::Err> {
+        let mut header_data = [0; HEADER_SIZE];
+        self.read_at(0, &mut header_data)?;
+        let header = Header::new(&header_data, &public_key)?;
+        Ok(header.clone())
+    }
+    
+    fn read_entries(&mut self) -> Result<Vec<Entry>, Self::Err> {
+        let header = self.header();
+        let entries_size = header.entries_size()
+            .and_then(|rslt| usize::try_from(rslt)
+                .map_err(Error::TryFromInt)
+            )?;
+        let mut entries_data = vec![0; entries_size];
+        self.read_at(HEADER_SIZE as u64, &mut entries_data)?;
+        let entries = header.entries(&entries_data)?;
+        Ok(entries.to_vec())
+    }
+    
+    /// Read from this src at a given entry's data with a given offset within that entry
+    fn read_entry(&mut self, entry: Entry, offset: u64, buf: &mut [u8]) -> Result<usize, Self::Err> {
+        if offset > entry.size {
+            return Ok(0);
+        }
+        
+        let mut end = usize::try_from(entry.size - offset)
+            .map_err(Error::TryFromInt)?;
+        
+        if end > buf.len() {
+            end = buf.len();
+        }
+        
+        let offset =
+            HEADER_SIZE as u64 +
+            self.header().entries_size()? +
+            entry.offset + offset;
+        
+        self.read_at(offset, &mut buf[..end])
+    }
+}
+
+//TODO: Test this impl...
+pub struct PackageBuf<'a> {
+    src: &'a [u8],
+    header: Header,
+}
+
+impl<'a> PackageBuf<'a> {
+    pub fn new(src: &'a [u8], public_key: &PublicKey) -> Result<PackageBuf<'a>, Error> {
+        let zeroes = [0; HEADER_SIZE];
+        let mut new = PackageBuf {
+            src,
+            header: unsafe { *Header::new_unchecked(&zeroes)? },
+        };
+        new.header = *Header::new(&new.src, &public_key)?;
+        Ok(new)
+    }
+}
+
+impl PackageSrc for PackageBuf<'_> {
+    type Err = Error;
+    
+    fn header(&self) -> Header {
+        self.header
+    }
+    
+    fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result<usize, Error> {
+        let start = usize::try_from(offset)
+            .map_err(Error::TryFromInt)?;
+        let len = self.src.len();
+        if start >= len {
+            return Ok(0);
+        }
+        let mut end = start.checked_add(buf.len())
+            .ok_or(Error::Overflow)?;
+        if end > len {
+            end = len;
+        }
+        buf.copy_from_slice(&self.src[start..end]);
+        Ok(end.checked_sub(start).unwrap())
+    }
+}
+

pkgar-keys/Cargo.toml

+[package]
+name = "pkgar-keys"
+version = "0.1.0"
+authors = ["Wesley Hershberger <mggmugginsmc@gmail.com>"]
+edition = "2018"
+
+[dependencies]
+clap = "2.33.1"
+dirs = "3.0.1"
+hex = { version = "0.4.2", features = ["serde"] }
+lazy_static = "1.4.0"
+seckey = "0.9.3"
+serde = { version = "1.0.114", default_features = false, features = ["derive"] }
+sodiumoxide = { version = "0.2.5", default_features = false }
+termion = "1.5.5"
+toml = "0.5.6"
+

pkgar-keys/README.md

+# `pkgar-keys`
+Key management for pkgar.
+
+Run the binary with `--help` for CLI documentation. Secret keys are stored in
+`~/.pkgar/keys` by default.
+
+Keys are stored in toml format, the key format is subject to change.
+

pkgar-keys/src/error.rs

+use std::error::Error as StdError;
+use std::fmt::{self, Display};
+use std::io;
+use std::path::PathBuf;
+
+#[derive(Debug)]
+pub struct FileError {
+    kind: Error,
+    src: PathBuf,
+}
+
+impl Display for FileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}: {}", self.src.display(), self.kind)
+    }
+}
+
+impl StdError for FileError {
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        self.kind.source()
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Custom(String),
+    Io(io::Error),
+    KeyInvalid,
+    KeyMismatch,
+    MAlloc,
+    NonceInvalid,
+    PassphraseIncorrect,
+    PassphraseMismatch,
+    Ser(toml::ser::Error),
+    Deser(toml::de::Error),
+}
+
+impl From<io::Error> for Error {
+    fn from(err: io::Error) -> Error {
+        Error::Io(err)
+    }
+}
+
+impl From<toml::de::Error> for Error {
+    fn from(err: toml::de::Error) -> Error {
+        Error::Deser(err)
+    }
+}
+
+impl From<toml::ser::Error> for Error {
+    fn from(err: toml::ser::Error) -> Error {
+        Error::Ser(err)
+    }
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let msg = match self {
+            Error::Custom(e) => e.clone(),
+            Error::KeyInvalid => "Key length invalid".to_string(),
+            Error::KeyMismatch => "Public and private keys do not match".to_string(),
+            Error::MAlloc => "Unable to allocate locked/zeroed memory".to_string(),
+            Error::NonceInvalid => "Nonce length invalid".to_string(),
+            Error::PassphraseIncorrect => "Incorrect passphrase".to_string(),
+            Error::PassphraseMismatch => "Passphrases do not match".to_string(),
+            Error::Io(err) => format!("{}", err),
+            Error::Ser(err) => format!("{}", err),
+            Error::Deser(err) => format!("{}", err),
+        };
+        
+        write!(f, "Error: {}", msg)
+    }
+}
+
+impl StdError for Error {
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        match self {
+            Error::Io(err) => Some(err),
+            Error::Ser(err) => Some(err),
+            Error::Deser(err) => Some(err),
+            _ => None,
+        }
+    }
+}
+

pkgar-keys/src/lib.rs

+mod error;
+
+use std::fs::File;
+use std::io::{self, Read, stdin, stdout, Write};
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+
+use hex::FromHex;
+use lazy_static::lazy_static;
+use seckey::SecKey;
+use serde::{Deserialize, Serialize};
+use sodiumoxide::crypto::{
+    pwhash,
+    secretbox,
+    sign,
+};
+use termion::input::TermRead;
+
+pub use error::Error;
+
+lazy_static! {
+    static ref HOMEDIR: PathBuf = {
+         dirs::home_dir()
+            .unwrap_or("./".into())
+    };
+    pub static ref DEFAULT_PUBKEY: PathBuf = {
+        Path::join(&HOMEDIR, ".pkgar/keys/id_ed25519.toml")
+    };
+    pub static ref DEFAULT_SECKEY: PathBuf = {
+        Path::join(&HOMEDIR, ".pkgar/keys/id_ed25519.pub.toml")
+    };