From 2aee857d19826b1a0cc9ec0c31a6e40bf09fc1ce Mon Sep 17 00:00:00 2001
From: Xavier L'Heureux <xavier.lheureux@icloud.com>
Date: Tue, 25 Jun 2019 13:44:49 -0400
Subject: [PATCH] Add simple documentation to the parser
---
src/lib/expansion/words/mod.rs | 4 +-
src/lib/parser/lexers/arguments.rs | 70 +++++++++++++------
src/lib/parser/lexers/assignments/keys.rs | 8 +++
src/lib/parser/lexers/assignments/operator.rs | 12 ++++
.../parser/lexers/assignments/primitive.rs | 11 +++
src/lib/parser/lexers/mod.rs | 2 +
src/lib/parser/mod.rs | 2 +
src/lib/parser/pipelines.rs | 22 ++++--
src/lib/parser/quotes.rs | 5 +-
src/lib/parser/statement/parse.rs | 15 +++-
src/lib/parser/statement/splitter.rs | 11 +++
src/lib/types.rs | 1 +
12 files changed, 127 insertions(+), 36 deletions(-)
diff --git a/src/lib/expansion/words/mod.rs b/src/lib/expansion/words/mod.rs
index 2f3a50ab..75439a60 100644
--- a/src/lib/expansion/words/mod.rs
+++ b/src/lib/expansion/words/mod.rs
@@ -155,8 +155,8 @@ impl<'a, E: Expander + 'a> WordIterator<'a, E> {
b'[' if self.quotes == Quotes::None => level += 1,
b']' if self.quotes == Quotes::None => {
if level == 0 {
- let elements = ArgumentSplitter::new(&self.data[start..self.read])
- .collect::<Vec<&str>>();
+ let elements =
+ ArgumentSplitter::new(&self.data[start..self.read]).collect::<Vec<_>>();
self.read += 1;
return if let Some(&b'[') = self.data.as_bytes().get(self.read) {
diff --git a/src/lib/parser/lexers/arguments.rs b/src/lib/parser/lexers/arguments.rs
index 37b235a3..781971fa 100644
--- a/src/lib/parser/lexers/arguments.rs
+++ b/src/lib/parser/lexers/arguments.rs
@@ -7,39 +7,54 @@ enum Comm {
None,
}
+/// A type of paired token
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Field {
+ /// Processes (ex: `$(..)`)
Proc,
+ /// Literal array (ex: `[ 1 .. 3 ]`)
Array,
+ /// Brace expansion (ex: `{a,b,c,d}`)
Braces,
}
use self::Field::*;
+/// The depth of various paired structures
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Levels {
- parens: i32,
- array: i32,
- braces: i32,
+ /// Parentheses
+ parens: u8,
+ /// Array literals
+ array: u8,
+ /// Braces
+ braces: u8,
}
+/// Error with paired tokens
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Error)]
pub enum LevelsError {
- // paired
- #[error(display = "unmatched left paren")]
+ /// Unmatched opening parenthese
+ #[error(display = "unmatched opening parenthese")]
UnmatchedParen,
- #[error(display = "unmatched left bracket")]
+ /// Unmatched opening bracket
+ #[error(display = "unmatched opening bracket")]
UnmatchedBracket,
- #[error(display = "unmatched left brace")]
+ /// Unmatched opening brace
+ #[error(display = "unmatched opening brace")]
UnmatchedBrace,
- #[error(display = "extra right paren(s)")]
+ /// Extra closing parenthese(s)
+ #[error(display = "extra closing parenthese(s)")]
ExtraParen,
- #[error(display = "extra right bracket(s)")]
+ /// Extra closing bracket(s)
+ #[error(display = "extra closing bracket(s)")]
ExtraBracket,
- #[error(display = "extra right brace(s)")]
+ /// Extra closing brace(s)
+ #[error(display = "extra closing brace(s)")]
ExtraBrace,
}
impl Levels {
+ /// Add a new depth level
pub fn up(&mut self, field: Field) {
let level = match field {
Proc => &mut self.parens,
@@ -49,17 +64,26 @@ impl Levels {
*level += 1;
}
- pub fn down(&mut self, field: Field) {
+ /// Close paired tokens
+ pub fn down(&mut self, field: Field) -> Result<(), LevelsError> {
let level = match field {
- Proc => &mut self.parens,
- Array => &mut self.array,
- Braces => &mut self.braces,
+ Proc if self.parens > 0 => &mut self.parens,
+ Array if self.array > 0 => &mut self.array,
+ Braces if self.braces > 0 => &mut self.braces,
+
+ // errors
+ Proc => return Err(LevelsError::ExtraParen),
+ Array => return Err(LevelsError::ExtraBracket),
+ Braces => return Err(LevelsError::ExtraBrace),
};
*level -= 1;
+ Ok(())
}
+ /// Check if all parens where matched
pub fn are_rooted(&self) -> bool { self.parens == 0 && self.array == 0 && self.braces == 0 }
+ /// Check if all is ok
pub fn check(&self) -> Result<(), LevelsError> {
if self.parens > 0 {
Err(LevelsError::UnmatchedParen)
@@ -67,12 +91,6 @@ impl Levels {
Err(LevelsError::UnmatchedBracket)
} else if self.braces > 0 {
Err(LevelsError::UnmatchedBrace)
- } else if self.parens < 0 {
- Err(LevelsError::ExtraParen)
- } else if self.array < 0 {
- Err(LevelsError::ExtraBracket)
- } else if self.braces < 0 {
- Err(LevelsError::ExtraBrace)
} else {
Ok(())
}
@@ -93,6 +111,7 @@ pub struct ArgumentSplitter<'a> {
}
impl<'a> ArgumentSplitter<'a> {
+ /// Create a new argument splitter based on the provided data
pub fn new(data: &'a str) -> ArgumentSplitter<'a> {
ArgumentSplitter {
data,
@@ -156,9 +175,14 @@ impl<'a> Iterator for ArgumentSplitter<'a> {
continue;
}
b'[' => levels.up(Array),
- b']' => levels.down(Array),
+ b']' => {
+ let _ = levels.down(Array);
+ }
b'{' => levels.up(Braces),
- b'}' => levels.down(Braces),
+ b'}' => {
+ // TODO: handle errors here
+ let _ = levels.down(Braces);
+ }
b'(' => {
// Disable VARIAB + ARRAY and enable METHOD.
// if variab or array are set
@@ -171,7 +195,7 @@ impl<'a> Iterator for ArgumentSplitter<'a> {
}
b')' => {
self.method = false;
- levels.down(Proc)
+ let _ = levels.down(Proc);
}
// Toggle double quote rules.
diff --git a/src/lib/parser/lexers/assignments/keys.rs b/src/lib/parser/lexers/assignments/keys.rs
index 02bebadf..10366f9a 100644
--- a/src/lib/parser/lexers/assignments/keys.rs
+++ b/src/lib/parser/lexers/assignments/keys.rs
@@ -5,7 +5,9 @@ use err_derive::Error;
/// types are being assigned.
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
pub struct Key<'a> {
+ /// What should be the type of the literal
pub kind: Primitive,
+ /// What name should be given to the variable
pub name: &'a str,
}
@@ -13,14 +15,19 @@ pub struct Key<'a> {
/// by eliminating the lifetime requirements via allocating a `String`.
#[derive(Debug, PartialEq, Clone)]
pub struct KeyBuf {
+ /// What type should the literal be
pub kind: Primitive,
+ /// What name should be given to the variable
pub name: String,
}
+/// Failed to parse the literal as a variable corresponding to key
#[derive(Debug, PartialEq, Error, Eq)]
pub enum TypeError {
+ /// The value supplied is invalid
#[error(display = "invalid type supplied: {}", _0)]
Invalid(String),
+ /// The primitive type does not correspond to that of the function argument
#[error(display = "expected {}", _0)]
BadValue(Primitive),
}
@@ -114,6 +121,7 @@ impl<'a> KeyIterator<'a> {
}
}
+ /// Create a new iterator based on given data
pub fn new(data: &'a str) -> KeyIterator<'a> { KeyIterator { data, read: 0 } }
}
diff --git a/src/lib/parser/lexers/assignments/operator.rs b/src/lib/parser/lexers/assignments/operator.rs
index c1c1e6ff..4c75956e 100644
--- a/src/lib/parser/lexers/assignments/operator.rs
+++ b/src/lib/parser/lexers/assignments/operator.rs
@@ -1,17 +1,29 @@
use std::fmt::{self, Display, Formatter};
+/// An operation to do on a value
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum Operator {
+ /// Addition (only works on numeric types)
Add,
+ /// Concatenation (will also concat numeric types)
Concatenate,
+ /// Prepend the value (will also concat numeric types)
ConcatenateHead,
+ /// Division (only works on numeric types)
Divide,
+ /// Assignment
Equal,
+ /// Assign a default value
OptionalEqual,
+ /// Exponent (only works on numeric types)
Exponent,
+ /// Filter the array to remove the matching values (only works on array and map-like types)
Filter,
+ /// Euclidian Division (only available on numeric types, and works on floats too)
IntegerDivide,
+ /// Muliplication (only works on numeric types)
Multiply,
+ /// Substraction (only works on numeric types)
Subtract,
}
diff --git a/src/lib/parser/lexers/assignments/primitive.rs b/src/lib/parser/lexers/assignments/primitive.rs
index e98c471e..4358e936 100644
--- a/src/lib/parser/lexers/assignments/primitive.rs
+++ b/src/lib/parser/lexers/assignments/primitive.rs
@@ -3,16 +3,27 @@ use std::fmt::{self, Display, Formatter};
/// A primitive defines the type that a requested value should satisfy.
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
pub enum Primitive {
+ /// A plain string (ex: `"a string"`)
Str,
+ /// An array of string (ex: `["-" b c d]`)
StrArray,
+ /// A true-false value
Boolean,
+ /// An array of booleans
BooleanArray,
+ /// An integer numeric type
Integer,
+ /// An array of integer numeric type
IntegerArray,
+ /// A floating-point value
Float,
+ /// A floating-point value array
FloatArray,
+ /// A hash map
HashMap(Box<Primitive>),
+ /// A btreemap
BTreeMap(Box<Primitive>),
+ /// An index variable (ex: `$array[0]`)
Indexed(String, Box<Primitive>),
}
diff --git a/src/lib/parser/lexers/mod.rs b/src/lib/parser/lexers/mod.rs
index b7c92a97..406d3f86 100644
--- a/src/lib/parser/lexers/mod.rs
+++ b/src/lib/parser/lexers/mod.rs
@@ -1,4 +1,6 @@
+/// Check functions & methods arguments
pub mod arguments;
+/// Check assignements
pub mod assignments;
pub use self::{arguments::*, assignments::*};
diff --git a/src/lib/parser/mod.rs b/src/lib/parser/mod.rs
index 943e9d0f..59a52745 100644
--- a/src/lib/parser/mod.rs
+++ b/src/lib/parser/mod.rs
@@ -1,4 +1,6 @@
+/// The terminal tokens associated with the parsing process
pub mod lexers;
+/// Parse the pipelines to a Pipeline struct
pub mod pipelines;
mod quotes;
mod statement;
diff --git a/src/lib/parser/pipelines.rs b/src/lib/parser/pipelines.rs
index d1f429fc..6c3be406 100644
--- a/src/lib/parser/pipelines.rs
+++ b/src/lib/parser/pipelines.rs
@@ -11,25 +11,33 @@ use crate::{
const ARG_DEFAULT_SIZE: usize = 10;
+/// An error produced during pipeline parsing
#[derive(Debug, Error)]
pub enum PipelineParsingError {
// redirections
+ /// No file was provided after the redirection output
#[error(display = "expected file argument after redirection for output")]
NoRedirection,
+ /// Heredocs are deprecated and were used
#[error(display = "heredocs are not a part of Ion. Use redirection and/or cat instead")]
HeredocsDeprecated,
+ /// No string was given to the herestring
#[error(display = "expected string argument after '<<<'")]
NoHereStringArg,
+ /// No file was provided after the input redirection
#[error(display = "expected file argument after redirection for input")]
NoRedirectionArg,
// quotes
+ /// Unterminated double quotes
#[error(display = "unterminated double quote")]
UnterminatedDoubleQuote,
+ /// Unterminated single quotes
#[error(display = "unterminated single quote")]
UnterminatedSingleQuote,
// paired
+ /// Error with paired tokens (parens, brackets & braces)
#[error(display = "{}", _0)]
Paired(#[error(cause)] LevelsError),
}
@@ -65,7 +73,8 @@ impl<'a> AddItem<'a> for Pipeline<'a> {
}
}
-#[derive(Debug)]
+/// Collect pipelines in the input
+#[derive(Debug, Clone)]
pub struct Collector<'a> {
data: &'a str,
}
@@ -107,7 +116,7 @@ impl<'a> Collector<'a> {
.map(|file| outputs.push(Redirection { from, file: file.into(), append }))
}
- pub fn parse<'builtins>(
+ fn parse<'builtins>(
&self,
builtins: &BuiltinMap<'builtins>,
) -> Result<Pipeline<'builtins>, PipelineParsingError> {
@@ -264,7 +273,7 @@ impl<'a> Collector<'a> {
bytes.next();
}
b')' => {
- levels.down(Field::Proc);
+ levels.down(Field::Proc)?;
bytes.next();
}
b'[' => {
@@ -273,7 +282,7 @@ impl<'a> Collector<'a> {
bytes.next();
}
b']' => {
- levels.down(Field::Array);
+ levels.down(Field::Array)?;
if array_brace_counter % 2 == 1 {
array_brace_counter = (array_brace_counter - 1) / 2;
bytes.next();
@@ -288,7 +297,7 @@ impl<'a> Collector<'a> {
}
b'}' => {
if array_brace_counter % 2 == 0 {
- levels.down(Field::Braces);
+ levels.down(Field::Braces)?;
array_brace_counter /= 2;
bytes.next();
} else {
@@ -404,6 +413,7 @@ impl<'a> Collector<'a> {
}
}
+ /// Collect a pipeline on the given data
pub fn run<'builtins>(
data: &'a str,
builtins: &BuiltinMap<'builtins>,
@@ -411,7 +421,7 @@ impl<'a> Collector<'a> {
Collector::new(data).parse(builtins)
}
- pub fn new(data: &'a str) -> Self { Collector { data } }
+ fn new(data: &'a str) -> Self { Collector { data } }
}
#[cfg(test)]
diff --git a/src/lib/parser/quotes.rs b/src/lib/parser/quotes.rs
index 171b57c8..3bf692a1 100644
--- a/src/lib/parser/quotes.rs
+++ b/src/lib/parser/quotes.rs
@@ -1,6 +1,6 @@
use std::{iter::Peekable, str};
-#[derive(Debug, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum Quotes {
Single,
Double,
@@ -9,8 +9,6 @@ enum Quotes {
/// Serves as a buffer for storing a string until that string can be terminated.
///
-/// # Examples
-///
/// This example comes from the shell's REPL, which ensures that the user's input
/// will only be submitted for execution once a terminated command is supplied.
#[derive(Debug)]
@@ -185,6 +183,7 @@ impl<I: Iterator<Item = u8>> Terminator<I> {
}
}
+ /// Create a new reader on the provided input
pub fn new(inner: I) -> Terminator<I> {
Terminator {
inner: RearPeekable { iter: inner.peekable(), now: None, last: None },
diff --git a/src/lib/parser/statement/parse.rs b/src/lib/parser/statement/parse.rs
index ac3c9d83..2603eb19 100644
--- a/src/lib/parser/statement/parse.rs
+++ b/src/lib/parser/statement/parse.rs
@@ -16,34 +16,45 @@ use crate::{
use err_derive::Error;
use std::char;
+/// Check if the given name is valid for functions, aliases & variables
pub fn is_valid_name(name: &str) -> bool {
let mut chars = name.chars();
chars.next().map_or(false, |b| char::is_alphabetic(b) || b == '_')
&& chars.all(|b| b.is_alphanumeric() || b == '_')
}
+/// An Error occured during parsing
#[derive(Debug, Error)]
pub enum ParseError {
+ /// The blocks were in a wrong order
#[error(display = "incomplete control flow statement")]
IncompleteFlowControl,
+ /// No keys were supplied for assignment
#[error(display = "no key supplied for assignment")]
NoKeySupplied,
+ /// No operator was supplied for assignment
#[error(display = "no operator supplied for assignment")]
NoOperatorSupplied,
+ /// No value supplied for assignment
#[error(display = "no values supplied for assignment")]
NoValueSupplied,
+ /// No value given for iteration in a for loop
#[error(display = "no value supplied for iteration in for loop")]
NoInKeyword,
+ /// Error with match statements
#[error(display = "case error: {}", _0)]
CaseError(#[error(cause)] CaseError),
+ /// The provided function name was invalid
#[error(
- display = "'{}' is not a valid function name\n Function names may only contain \
- alphanumeric characters",
+ display = "'{}' is not a valid function name
+ Function names may only contain alphanumeric characters",
_0
)]
InvalidFunctionName(String),
+ /// The arguments did not match the function's signature
#[error(display = "function argument error: {}", _0)]
InvalidFunctionArgument(#[error(cause)] FunctionParseError),
+ /// Error occured during parsing of a pipeline
#[error(display = "{}", _0)]
PipelineParsingError(#[error(cause)] PipelineParsingError),
}
diff --git a/src/lib/parser/statement/splitter.rs b/src/lib/parser/statement/splitter.rs
index e88d5d02..519681bb 100644
--- a/src/lib/parser/statement/splitter.rs
+++ b/src/lib/parser/statement/splitter.rs
@@ -11,22 +11,31 @@ enum LogicalOp {
None,
}
+/// Error during statement parsing
#[derive(Debug, PartialEq, Error)]
pub enum StatementError {
+ /// The command name is illegal
#[error(display = "illegal command name: {}", _0)]
IllegalCommandName(String),
+ /// Invalid character found
#[error(display = "syntax error: '{}' at position {} is out of place", _0, _1)]
InvalidCharacter(char, usize),
+ /// Unterminated subshell
#[error(display = "syntax error: unterminated subshell")]
UnterminatedSubshell,
+ /// Unterminated namespaced variable
#[error(display = "syntax error: unterminated brace")]
UnterminatedBracedVar,
+ /// Unterminated brace expansion
#[error(display = "syntax error: unterminated braced var")]
UnterminatedBrace,
+ /// Unterminated method
#[error(display = "syntax error: unterminated method")]
UnterminatedMethod,
+ /// Unterminated arithmetic expression
#[error(display = "syntax error: unterminated arithmetic subexpression")]
UnterminatedArithmetic,
+ /// Expected command but found ...
#[error(display = "expected command, but found {}", _0)]
ExpectedCommandButFound(&'static str),
}
@@ -38,6 +47,7 @@ pub enum StatementVariant<'a> {
Default(&'a str),
}
+/// Split an input data into a set of statements
#[derive(Debug)]
pub struct StatementSplitter<'a> {
data: &'a str,
@@ -53,6 +63,7 @@ pub struct StatementSplitter<'a> {
}
impl<'a> StatementSplitter<'a> {
+ /// Create a new statement splitter on data
pub fn new(data: &'a str) -> Self {
StatementSplitter {
data,
diff --git a/src/lib/types.rs b/src/lib/types.rs
index a6488dda..6334645c 100644
--- a/src/lib/types.rs
+++ b/src/lib/types.rs
@@ -3,6 +3,7 @@ use smallvec::SmallVec;
pub use types_rs::types::*;
pub use crate::shell::flow_control::Function;
+/// A owned version of a set of arguments for spawning a command
pub type Args = SmallVec<[small::String; 4]>;
/// Construct a new Array containing the given arguments
///
--
GitLab