From b0c437afceb3465eecbf658077bfc1b74d611c1e Mon Sep 17 00:00:00 2001
From: Hunter Goldstein <hunter.d.goldstein@gmail.com>
Date: Thu, 13 Jul 2017 14:13:28 -0400
Subject: [PATCH] Implement Herestrings (#405)
---
examples/herestring.ion | 14 ++++++++
examples/herestring.out | 3 ++
src/parser/peg.rs | 40 +++++++++++++++++-----
src/parser/pipelines.rs | 75 ++++++++++++++++++++++++++++-------------
src/shell/pipe.rs | 68 ++++++++++++++++++++++++++++++-------
5 files changed, 156 insertions(+), 44 deletions(-)
create mode 100644 examples/herestring.ion
create mode 100644 examples/herestring.out
diff --git a/examples/herestring.ion b/examples/herestring.ion
new file mode 100644
index 00000000..55b988e2
--- /dev/null
+++ b/examples/herestring.ion
@@ -0,0 +1,14 @@
+echo $(tr '[a-z]' '[A-Z]' <<< foo)
+
+let output = [foo bar baz bing]
+
+fn find elem array
+ if grep -q $elem <<< $array
+ echo "true"
+ else
+ echo "false"
+ end
+end
+
+find "bar" "@output"
+find "zar" "@output"
diff --git a/examples/herestring.out b/examples/herestring.out
new file mode 100644
index 00000000..46bae8fc
--- /dev/null
+++ b/examples/herestring.out
@@ -0,0 +1,3 @@
+FOO
+true
+false
diff --git a/src/parser/peg.rs b/src/parser/peg.rs
index 9f606e18..ce1db239 100644
--- a/src/parser/peg.rs
+++ b/src/parser/peg.rs
@@ -18,15 +18,24 @@ pub struct Redirection {
pub append: bool
}
+/// Represents input that a process could initially receive from `stdin`
+#[derive(Debug, PartialEq, Clone)]
+pub enum Input {
+ /// A file; the contents of said file will be written to the `stdin` of a process
+ File(String),
+ /// A string literal that is written to the `stdin` of a process
+ HereString(String),
+}
+
#[derive(Debug, PartialEq, Clone)]
pub struct Pipeline {
pub jobs: Vec<Job>,
pub stdout: Option<Redirection>,
- pub stdin: Option<Redirection>,
+ pub stdin: Option<Input>,
}
impl Pipeline {
- pub fn new(jobs: Vec<Job>, stdin: Option<Redirection>, stdout: Option<Redirection>) -> Self {
+ pub fn new(jobs: Vec<Job>, stdin: Option<Input>, stdout: Option<Redirection>) -> Self {
Pipeline { jobs, stdin, stdout }
}
@@ -36,9 +45,17 @@ impl Pipeline {
job.expand(&expanders);
}
- if let Some(stdin) = self.stdin.iter_mut().next() {
- stdin.file = expand_string(stdin.file.as_str(), &expanders, false).join(" ");
- }
+ let stdin = match self.stdin {
+ Some(Input::File(ref s)) => {
+ Some(Input::File(expand_string(s, &expanders, false).join(" ")))
+ },
+ Some(Input::HereString(ref s)) => {
+ Some(Input::HereString(expand_string(s, &expanders, true).join(" ")))
+ },
+ None => None
+ };
+
+ self.stdin = stdin;
if let Some(stdout) = self.stdout.iter_mut().next() {
stdout.file = expand_string(stdout.file.as_str(), &expanders, false).join(" ");
@@ -62,9 +79,16 @@ impl fmt::Display for Pipeline {
JobKind::Pipe(RedirectFrom::Both) => tokens.push("&|".into()),
}
}
- if let Some(ref infile) = self.stdin {
- tokens.push("<".into());
- tokens.push(infile.file.clone());
+ match self.stdin {
+ None => (),
+ Some(Input::File(ref file)) => {
+ tokens.push("<".into());
+ tokens.push(file.clone());
+ },
+ Some(Input::HereString(ref string)) => {
+ tokens.push("<<<".into());
+ tokens.push(string.clone());
+ },
}
if let Some(ref outfile) = self.stdout {
match outfile.from {
diff --git a/src/parser/pipelines.rs b/src/parser/pipelines.rs
index e6d444ab..fc2fe2a1 100644
--- a/src/parser/pipelines.rs
+++ b/src/parser/pipelines.rs
@@ -9,7 +9,7 @@
use std::collections::HashSet;
use std::iter::Peekable;
-use parser::peg::{Pipeline, Redirection, RedirectFrom};
+use parser::peg::{Pipeline, Input, Redirection, RedirectFrom};
use shell::{Job, JobKind};
use types::*;
@@ -165,7 +165,7 @@ impl<'a> Collector<'a> {
let mut bytes = self.data.bytes().enumerate().peekable();
let mut args = Array::new();
let mut jobs: Vec<Job> = Vec::new();
- let mut infile: Option<Redirection> = None;
+ let mut input: Option<Input> = None;
let mut outfile: Option<Redirection> = None;
/// Attempt to create a new job given a list of collected arguments
@@ -270,16 +270,22 @@ impl<'a> Collector<'a> {
},
b'<' => {
bytes.next();
- if let Some(file) = self.arg(&mut bytes)? {
- infile = Some(Redirection {
- from: RedirectFrom::Stdout,
- file: file.into(),
- append: false,
- });
+ if Some(b'<') == self.peek(i + 1) && Some(b'<') == self.peek(i + 2) {
+ // If the next two characters are arrows, then interpret
+ // the next argument as a herestring
+ bytes.next();
+ bytes.next();
+ if let Some(cmd) = self.arg(&mut bytes)? {
+ input = Some(Input::HereString(cmd.into()));
+ } else {
+ return Err("expected string argument after '<<<'");
+ }
+ } else if let Some(file) = self.arg(&mut bytes)? {
+ // Otherwise interpret it as stdin redirection
+ input = Some(Input::File(file.into()));
} else {
return Err("expected file argument after redirection for input");
}
-
}
// Skip over whitespace between jobs
b' ' | b'\t' => {
@@ -294,7 +300,7 @@ impl<'a> Collector<'a> {
jobs.push(Job::new(args, JobKind::Last));
}
- Ok(Pipeline::new(jobs, infile, outfile))
+ Ok(Pipeline::new(jobs, input, outfile))
}
}
@@ -302,7 +308,7 @@ impl<'a> Collector<'a> {
#[cfg(test)]
mod tests {
use shell::flow_control::Statement;
- use parser::peg::{parse, Pipeline, RedirectFrom, Redirection};
+ use parser::peg::{parse, Input, Pipeline, RedirectFrom, Redirection};
use shell::{Job, JobKind};
use types::Array;
@@ -631,7 +637,7 @@ mod tests {
assert_eq!("echo", &pipeline.clone().jobs[1].args[0]);
assert_eq!("hello", &pipeline.clone().jobs[1].args[1]);
assert_eq!("cat", &pipeline.clone().jobs[2].args[0]);
- assert_eq!("stuff", &pipeline.clone().stdin.unwrap().file);
+ assert_eq!(Some(Input::File("stuff".into())), pipeline.stdin);
assert_eq!("other", &pipeline.clone().stdout.unwrap().file);
assert!(!pipeline.clone().stdout.unwrap().append);
assert_eq!(input.to_owned(), pipeline.to_string());
@@ -644,7 +650,7 @@ mod tests {
fn pipeline_with_redirection_append() {
if let Statement::Pipeline(pipeline) = parse("cat | echo hello | cat < stuff >> other") {
assert_eq!(3, pipeline.jobs.len());
- assert_eq!("stuff", &pipeline.clone().stdin.unwrap().file);
+ assert_eq!(Some(Input::File("stuff".into())), pipeline.stdin);
assert_eq!("other", &pipeline.clone().stdout.unwrap().file);
assert!(pipeline.clone().stdout.unwrap().append);
} else {
@@ -661,11 +667,7 @@ mod tests {
Job::new(array!["echo", "hello"], JobKind::Pipe(RedirectFrom::Stdout)),
Job::new(array!["cat"], JobKind::Last)
],
- stdin: Some(Redirection {
- from: RedirectFrom::Stdout,
- file: "stuff".into(),
- append: false
- }),
+ stdin: Some(Input::File("stuff".into())),
stdout: Some(Redirection {
from: RedirectFrom::Stderr,
file: "other".into(),
@@ -684,11 +686,7 @@ mod tests {
Job::new(array!["echo", "hello"], JobKind::Pipe(RedirectFrom::Stdout)),
Job::new(array!["cat"], JobKind::Last)
],
- stdin: Some(Redirection {
- from: RedirectFrom::Stdout,
- file: "stuff".into(),
- append: false
- }),
+ stdin: Some(Input::File("stuff".into())),
stdout: Some(Redirection {
from: RedirectFrom::Both,
file: "other".into(),
@@ -702,7 +700,7 @@ mod tests {
fn pipeline_with_redirection_reverse_order() {
if let Statement::Pipeline(pipeline) = parse("cat | echo hello | cat > stuff < other") {
assert_eq!(3, pipeline.jobs.len());
- assert_eq!("other", &pipeline.clone().stdin.unwrap().file);
+ assert_eq!(Some(Input::File("other".into())), pipeline.stdin);
assert_eq!("stuff", &pipeline.clone().stdout.unwrap().file);
} else {
assert!(false);
@@ -731,6 +729,35 @@ mod tests {
}
}
+ #[test]
+ fn herestring() {
+ let input = "calc <<< $(cat math.txt)";
+ let expected = Pipeline {
+ jobs: vec![Job::new(array!["calc"], JobKind::Last)],
+ stdin: Some(Input::HereString("$(cat math.txt)".into())),
+ stdout: None,
+ };
+ assert_eq!(Statement::Pipeline(expected), parse(input));
+ }
+
+ #[test]
+ fn piped_herestring() {
+ let input = "cat | tr 'o' 'x' <<< $VAR > out.log";
+ let expected = Pipeline {
+ jobs: vec![
+ Job::new(array!["cat"], JobKind::Pipe(RedirectFrom::Stdout)),
+ Job::new(array!["tr", "'o'", "'x'"], JobKind::Last)
+ ],
+ stdin: Some(Input::HereString("$VAR".into())),
+ stdout: Some(Redirection {
+ from: RedirectFrom::Stdout,
+ file: "out.log".into(),
+ append: false
+ })
+ };
+ assert_eq!(Statement::Pipeline(expected), parse(input));
+ }
+
#[test]
fn awk_tests() {
if let Statement::Pipeline(pipeline) = parse("awk -v x=$x '{ if (1) print $1 }' myfile") {
diff --git a/src/shell/pipe.rs b/src/shell/pipe.rs
index 99c8579c..281f72c1 100644
--- a/src/shell/pipe.rs
+++ b/src/shell/pipe.rs
@@ -11,7 +11,7 @@ use super::job_control::JobControl;
use super::{JobKind, Shell};
use super::status::*;
use super::signals::{self, SignalHandler};
-use parser::peg::{Pipeline, RedirectFrom};
+use parser::peg::{Pipeline, Input, RedirectFrom};
use self::crossplat::*;
/// The `crossplat` module contains components that are meant to be abstracted across
@@ -21,7 +21,7 @@ pub mod crossplat {
use nix::{fcntl, unistd};
use parser::peg::{RedirectFrom};
use std::fs::File;
- use std::io::Error;
+ use std::io::{Write, Error};
use std::os::unix::io::{IntoRawFd, FromRawFd};
use std::process::{Stdio, Command};
@@ -34,6 +34,21 @@ pub mod crossplat {
unistd::getpid() as u32
}
+ /// Create an instance of Stdio from a byte slice that will echo the
+ /// contents of the slice when read. This can be called with owned or
+ /// borrowed strings
+ pub unsafe fn stdin_of<T: AsRef<[u8]>>(input: T) -> Result<Stdio, Error> {
+ let (reader, writer) = unistd::pipe2(fcntl::O_CLOEXEC)?;
+ let mut infile = File::from_raw_fd(writer);
+ // Write the contents; make sure to use write_all so that we block until
+ // the entire string is written
+ infile.write_all(input.as_ref())?;
+ infile.flush()?;
+ // `infile` currently owns the writer end RawFd. If we just return the reader end
+ // and let `infile` go out of scope, it will be closed, sending EOF to the reader!
+ Ok(Stdio::from_raw_fd(reader))
+ }
+
/// Set up pipes such that the relevant output of parent is sent to the stdin of child.
/// The content that is sent depends on `mode`
pub unsafe fn create_pipe (
@@ -68,7 +83,7 @@ pub mod crossplat {
pub mod crossplat {
use parser::peg::{RedirectFrom};
use std::fs::File;
- use std::io;
+ use std::io::{self, Write};
use std::os::unix::io::{IntoRawFd, FromRawFd};
use std::process::{Stdio, Command};
use syscall;
@@ -95,6 +110,19 @@ pub mod crossplat {
fn from(data: syscall::Error) -> Error { Error::Sys(data) }
}
+ pub unsafe fn stdin_of<T: AsRef<[u8]>>(input: T) -> Result<Stdio, Error> {
+ let mut fds: [usize; 2] = [0; 2];
+ syscall::call::pipe2(&mut fds, syscall::flag::O_CLOEXEC)?;
+ let (reader, writer) = (fds[0], fds[1]);
+ let infile = File::from_raw_fd(writer);
+ // Write the contents; make sure to use write_all so that we block until
+ // the entire string is written
+ infile.write_all(input.as_ref())?;
+ // `infile` currently owns the writer end RawFd. If we just return the reader end
+ // and let `infile` go out of scope, it will be closed, sending EOF to the reader!
+ Ok(Stdio::from_raw_fd(reader))
+ }
+
/// Set up pipes such that the relevant output of parent is sent to the stdin of child.
/// The content that is sent depends on `mode`
pub unsafe fn create_pipe (
@@ -159,15 +187,31 @@ impl<'a> PipelineExecution for Shell<'a> {
.drain(..).map(|mut job| {
(job.build_command(), job.kind)
}).collect();
-
- if let Some(ref stdin) = pipeline.stdin {
- if let Some(command) = piped_commands.first_mut() {
- match File::open(&stdin.file) {
- Ok(file) => unsafe { command.0.stdin(Stdio::from_raw_fd(file.into_raw_fd())); },
- Err(err) => {
- let stderr = io::stderr();
- let mut stderr = stderr.lock();
- let _ = writeln!(stderr, "ion: failed to redirect stdin into {}: {}", stdin.file, err);
+ match pipeline.stdin {
+ None => (),
+ Some(Input::File(ref filename)) => {
+ if let Some(command) = piped_commands.first_mut() {
+ match File::open(filename) {
+ Ok(file) => unsafe {
+ command.0.stdin(Stdio::from_raw_fd(file.into_raw_fd()));
+ },
+ Err(e) => {
+ eprintln!("ion: failed to redirect '{}' into stdin: {}", filename, e);
+ }
+ }
+ }
+ },
+ Some(Input::HereString(ref mut string)) => {
+ if let Some(command) = piped_commands.first_mut() {
+ if !string.ends_with('\n') { string.push('\n'); }
+ match unsafe { crossplat::stdin_of(&string) } {
+ Ok(stdio) => {
+ command.0.stdin(stdio);
+ },
+ Err(e) => {
+ eprintln!("ion: failed to redirect herestring '{}' into stdin: {}",
+ string, e);
+ }
}
}
}
--
GitLab