Lisi - LIterate Scripting Interpreter

use asciidoctrine::*;

asciidoctrine = { path = "../asciidoctrine", version = "0.1" }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
<<required_crates>>

<<internal_modules>>

<<crate_usages>>

<<internal_structs|join="\n\n">>

pub struct Lisi {
  <<lisi_internal_variables>>
}

impl Lisi {
  pub fn new() -> Self {
    Lisi {
      <<lisi_init_variables>>
    }
  }

  <<internal_functions|join="\n\n">>

  /// Gets all snippets from the ast
  pub fn extract_ast(&mut self, input: &AST) -> Result<SnippetDB, Error> { (3)
    let snippets = SnippetDB::new();

    // extract snippets from all inner elements
    input.elements.iter().try_fold(snippets, |snippets, element| {
      self.extract(snippets, element)
    })
  }

  /// Build all snippets (Runs the vm)
  pub fn generate_outputs(&mut self, snippets: SnippetDB, ast: &AST) -> Result<(), Error> { (4)
    <<generate_outputs>>

    Ok(())
  }
}

impl Extension for Lisi { (1)
  fn transform<'a>(&mut self, input: AST<'a>) -> anyhow::Result<AST<'a>> { (2)
    let snippets = self.extract_ast(&input)?; (3)

    self.calculate_snippet_ordering(&snippets); (5)

    self.generate_outputs(snippets, &input)?; (4)

    Ok(input)
  }
}

let content = input
  .get_attribute("content")
  .unwrap_or(input.content);

let content = input
  .get_attribute("content")
  .unwrap_or(input.content);

#[macro_use]
extern crate pest_derive;

pest = "2.1.0"
pest_derive = "2.1.0"

mod codeblock_parser;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
use pest::Parser;

use crate::*;

#[derive(Parser, Debug)]
#[grammar = "codeblock.pest"]
pub struct CodeblockParser;

<<codeblock_parser_internal_structs>>

<<codeblock_parser_functions|join="\n\n">>

/// Extracts the ids of used snippets from a depending snippet
pub fn get_dependencies(input: &str) -> Vec<String> {
  <<get_dependencies>>
}

/// Merges the snippets into the depending snippet
pub fn merge_dependencies(input: &str, snippets: &SnippetDB, key: &str) -> String {
  <<merge_dependencies>>
}

1
2
3
4
5
6
7
8
codeblock = _{ (code | indented_reference | reference | comment)* ~ EOI }

reference = { <<reference>> }
indented_reference = { <<indented_reference>> }
code = { <<code_gramma>> }
comment = { <<comment>> }

<<internal_gramma_elements>>

"<<" ~ identifier ~ (empty ~ "|" ~ empty ~ attributes)? ~ ">>"

identifier = @{ (ASCII_ALPHANUMERIC | "_") ~ (ASCII_ALPHANUMERIC | "_" | "-" )* }

attributes = { (attribute | attribute_param) ~ empty
  ~ ("," ~ empty ~ (attribute | attribute_param) ~ empty)* }
attribute = { identifier ~ "=" ~ "\"" ~ value ~ "\"" }
attribute_param = { identifier ~ empty ~ ":=" ~ empty
  ~ (("\"" ~ value ~ "\"") | reference) }
value = @{ ( !"\"" ~ ANY | "\\\"")* }

#[derive(Debug, Clone)]
enum ReferenceParam {
  Value(String),
  Reference(String, String),
}

type SnippetParams = Vec<HashMap<String, ReferenceParam>>;

(SOI | NEWLINE) ~ indentation ~ reference

whitespace = @{ (" " | "\t") }
indentation = @{ whitespace+ }
empty = @{ (" " | "\t" | "\n" | "\r")* }

(!indented_reference ~ !reference ~ !comment ~ ANY)+

optspaces ~ ("//" | "#" | ";;" ) ~ optspaces ~ "<" ~ ASCII_DIGIT+ ~ ">" ~ optspaces ~ &(EOI | NEWLINE)

optspaces = @{ whitespace* }

if snippet.children.len() > 0 {
  let mut children = Vec::new();
  for mut child in snippet.children.into_iter() {
    let content = child.content;
    let content = codeblock_parser::merge_dependencies(content.as_str(), &snippets, key);
    child.content = content;
    children.push(child);
  }
  snippet.children = children;
} else {
  let content = snippet.content;
  let content = codeblock_parser::merge_dependencies(content.as_str(), &snippets, key);
  snippet.content = content;
}

let ast = CodeblockParser::parse(Rule::codeblock, input).expect("couldn't parse input.");
let snippet_params = extract_snippet_params(Vec::from([HashMap::default()]), input);

merge_dependencies_inner(ast, snippets, snippet_params, key)

fn merge_dependencies_inner<'a>(
  ast: pest::iterators::Pairs<'a, codeblock_parser::Rule>,
  snippets: &SnippetDB,
  snippet_params: SnippetParams,
  key: &str,
) -> String {
  let mut output = String::new();

  for element in ast {
    match element.as_rule() {
      Rule::reference => {
        <<get_modified_snippet>>
      }
      Rule::indented_reference => {
        let mut indented_output = String::new();
        let indentation = extract_indentation(&element);
        <<get_modified_snippet|
            output_variable := <<indented_output>>>>
        indent(&indented_output, indentation, &mut output); (2)
      }
      Rule::code => {
        output.push_str(element.as_str());
      }
      _ => (),
    }
  }
  output
}

let key = element.as_str().trim_start();
let identifier = extract_identifier(&element);
let join_str = extract_join_str(&element).replace("\\n", "\n");

substitude_params(
  identifier,
  snippets,
  snippet_params.clone(),
  &join_str,
  key,
  &mut <<output_variable>>,
);

fn substitude_params(
  identifier: &str,
  snippets: &SnippetDB,
  snippet_params_history: SnippetParams,
  join_str: &str,
  key: &str,
  output: &mut String,
) {
  let mut snippet_params_history = snippet_params_history;
  let snippet_params = snippet_params_history.pop().unwrap_or_default();

  match snippet_params.get(identifier) {
    Some(param) => match param {
      ReferenceParam::Value(param) => output.push_str(&param),
      ReferenceParam::Reference(param, subparams) => {
        <<use_snippet_parameter>>
      }
    }
    None => match snippets.get(identifier) {
      <<use_snippet_from_database>>
    }
  }
}

if param != identifier {
  snippet_params_history.push(snippet_params.clone());
}
substitude_params(
  &param,
  snippets,
  snippet_params_history,
  join_str,
  &subparams, (1)
  output,
);

Some(snippet) => {
  if let SnippetType::Pipe = snippet.kind {
    warn!("depends on pipe snippet {}", identifier);
  }

  let input = snippet.get_raw_content(&join_str);

  let content = if snippet.raw {
    input
  } else {
    let ast = CodeblockParser::parse(Rule::codeblock, &input).expect("couldn't parse input.");

    snippet_params_history.push(snippet_params);
    let snippet_params = extract_snippet_params(snippet_params_history, key);

    merge_dependencies_inner(ast, snippets, snippet_params, key)
  };
  output.push_str(&content);
}
None => {
  snippet_params_history.pop(); (1)
  if let Some(params) = snippet_params_history.pop() {
    if params.get(identifier).is_some() {
      snippet_params_history.push(params);
      substitude_params(
        identifier,
        snippets,
        snippet_params_history,
        join_str,
        key,
        output,
      );
    } else {
      warn!(
        "Couldn't find snippet dependency `{}` for `{}`",
        identifier, key
      );
    }
  } else {
    warn!(
      "Couldn't find snippet dependency `{}` for `{}`",
      identifier, key
    );
  }
}

fn extract_identifier<'a>(element: &pest::iterators::Pair<'a, codeblock_parser::Rule>) -> &'a str {
  match element.as_rule() {
    Rule::reference => element.clone().into_inner().next().unwrap().as_str(),
    Rule::indented_reference => {
      let mut output = "";
      for element in element.clone().into_inner() {
        match element.as_rule() {
          Rule::reference => {
            output = element.into_inner().next().unwrap().as_str();
            break;
          }
          _ => (),
        }
      }
      output
    }
    _ => "",
  }
}

fn extract_join_str<'a>(element: &pest::iterators::Pair<'a, codeblock_parser::Rule>) -> &'a str {
  match element.as_rule() {
    Rule::reference => {
      match element
        .clone()
        .into_inner()
        .find(|element| match element.as_rule() {
          Rule::attributes => true,
          _ => false,
        }) {
        Some(element) => extract_join_str(&element),
        None => "\n",
      }
    }
    Rule::attributes => {
      match element
        .clone()
        .into_inner()
        .find(|element| match element.as_rule() {
          Rule::attribute => {
            let mut attribute = element.clone().into_inner();
            let key = attribute.next().unwrap();

            key.as_str() == "join"
          }
          _ => false,
        }) {
        Some(element) => {
          let mut attribute = element.clone().into_inner();
          attribute.next();
          let value = attribute.next().unwrap();

          value.as_str()
        }
        None => "\n",
      }
    }
    Rule::indented_reference => {
      match element
        .clone()
        .into_inner()
        .find(|element| match element.as_rule() {
          Rule::reference => true,
          _ => false,
        }) {
        Some(element) => extract_join_str(&element),
        None => "\n",
      }
    }
    _ => "\n",
  }
}

fn extract_indentation<'a>(element: &pest::iterators::Pair<'a, codeblock_parser::Rule>) -> &'a str {
  let mut output = "";
  for element in element.clone().into_inner() {
    match element.as_rule() {
      Rule::indentation => {
        output = element.as_str();
        break;
      }
      _ => (),
    }
  }
  output
}

fn indent(content: &str, indentation: &str, output: &mut String) -> () {
  for line in content.lines() {
    output.push_str("\n");
    output.push_str(indentation);
    output.push_str(line);
  }
}

fn extract_snippet_params(snippet_params_history: SnippetParams, param: &str) -> SnippetParams {
  let mut snippet_params = snippet_params_history.clone().pop().unwrap_or_default();
  let mut new_params = HashMap::default();
  let ast = CodeblockParser::parse(Rule::codeblock, &param).expect("couldn't parse input.");
  let mut snippet_params_history = snippet_params_history;

  let ref_iter = ast.clone().filter(|element| match element.as_rule() {
    Rule::reference => true,
    _ => false,
  });
  let indent_ref_iter = ast
    .clone()
    .filter(|element| match element.as_rule() {
      Rule::indented_reference => true,
      _ => false,
    })
    .flat_map(|element| element.clone().into_inner())
    .filter(|element| match element.as_rule() {
      Rule::reference => true,
      _ => false,
    });

  for element in ref_iter.chain(indent_ref_iter) {
    for element in element
      .clone()
      .into_inner()
      .filter(|element| match element.as_rule() {
        Rule::attributes => true,
        _ => false,
      })
      .flat_map(|element| element.clone().into_inner())
      .filter(|element| match element.as_rule() {
        Rule::attribute_param => true,
        _ => false,
      })
    {
      let identifier = element
        .clone()
        .into_inner()
        .find(|element| match element.as_rule() {
          Rule::identifier => true,
          _ => false,
        })
        .map(|element| element.as_str().to_string())
        .unwrap();
      let value = element
        .into_inner()
        .find(|element| match element.as_rule() {
          Rule::value => true,
          Rule::reference => true,
          _ => false,
        })
        .map(|element| match element.as_rule() {
          Rule::value => Some(ReferenceParam::Value(element.as_str().to_string())),
          Rule::reference => match snippet_params.remove(&identifier) {
            Some(param) => Some(param),
            None => {
              let inner_params = element.as_str().to_string();
              let identifier = extract_identifier(&element);

              match snippet_params.remove(identifier) {
                Some(param) => Some(param),
                None => Some(ReferenceParam::Reference(
                  identifier.to_string(),
                  inner_params,
                )),
              }
            }
          },
          _ => None,
        })
        .unwrap();

      if let Some(value) = value {
        new_params.insert(identifier, value);
      }
    }
  }

  snippet_params_history.push(new_params);
  snippet_params_history
}

fn get_raw_content(&self, join_str: &str) -> String {
  if self.children.len() > 0 {
    let mut iter = self.children.iter();
    let start = iter.next().unwrap().raw_content.clone();
    iter.fold(start, |mut base, snippet| {
      base.push_str(join_str);
      base.push_str(&snippet.raw_content);
      base
    })
  } else {
    self.raw_content.to_string()
  }
}

topological-sort = "0.2"

use topological_sort::TopologicalSort;

dependencies: TopologicalSort<String>,

dependencies: TopologicalSort::new(),

/// Builds the dependency tree for topological sorting
pub fn calculate_snippet_ordering(&mut self, snippets: &SnippetDB) {
  for (key, snippet) in snippets.iter() {
    // TODO Vielleicht sollten nur `save` und `eval` snippets
    // unabhängig von dependencies aufgenommen werden?
    self.dependencies.insert(key); (1)

    for child in snippet.children.iter() { (2)
      for dependency in child.depends_on.iter() {
        self.dependencies.add_dependency(dependency, key);
      }
    }
    for dependency in snippet.depends_on.iter() { (2)
      self.dependencies.add_dependency(dependency, key);
    }
  }
}

let mut dependencies = Vec::new();
for dependency in codeblock_parser::get_dependencies(content).iter() {
  dependencies.push(dependency.to_string());
}

let mut depends_on_ids = Vec::new();

let ast = CodeblockParser::parse(Rule::codeblock, input).expect("couldn't parse input.");

for element in ast {
  match element.as_rule() {
    Rule::reference => {
      depends_on_ids.push(extract_identifier(&element).to_string());
      <<get_dependencies_of_parameters>>
    }
    Rule::indented_reference => {
      depends_on_ids.push(extract_identifier(&element).to_string());
      <<get_dependencies_of_parameters>>
    }
    _ => (),
  }
}

depends_on_ids

let params = extract_snippet_params(Vec::default(), element.as_str())
  .pop()
  .unwrap_or_default();
for param in params.into_values() {
  if let ReferenceParam::Reference(identifier, _) = param {
    depends_on_ids.push(identifier.clone());
  }
}

let source = ast.get_attribute("source").unwrap_or("");
let db = Rc::new(RefCell::new(snippets));
let snippets = Rc::clone(&db);

loop {
  let key = self.dependencies.pop(); (1)
  let snippet = match &key {
    Some(key) => {
      let mut snippets = snippets.borrow_mut();
      let snippet = snippets.pop(&key);

      match snippet {
        Some(mut snippet) => {
          if !snippet.raw {
            <<merge_snippet_content>>
          };

          snippets.store(key.to_string(), snippet.clone());
          Some(snippet)
        }
        None => { (4)
          // TODO Fehlermeldung im AST. Ein Snippet sollte zu
          // diesem Zeitpunkt immer bereits erstellt sein.
          warn!("{}: Dependency `{}` nicht gefunden", source, key);
          None
        }
      }
    }
    None => { (2)
      if !self.dependencies.is_empty() { (3)
        error!(
          "Es ist ein Ring in den Abhängigkeiten ({:#?})",
          self.dependencies
        );
      }
      break; (2)
    }
  };

  if let Some(snippet) = snippet {
    <<execute_snippet_action>>
  }
}

match &snippet.kind {
  SnippetType::Eval(interpreter) => {
    self.eval(interpreter.to_string(), snippet.content)?;
  }
  SnippetType::Plain => {}
  SnippetType::Save(path) => {
    <<get_filepath>>
    self.save(path, &snippet.content)?;
  }
  SnippetType::Pipe => {
    self.pipe(&snippet.content, &db)?;
  }
}