aitzol
/
Plume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'add support for right to left languages in post content' (#853) from bidi-md into main 

Reviewed-on: https://git.joinplu.me/Plume/Plume/pulls/853
Reviewed-by: KitaitiMakoto <kitaitimakoto@noreply@joinplu.me>
This commit is contained in:
KitaitiMakoto 2020-12-28 12:22:24 +00:00
parent 80675f85f3 6187c62cb3
commit 11acc4172c
3 changed files with 50 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@ -2600,7 +2600,7 @@ dependencies = [
"hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl 0.10.30 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.30 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.8.0 (git+https://git.joinplu.me/Plume/pulldown-cmark?branch=bidi-plume)",
"regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2770,10 +2770,12 @@ dependencies = [
[[package]] [[package]]
name = "pulldown-cmark" name = "pulldown-cmark"
version = "0.2.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://git.joinplu.me/Plume/pulldown-cmark?branch=bidi-plume#f485d6bf18921277f493e1300e160b07e3756c3b"
dependencies = [ dependencies = [
"bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -4883,7 +4885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
"checksum proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" "checksum proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
"checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" "checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b"
"checksum pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eef52fac62d0ea7b9b4dc7da092aa64ea7ec3d90af6679422d3d7e0e14b6ee15" "checksum pulldown-cmark 0.8.0 (git+https://git.joinplu.me/Plume/pulldown-cmark?branch=bidi-plume)" = "<none>"
"checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
"checksum quick-xml 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1d8065cbb01701c11cc195cde85cbf39d1c6a80705b67a157ebb3042e0e5777f" "checksum quick-xml 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1d8065cbb01701c11cc195cde85cbf39d1c6a80705b67a157ebb3042e0e5777f"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"

3
plume-common/Cargo.toml
View File

@ -30,4 +30,5 @@ version = "0.4"
[dependencies.pulldown-cmark] [dependencies.pulldown-cmark]
default-features = false default-features = false
version = "0.2.0" git = "https://git.joinplu.me/Plume/pulldown-cmark"
branch = "bidi-plume"

73
plume-common/src/utils.rs
View File

@ -1,12 +1,11 @@
use heck::CamelCase; use heck::CamelCase;
use openssl::rand::rand_bytes; use openssl::rand::rand_bytes;
use pulldown_cmark::{html, Event, Options, Parser, Tag}; use pulldown_cmark::{html, LinkType, Event, Options, Parser, Tag, CodeBlockKind, CowStr};
use regex_syntax::is_word_character; use regex_syntax::is_word_character;
use rocket::{ use rocket::{
http::uri::Uri, http::uri::Uri,
response::{Flash, Redirect}, response::{Flash, Redirect},
}; };
use std::borrow::Cow;
use std::collections::HashSet; use std::collections::HashSet;
use syntect::html::ClassedHTMLGenerator; use syntect::html::ClassedHTMLGenerator;
use syntect::parsing::SyntaxSet; use syntect::parsing::SyntaxSet;
@ -51,10 +50,10 @@ enum State {
fn to_inline(tag: Tag<'_>) -> Tag<'_> { fn to_inline(tag: Tag<'_>) -> Tag<'_> {
match tag { match tag {
Tag::Header(_) | Tag::Table(_) | Tag::TableHead | Tag::TableRow | Tag::TableCell => { Tag::Heading(_) | Tag::Table(_) | Tag::TableHead | Tag::TableRow | Tag::TableCell => {
Tag::Paragraph Tag::Paragraph
} }
Tag::Image(url, title) => Tag::Link(url, title), Tag::Image(typ, url, title) => Tag::Link(typ, url, title),
t => t, t => t,
} }
} }
@ -66,21 +65,31 @@ fn highlight_code<'a>(
evt: Event<'a>, evt: Event<'a>,
) -> Option<Vec<Event<'a>>> { ) -> Option<Vec<Event<'a>>> {
match evt { match evt {
Event::Start(Tag::CodeBlock(lang)) => { Event::Start(Tag::CodeBlock(kind)) => {
if lang.is_empty() { match &kind {
Some(vec![Event::Start(Tag::CodeBlock(lang))]) CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
} else { *context = Some(HighlighterContext { content: vec![] });
*context = Some(HighlighterContext { content: vec![] }); },
Some(vec![Event::Start(Tag::CodeBlock(lang))]) _ => {}
} }
Some(vec![Event::Start(Tag::CodeBlock(kind))])
} }
Event::End(Tag::CodeBlock(x)) => { Event::End(Tag::CodeBlock(kind)) => {
let mut result = vec![]; let mut result = vec![];
if let Some(ctx) = context.take() { if let Some(ctx) = context.take() {
let lang = if let CodeBlockKind::Fenced(lang) = &kind {
if lang.is_empty() {
unreachable!();
} else {
lang
}
} else {
unreachable!();
};
let syntax_set = SyntaxSet::load_defaults_newlines(); let syntax_set = SyntaxSet::load_defaults_newlines();
let syntax = syntax_set.find_syntax_by_token(&x).unwrap_or_else(|| { let syntax = syntax_set.find_syntax_by_token(&lang).unwrap_or_else(|| {
syntax_set syntax_set
.find_syntax_by_name(&x) .find_syntax_by_name(&lang)
.unwrap_or_else(|| syntax_set.find_syntax_plain_text()) .unwrap_or_else(|| syntax_set.find_syntax_plain_text())
}); });
let mut html = ClassedHTMLGenerator::new(&syntax, &syntax_set); let mut html = ClassedHTMLGenerator::new(&syntax, &syntax_set);
@ -90,7 +99,7 @@ fn highlight_code<'a>(
let q = html.finalize(); let q = html.finalize();
result.push(Event::Html(q.into())); result.push(Event::Html(q.into()));
} }
result.push(Event::End(Tag::CodeBlock(x))); result.push(Event::End(Tag::CodeBlock(kind)));
*context = None; *context = None;
Some(result) Some(result)
} }
@ -113,10 +122,10 @@ fn flatten_text<'a>(state: &mut Option<String>, evt: Event<'a>) -> Option<Vec<Ev
prev_txt.push_str(&txt); prev_txt.push_str(&txt);
(Some(prev_txt), vec![]) (Some(prev_txt), vec![])
} }
None => (Some(txt.into_owned()), vec![]), None => (Some(txt.into_string()), vec![]),
}, },
e => match state.take() { e => match state.take() {
Some(prev) => (None, vec![Event::Text(Cow::Owned(prev)), e]), Some(prev) => (None, vec![Event::Text(CowStr::Boxed(prev.into())), e]),
None => (None, vec![e]), None => (None, vec![e]),
}, },
}; };
@ -156,11 +165,11 @@ fn process_image<'a, 'b>(
) -> Event<'a> { ) -> Event<'a> {
if let Some(ref processor) = *processor { if let Some(ref processor) = *processor {
match evt { match evt {
Event::Start(Tag::Image(id, title)) => { Event::Start(Tag::Image(typ, id, title)) => {
if let Some((url, cw)) = id.parse::<i32>().ok().and_then(processor.as_ref()) { if let Some((url, cw)) = id.parse::<i32>().ok().and_then(processor.as_ref()) {
if let (Some(cw), false) = (cw, inline) { if let (Some(cw), false) = (cw, inline) {
// there is a cw, and where are not inline // there is a cw, and where are not inline
Event::Html(Cow::Owned(format!( Event::Html(CowStr::Boxed(format!(
r#"<label for="postcontent-cw-{id}"> r#"<label for="postcontent-cw-{id}">
<input type="checkbox" id="postcontent-cw-{id}" checked="checked" class="cw-checkbox"> <input type="checkbox" id="postcontent-cw-{id}" checked="checked" class="cw-checkbox">
<span class="cw-container"> <span class="cw-container">
@ -171,27 +180,27 @@ fn process_image<'a, 'b>(
id = random_hex(), id = random_hex(),
cw = cw, cw = cw,
url = url url = url
))) ).into()))
} else { } else {
Event::Start(Tag::Image(Cow::Owned(url), title)) Event::Start(Tag::Image(typ, CowStr::Boxed(url.into()), title))
} }
} else { } else {
Event::Start(Tag::Image(id, title)) Event::Start(Tag::Image(typ, id, title))
} }
} }
Event::End(Tag::Image(id, title)) => { Event::End(Tag::Image(typ, id, title)) => {
if let Some((url, cw)) = id.parse::<i32>().ok().and_then(processor.as_ref()) { if let Some((url, cw)) = id.parse::<i32>().ok().and_then(processor.as_ref()) {
if inline || cw.is_none() { if inline || cw.is_none() {
Event::End(Tag::Image(Cow::Owned(url), title)) Event::End(Tag::Image(typ, CowStr::Boxed(url.into()), title))
} else { } else {
Event::Html(Cow::Borrowed( Event::Html(CowStr::Borrowed(
r#""/> r#""/>
</span> </span>
</label>"#, </label>"#,
)) ))
} }
} else { } else {
Event::End(Tag::Image(id, title)) Event::End(Tag::Image(typ, id, title))
} }
} }
e => e, e => e,
@ -231,19 +240,19 @@ pub fn md_to_html<'a>(
// Ignore headings, images, and tables if inline = true // Ignore headings, images, and tables if inline = true
.scan((vec![], inline), inline_tags) .scan((vec![], inline), inline_tags)
.scan(&mut DocumentContext::default(), |ctx, evt| match evt { .scan(&mut DocumentContext::default(), |ctx, evt| match evt {
Event::Start(Tag::CodeBlock(_)) | Event::Start(Tag::Code) => { Event::Start(Tag::CodeBlock(_)) => {
ctx.in_code = true; ctx.in_code = true;
Some((vec![evt], vec![], vec![])) Some((vec![evt], vec![], vec![]))
} }
Event::End(Tag::CodeBlock(_)) | Event::End(Tag::Code) => { Event::End(Tag::CodeBlock(_)) => {
ctx.in_code = false; ctx.in_code = false;
Some((vec![evt], vec![], vec![])) Some((vec![evt], vec![], vec![]))
} }
Event::Start(Tag::Link(_, _)) => { Event::Start(Tag::Link(_, _, _)) => {
ctx.in_link = true; ctx.in_link = true;
Some((vec![evt], vec![], vec![])) Some((vec![evt], vec![], vec![]))
} }
Event::End(Tag::Link(_, _)) => { Event::End(Tag::Link(_, _, _)) => {
ctx.in_link = false; ctx.in_link = false;
Some((vec![evt], vec![], vec![])) Some((vec![evt], vec![], vec![]))
} }
@ -264,6 +273,7 @@ pub fn md_to_html<'a>(
let mention = text_acc; let mention = text_acc;
let short_mention = mention.splitn(1, '@').next().unwrap_or(""); let short_mention = mention.splitn(1, '@').next().unwrap_or("");
let link = Tag::Link( let link = Tag::Link(
LinkType::Inline,
format!("{}@/{}/", base_url, &mention).into(), format!("{}@/{}/", base_url, &mention).into(),
short_mention.to_owned().into(), short_mention.to_owned().into(),
); );
@ -294,6 +304,7 @@ pub fn md_to_html<'a>(
} }
let hashtag = text_acc; let hashtag = text_acc;
let link = Tag::Link( let link = Tag::Link(
LinkType::Inline,
format!("{}tag/{}", base_url, &hashtag).into(), format!("{}tag/{}", base_url, &hashtag).into(),
hashtag.to_owned().into(), hashtag.to_owned().into(),
); );
@ -459,11 +470,11 @@ mod tests {
fn test_inline() { fn test_inline() {
assert_eq!( assert_eq!(
md_to_html("# Hello", None, false, None).0, md_to_html("# Hello", None, false, None).0,
String::from("<h1>Hello</h1>\n") String::from("<h1 dir=\"auto\">Hello</h1>\n")
); );
assert_eq!( assert_eq!(
md_to_html("# Hello", None, true, None).0, md_to_html("# Hello", None, true, None).0,
String::from("<p>Hello</p>\n") String::from("<p dir=\"auto\">Hello</p>\n")
); );
} }
} }