1use std::{fmt::Write as _, fs, io, mem, path::PathBuf, sync::Arc};
4
5use litrs::StringLit;
6use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree};
7use rayon::prelude::*;
8
9pub fn scrape_fluent_text(code_files_glob: &str, custom_macro_names: &[&str]) -> FluentTemplate {
22 let num_threads = rayon::max_num_threads();
23 let mut buf = Vec::with_capacity(num_threads);
24
25 let mut r = FluentTemplate::default();
26 for file in glob::glob(code_files_glob).unwrap_or_else(|e| fatal!("{e}")) {
27 let file = file.unwrap_or_else(|e| fatal!("{e}"));
28 if file.is_dir() {
29 continue;
30 }
31 buf.push(file);
32 if buf.len() == num_threads {
33 r.extend(scrape_files(&mut buf, custom_macro_names));
34 }
35 }
36 buf.sort();
37 if !buf.is_empty() {
38 r.extend(scrape_files(&mut buf, custom_macro_names));
39 }
40
41 r
42}
43fn scrape_files(buf: &mut Vec<PathBuf>, custom_macro_names: &[&str]) -> FluentTemplate {
44 buf.par_drain(..).map(|f| scrape_file(f, custom_macro_names)).reduce(
45 || FluentTemplate {
46 notes: vec![],
47 entries: vec![],
48 },
49 |mut a, b| {
50 a.extend(b);
51 a
52 },
53 )
54}
55fn scrape_file(rs_file: PathBuf, custom_macro_names: &[&str]) -> FluentTemplate {
56 let mut r = FluentTemplate::default();
57
58 let file = fs::read_to_string(&rs_file).unwrap_or_else(|e| fatal!("cannot open `{}`, {e}", rs_file.display()));
59
60 if !["l10n!", "command!", "l10n-"]
61 .iter()
62 .chain(custom_macro_names)
63 .any(|h| file.contains(h))
64 {
65 return FluentTemplate::default();
66 }
67
68 let file = file.strip_prefix('\u{feff}').unwrap_or(file.as_str());
70 let file = if file.starts_with("#!") && !file.starts_with("#![") {
72 &file[file.find('\n').unwrap_or(file.len())..]
73 } else {
74 file
75 };
76
77 let mut sections = vec![(0, Arc::new(String::new()))];
78 let mut comments = vec![];
79
80 let mut str_lit = false;
82 for (ln, mut line) in file.lines().enumerate() {
83 if str_lit {
84 while let Some(i) = line.find('"') {
86 let str_end = i == 0 || !line[..i].ends_with('\\');
87 line = &line[i + 1..];
88 if str_end {
89 break;
90 }
91 }
92 }
93 let line = line.trim();
94 if let Some(line) = line.strip_prefix("//") {
95 let line = line.trim_start();
96 if let Some(c) = line.strip_prefix("l10n-") {
97 if let Some(i) = c.find("###") {
99 let file_name = c[..i].trim_end_matches('-');
100 let c = &c[i + "###".len()..];
101
102 r.notes.push(FluentNote {
103 file: file_name.to_owned(),
104 note: c.trim().to_owned(),
105 });
106 } else if let Some(c) = c.strip_prefix("##") {
107 sections.push((ln + 1, Arc::new(c.trim().to_owned())));
108 } else if let Some(c) = c.strip_prefix('#') {
109 comments.push((ln + 1, c.trim()));
110 }
111 }
112 } else {
113 let mut line = line;
114 while !line.is_empty() {
115 if let Some((code, comment)) = line.split_once("//") {
116 let mut escape = false;
117 for c in code.chars() {
118 if mem::take(&mut escape) {
119 continue;
120 }
121 match c {
122 '\\' => escape = true,
123 '"' => str_lit = !str_lit,
124 _ => {}
125 }
126 }
127 if str_lit {
128 line = comment;
129 } else {
130 if let Some(c) = comment.trim_start().strip_prefix("l10n-#") {
131 if !c.starts_with('#') {
132 comments.push((ln + 1, c.trim()));
133 }
134 }
135
136 break;
138 }
139 } else {
140 break;
142 }
143 }
144 }
145 }
146
147 let file: TokenStream = file.parse().unwrap_or_else(|e| fatal!("cannot parse `{}`, {e}", rs_file.display()));
148
149 let mut stream_stack = vec![file.into_iter()];
151 let next = |stack: &mut Vec<proc_macro2::token_stream::IntoIter>| {
152 while !stack.is_empty() {
153 let tt = stack.last_mut().unwrap().next();
154 if tt.is_some() {
155 return tt;
156 }
157 stack.pop();
158 }
159 None
160 };
161
162 let mut tail2 = Vec::with_capacity(2);
163 while let Some(tt) = next(&mut stream_stack) {
164 match tt {
165 TokenTree::Group(g) => {
166 if matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket)
167 && tail2.len() == 2
168 && matches!(&tail2[0], TokenTree::Punct(p) if p.as_char() == '!')
169 && matches!(&tail2[1], TokenTree::Ident(i) if ["l10n", "command"].iter().chain(custom_macro_names).any(|n| i == n))
170 {
171 let macro_ln = match &tail2[1] {
174 TokenTree::Ident(i) => i.span().start().line,
175 _ => unreachable!(),
176 };
177
178 tail2.clear();
179
180 if let Ok(args) = L10nMacroArgs::try_from(g.stream()) {
181 let (file, id, attribute) = match parse_validate_id(&args.id) {
182 Ok(t) => t,
183 Err(e) => {
184 let lc = args.id_span.start();
185 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
186 continue;
187 }
188 };
189
190 debug_assert!(!sections.is_empty()); let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
193 let section = sections[section - 1].1.clone();
194
195 let last_ln = g.span_close().end().line;
197 let mut t = String::new();
198 let mut sep = "";
199 for (l, c) in &comments {
200 if *l <= last_ln {
201 if (macro_ln - 1..=last_ln).contains(l) {
202 t.push_str(sep);
203 t.push_str(c);
204 sep = "\n";
205 }
206 } else {
207 break;
208 }
209 }
210
211 r.entries.push(FluentEntry {
212 section,
213 comments: t,
214 file,
215 id,
216 attribute,
217 message: args.msg,
218 })
219 } else {
220 match CommandMacroArgs::try_from(g.stream()) {
221 Ok(cmds) => {
222 for cmd in cmds.entries {
223 let (file, id, _attribute) = match parse_validate_id(&cmd.id) {
224 Ok(t) => t,
225 Err(e) => {
226 let lc = cmd.file_span.start();
227 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
228 continue;
229 }
230 };
231 debug_assert!(_attribute.is_empty());
232
233 let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
235 let section = sections[section - 1].1.clone();
236
237 for meta in cmd.metadata {
238 let ln = meta.name.span().start().line;
240 let last_ln = meta.value_span.end().line;
241
242 let mut t = String::new();
243 let mut sep = "";
244 for (l, c) in &comments {
245 if *l <= last_ln {
246 if (ln - 1..=last_ln).contains(l) {
247 t.push_str(sep);
248 t.push_str(c);
249 sep = "\n";
250 }
251 } else {
252 break;
253 }
254 }
255
256 r.entries.push(FluentEntry {
257 section: section.clone(),
258 comments: t,
259 file: file.clone(),
260 id: id.clone(),
261 attribute: meta.name.to_string(),
262 message: meta.value,
263 })
264 }
265 }
266 }
267 Err(e) => {
268 if let Some((e, span)) = e {
269 let lc = span.start();
270 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
271 }
272 stream_stack.push(g.stream().into_iter());
273 }
274 }
275 }
276 } else {
277 stream_stack.push(g.stream().into_iter());
278 }
279 }
280 tt => {
281 if tail2.len() == 2 {
282 tail2.pop();
283 }
284 tail2.insert(0, tt);
285 }
286 }
287 }
288
289 r
290}
291struct L10nMacroArgs {
292 id: String,
293 id_span: Span,
294 msg: String,
295}
296impl TryFrom<TokenStream> for L10nMacroArgs {
297 type Error = String;
298
299 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
300 let three: Vec<_> = macro_group_stream.into_iter().take(3).collect();
301 match &three[..] {
302 [TokenTree::Literal(l0), TokenTree::Punct(p), TokenTree::Literal(l1)] if p.as_char() == ',' => {
303 match (StringLit::try_from(l0), StringLit::try_from(l1)) {
304 (Ok(s0), Ok(s1)) => Ok(Self {
305 id: s0.into_value().into_owned(),
306 id_span: l0.span(),
307 msg: s1.into_value().into_owned(),
308 }),
309 _ => Err(String::new()),
310 }
311 }
312 _ => Err(String::new()),
313 }
314 }
315}
316
317struct CommandMacroArgs {
318 entries: Vec<CommandMacroEntry>,
319}
320impl TryFrom<TokenStream> for CommandMacroArgs {
321 type Error = Option<(String, Span)>;
322
323 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
324 let mut entries = vec![];
325 let mut tail4 = Vec::with_capacity(4);
327 for tt in macro_group_stream.into_iter() {
328 tail4.push(tt);
329 if tail4.len() > 4 {
330 tail4.remove(0);
331 match &tail4[..] {
332 [
333 TokenTree::Ident(i0),
334 TokenTree::Ident(id),
335 TokenTree::Punct(p0),
336 TokenTree::Group(g),
337 ] if i0 == "static"
338 && p0.as_char() == '='
339 && matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket) =>
340 {
341 match CommandMacroEntry::try_from(g.stream()) {
342 Ok(mut entry) => {
343 entry.id.push('/');
344 entry.id.push_str(&id.to_string());
345 entries.push(entry);
346 }
347 Err(e) => {
348 if e.is_some() {
349 return Err(e);
350 }
351 }
352 }
353 }
354 _ => {}
355 }
356 }
357 }
358 if entries.is_empty() { Err(None) } else { Ok(Self { entries }) }
359 }
360}
361struct CommandMacroEntry {
362 id: String,
363 file_span: Span,
364 metadata: Vec<CommandMetaEntry>,
365}
366impl TryFrom<TokenStream> for CommandMacroEntry {
367 type Error = Option<(String, Span)>;
368
369 fn try_from(command_meta_group_stream: TokenStream) -> Result<Self, Self::Error> {
370 let mut tts = command_meta_group_stream.into_iter();
372
373 let mut r = CommandMacroEntry {
374 id: String::new(),
375 file_span: Span::call_site(),
376 metadata: vec![],
377 };
378
379 let mut buf: Vec<_> = (&mut tts).take(5).collect();
381 match &buf[..] {
382 [
383 TokenTree::Ident(i),
384 TokenTree::Punct(p0),
385 TokenTree::Punct(p1),
386 value,
387 TokenTree::Punct(p2),
388 ] if i == "l10n" && p0.as_char() == '!' && p1.as_char() == ':' && p2.as_char() == ',' => {
389 match litrs::Literal::try_from(value) {
390 Ok(litrs::Literal::String(str)) => {
391 r.id = str.into_value().into_owned();
392 r.file_span = value.span();
393 }
394 Ok(litrs::Literal::Bool(b)) => {
395 if !b.value() {
396 return Err(None);
397 }
398 }
399 _ => {
400 return Err(Some((
401 "unexpected l10n: value, must be string or bool literal".to_owned(),
402 value.span(),
403 )));
404 }
405 }
406 }
407 _ => return Err(None),
408 }
409
410 buf.clear();
412 for tt in tts {
413 if buf.is_empty() && matches!(&tt, TokenTree::Punct(p) if p.as_char() == ',') {
414 continue;
415 }
416
417 buf.push(tt);
418 if buf.len() == 3 {
419 match &buf[..] {
420 [TokenTree::Ident(i), TokenTree::Punct(p), TokenTree::Literal(l)] if p.as_char() == ':' => {
421 if let Ok(s) = StringLit::try_from(l) {
422 r.metadata.push(CommandMetaEntry {
423 name: i.clone(),
424 value: s.into_value().into_owned(),
425 value_span: l.span(),
426 })
427 }
428 }
429 _ => {}
430 }
431 buf.clear();
432 }
433 }
434
435 if r.metadata.is_empty() { Err(None) } else { Ok(r) }
436 }
437}
438struct CommandMetaEntry {
439 name: Ident,
440 value: String,
441 value_span: Span,
442}
443
444#[derive(Debug, Clone, PartialEq, Eq)]
446pub struct FluentNote {
447 pub file: String,
449
450 pub note: String,
452}
453
454#[derive(Debug, Clone, PartialEq, Eq)]
458pub struct FluentEntry {
459 pub section: Arc<String>,
461
462 pub comments: String,
464
465 pub file: String,
467 pub id: String,
469 pub attribute: String,
471
472 pub message: String,
474}
475
476#[derive(Default)]
480pub struct FluentTemplate {
481 pub notes: Vec<FluentNote>,
483
484 pub entries: Vec<FluentEntry>,
488}
489impl FluentTemplate {
490 pub fn extend(&mut self, other: Self) {
492 self.notes.extend(other.notes);
493 self.entries.extend(other.entries);
494 }
495
496 pub fn sort(&mut self) {
499 if self.entries.is_empty() {
500 return;
501 }
502
503 self.entries.sort_unstable_by(|a, b| {
505 match a.file.cmp(&b.file) {
506 core::cmp::Ordering::Equal => {}
507 ord => return ord,
508 }
509 match a.id.cmp(&b.id) {
510 core::cmp::Ordering::Equal => {}
511 ord => return ord,
512 }
513 a.attribute.cmp(&b.attribute)
514 });
515 let mut file = None;
517 let mut id = None;
518 let mut id_section = None;
519 for entry in &mut self.entries {
520 let f = Some(&entry.file);
521 let i = Some(&entry.id);
522
523 if (&file, &id) != (&f, &i) {
524 file = f;
525 id = i;
526 id_section = Some(&entry.section);
527 } else {
528 entry.section = Arc::clone(id_section.as_ref().unwrap());
529 }
530 }
531
532 let mut rmv_marker = None;
534 let mut id_start = 0;
535 for i in 1..self.entries.len() {
536 let prev = &self.entries[i - 1];
537 let e = &self.entries[i];
538
539 if e.id == prev.id && e.file == prev.file {
540 if let Some(already_i) = self.entries[id_start..i].iter().position(|s| s.attribute == e.attribute) {
541 let already_i = already_i + id_start;
542 self.entries[i].section = rmv_marker.get_or_insert_with(|| Arc::new(String::new())).clone();
546
547 let comment = mem::take(&mut self.entries[i].comments);
549 let c = &mut self.entries[already_i].comments;
550 if c.is_empty() {
551 *c = comment;
552 } else if !comment.is_empty() && !c.contains(&comment) {
553 c.push_str("\n\n");
554 c.push_str(&comment);
555 }
556 }
557 } else {
558 id_start = i;
559 }
560 }
561 if let Some(marker) = rmv_marker.take() {
562 let mut i = 0;
564 while i < self.entries.len() {
565 while Arc::ptr_eq(&marker, &self.entries[i].section) {
566 self.entries.swap_remove(i);
567 }
568 i += 1;
569 }
570 }
571
572 self.entries.sort_unstable_by(|a, b| {
574 match a.file.cmp(&b.file) {
575 core::cmp::Ordering::Equal => {}
576 ord => return ord,
577 }
578 match a.section.cmp(&b.section) {
579 core::cmp::Ordering::Equal => {}
580 ord => return ord,
581 }
582 match a.id.cmp(&b.id) {
583 core::cmp::Ordering::Equal => {}
584 ord => return ord,
585 }
586 a.attribute.cmp(&b.attribute)
587 });
588 }
589
590 pub fn write(&self, write_file: impl Fn(&str, &str) -> io::Result<()> + Send + Sync) -> io::Result<()> {
599 let mut file = None;
600 let mut output = String::new();
601 let mut section = "";
602 let mut id = "";
603
604 for (i, entry) in self.entries.iter().enumerate() {
605 if file != Some(&entry.file) {
606 if let Some(prev) = &file {
607 write_file(prev, &output)?;
608 output.clear();
609 section = "";
610 id = "";
611 }
612 file = Some(&entry.file);
613
614 if !self.notes.is_empty() {
617 for n in &self.notes {
618 let matches_file = if n.file.contains('*') {
619 match glob::Pattern::new(&n.file) {
620 Ok(b) => b.matches(&entry.file),
621 Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
622 }
623 } else {
624 n.file == entry.file
625 };
626
627 if matches_file {
628 writeln!(&mut output, "### {}", n.note).unwrap();
629 }
630 }
631 writeln!(&mut output).unwrap();
632 }
633 }
634
635 if id != entry.id && !id.is_empty() {
636 writeln!(&mut output).unwrap();
637 }
638
639 if section != entry.section.as_str() {
640 for line in entry.section.lines() {
642 writeln!(&mut output, "## {line}").unwrap();
643 }
644 writeln!(&mut output).unwrap();
645 section = entry.section.as_str();
646 }
647
648 if id != entry.id {
662 id = &entry.id;
663
664 for entry in self.entries[i..].iter() {
665 if entry.id != id {
666 break;
667 }
668
669 if entry.comments.is_empty() {
670 continue;
671 }
672 let mut prefix = "";
673 if !entry.attribute.is_empty() {
674 writeln!(&mut output, "# {}:", entry.attribute).unwrap();
675 prefix = " ";
676 }
677 for line in entry.comments.lines() {
678 writeln!(&mut output, "# {prefix}{line}").unwrap();
679 }
680 }
681
682 write!(&mut output, "{id} =").unwrap();
683 if entry.attribute.is_empty() {
684 let mut prefix = " ";
685
686 for line in entry.message.lines() {
687 writeln!(&mut output, "{prefix}{line}").unwrap();
688 prefix = " ";
689 }
690 } else {
691 writeln!(&mut output).unwrap();
692 }
693 }
694 if !entry.attribute.is_empty() {
695 write!(&mut output, " .{} = ", entry.attribute).unwrap();
696 let mut prefix = "";
697 for line in entry.message.lines() {
698 writeln!(&mut output, "{prefix}{line}").unwrap();
699 prefix = " ";
700 }
701 }
702 }
703
704 if let Some(prev) = &file {
705 write_file(prev, &output)?;
706 }
707
708 Ok(())
709 }
710}
711
712fn parse_validate_id(s: &str) -> Result<(String, String, String), String> {
714 let mut id = s;
715 let mut file = "";
716 let mut attribute = "";
717 if let Some((f, rest)) = id.rsplit_once('/') {
718 file = f;
719 id = rest;
720 }
721 if let Some((i, a)) = id.rsplit_once('.') {
722 id = i;
723 attribute = a;
724 }
725
726 if !file.is_empty() {
728 let mut first = true;
729 let mut valid = true;
730 let path: &std::path::Path = file.as_ref();
731 for c in path.components() {
732 if !first || !matches!(c, std::path::Component::Normal(_)) {
733 valid = false;
734 break;
735 }
736 first = false;
737 }
738 if !valid {
739 return Err(format!("invalid file {file:?}, must be a single file name"));
740 }
741 }
742
743 fn validate(value: &str) -> bool {
746 let mut first = true;
747 if !value.is_empty() {
748 for c in value.chars() {
749 if !first && (c == '_' || c == '-' || c.is_ascii_digit()) {
750 continue;
751 }
752 if !c.is_ascii_lowercase() && !c.is_ascii_uppercase() {
753 return false;
754 }
755
756 first = false;
757 }
758 } else {
759 return false;
760 }
761 true
762 }
763 if !validate(id) {
764 return Err(format!(
765 "invalid id {id:?}, must start with letter, followed by any letters, digits, `_` or `-`"
766 ));
767 }
768 if !attribute.is_empty() && !validate(attribute) {
769 return Err(format!(
770 "invalid id {attribute:?}, must start with letter, followed by any letters, digits, `_` or `-`"
771 ));
772 }
773
774 Ok((file.to_owned(), id.to_owned(), attribute.to_owned()))
775}