1use std::{fmt::Write as _, fs, io, mem, path::PathBuf, sync::Arc};
4
5use litrs::StringLit;
6use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree};
7use rayon::prelude::*;
8
9use crate::util::glob_walker;
10
11pub fn scrape_fluent_text(code_files_glob: &str, custom_macro_names: &[&str]) -> FluentTemplate {
24 let num_threads = rayon::max_num_threads();
25 let mut buf = Vec::with_capacity(num_threads);
26
27 let mut r = FluentTemplate::default();
28 for file in glob_walker(code_files_glob, true).unwrap_or_else(|e| fatal!("{e}")) {
29 let file = file.unwrap_or_else(|e| fatal!("{e}")).into_path();
30 if file.is_dir() {
31 continue;
32 }
33 buf.push(file);
34 if buf.len() == num_threads {
35 buf.sort();
36 r.extend(scrape_files(&mut buf, custom_macro_names));
37 }
38 }
39 if !buf.is_empty() {
40 buf.sort();
41 r.extend(scrape_files(&mut buf, custom_macro_names));
42 }
43
44 r
45}
46fn scrape_files(buf: &mut Vec<PathBuf>, custom_macro_names: &[&str]) -> FluentTemplate {
47 buf.par_drain(..).map(|f| scrape_file(f, custom_macro_names)).reduce(
48 || FluentTemplate {
49 notes: vec![],
50 entries: vec![],
51 },
52 |mut a, b| {
53 a.extend(b);
54 a
55 },
56 )
57}
58fn scrape_file(rs_file: PathBuf, custom_macro_names: &[&str]) -> FluentTemplate {
59 let mut r = FluentTemplate::default();
60
61 let file = fs::read_to_string(&rs_file).unwrap_or_else(|e| fatal!("cannot open `{}`, {e}", rs_file.display()));
62
63 if !["l10n!", "command!", "l10n-"]
64 .iter()
65 .chain(custom_macro_names)
66 .any(|h| file.contains(h))
67 {
68 return FluentTemplate::default();
69 }
70
71 let file = file.strip_prefix('\u{feff}').unwrap_or(file.as_str());
73 let file = if file.starts_with("#!") && !file.starts_with("#![") {
75 &file[file.find('\n').unwrap_or(file.len())..]
76 } else {
77 file
78 };
79
80 let mut sections = vec![(0, Arc::new(String::new()))];
81 let mut comments = vec![];
82
83 let mut str_lit = false;
85 for (ln, mut line) in file.lines().enumerate() {
86 if str_lit {
87 while let Some(i) = line.find('"') {
89 let str_end = i == 0 || !line[..i].ends_with('\\');
90 line = &line[i + 1..];
91 if str_end {
92 break;
93 }
94 }
95 }
96 let line = line.trim();
97 if let Some(line) = line.strip_prefix("//") {
98 let line = line.trim_start();
99 if let Some(c) = line.strip_prefix("l10n-") {
100 if let Some(i) = c.find("###") {
102 let file_name = c[..i].trim_end_matches('-');
103 let c = &c[i + "###".len()..];
104
105 r.notes.push(FluentNote {
106 file: file_name.to_owned(),
107 note: c.trim().to_owned(),
108 });
109 } else if let Some(c) = c.strip_prefix("##") {
110 sections.push((ln + 1, Arc::new(c.trim().to_owned())));
111 } else if let Some(c) = c.strip_prefix('#') {
112 comments.push((ln + 1, c.trim()));
113 }
114 }
115 } else {
116 let mut line = line;
117 while !line.is_empty() {
118 if let Some((code, comment)) = line.split_once("//") {
119 let mut escape = false;
120 for c in code.chars() {
121 if mem::take(&mut escape) {
122 continue;
123 }
124 match c {
125 '\\' => escape = true,
126 '"' => str_lit = !str_lit,
127 _ => {}
128 }
129 }
130 if str_lit {
131 line = comment;
132 } else {
133 if let Some(c) = comment.trim_start().strip_prefix("l10n-#")
134 && !c.starts_with('#')
135 {
136 comments.push((ln + 1, c.trim()));
137 }
138
139 break;
141 }
142 } else {
143 break;
145 }
146 }
147 }
148 }
149
150 let file: TokenStream = file.parse().unwrap_or_else(|e| fatal!("cannot parse `{}`, {e}", rs_file.display()));
151
152 let mut stream_stack = vec![file.into_iter()];
154 let next = |stack: &mut Vec<proc_macro2::token_stream::IntoIter>| {
155 while !stack.is_empty() {
156 let tt = stack.last_mut().unwrap().next();
157 if tt.is_some() {
158 return tt;
159 }
160 stack.pop();
161 }
162 None
163 };
164
165 let mut tail2 = Vec::with_capacity(2);
166 while let Some(tt) = next(&mut stream_stack) {
167 match tt {
168 TokenTree::Group(g) => {
169 if matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket)
170 && tail2.len() == 2
171 && matches!(&tail2[0], TokenTree::Punct(p) if p.as_char() == '!')
172 && matches!(&tail2[1], TokenTree::Ident(i) if ["l10n", "command"].iter().chain(custom_macro_names).any(|n| i == n))
173 {
174 let macro_ln = match &tail2[1] {
177 TokenTree::Ident(i) => i.span().start().line,
178 _ => unreachable!(),
179 };
180
181 tail2.clear();
182
183 if let Ok(args) = L10nMacroArgs::try_from(g.stream()) {
184 let (file, id, attribute) = match parse_validate_id(&args.id) {
185 Ok(t) => t,
186 Err(e) => {
187 let lc = args.id_span.start();
188 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
189 continue;
190 }
191 };
192
193 debug_assert!(!sections.is_empty()); let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
196 let section = sections[section - 1].1.clone();
197
198 let last_ln = g.span_close().end().line;
200 let mut t = String::new();
201 let mut sep = "";
202 for (l, c) in &comments {
203 if *l <= last_ln {
204 if (macro_ln - 1..=last_ln).contains(l) {
205 t.push_str(sep);
206 t.push_str(c);
207 sep = "\n";
208 }
209 } else {
210 break;
211 }
212 }
213
214 r.entries.push(FluentEntry {
215 section,
216 comments: t,
217 file,
218 id,
219 attribute,
220 message: args.msg,
221 })
222 } else {
223 match CommandMacroArgs::try_from(g.stream()) {
224 Ok(cmds) => {
225 for cmd in cmds.entries {
226 let (file, id, _attribute) = match parse_validate_id(&cmd.id) {
227 Ok(t) => t,
228 Err(e) => {
229 let lc = cmd.file_span.start();
230 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
231 continue;
232 }
233 };
234 debug_assert!(_attribute.is_empty());
235
236 let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
238 let section = sections[section - 1].1.clone();
239
240 for meta in cmd.metadata {
241 let ln = meta.name.span().start().line;
243 let last_ln = meta.value_span.end().line;
244
245 let mut t = String::new();
246 let mut sep = "";
247 for (l, c) in &comments {
248 if *l <= last_ln {
249 if (ln - 1..=last_ln).contains(l) {
250 t.push_str(sep);
251 t.push_str(c);
252 sep = "\n";
253 }
254 } else {
255 break;
256 }
257 }
258
259 r.entries.push(FluentEntry {
260 section: section.clone(),
261 comments: t,
262 file: file.clone(),
263 id: id.clone(),
264 attribute: meta.name.to_string(),
265 message: meta.value,
266 })
267 }
268 }
269 }
270 Err(e) => {
271 if let Some((e, span)) = e {
272 let lc = span.start();
273 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
274 }
275 stream_stack.push(g.stream().into_iter());
276 }
277 }
278 }
279 } else {
280 stream_stack.push(g.stream().into_iter());
281 }
282 }
283 tt => {
284 if tail2.len() == 2 {
285 tail2.pop();
286 }
287 tail2.insert(0, tt);
288 }
289 }
290 }
291
292 r
293}
294struct L10nMacroArgs {
295 id: String,
296 id_span: Span,
297 msg: String,
298}
299impl TryFrom<TokenStream> for L10nMacroArgs {
300 type Error = String;
301
302 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
303 let three: Vec<_> = macro_group_stream.into_iter().take(3).collect();
304 match &three[..] {
305 [TokenTree::Literal(l0), TokenTree::Punct(p), TokenTree::Literal(l1)] if p.as_char() == ',' => {
306 match (StringLit::try_from(l0), StringLit::try_from(l1)) {
307 (Ok(s0), Ok(s1)) => Ok(Self {
308 id: s0.into_value(),
309 id_span: l0.span(),
310 msg: s1.into_value(),
311 }),
312 _ => Err(String::new()),
313 }
314 }
315 _ => Err(String::new()),
316 }
317 }
318}
319
320struct CommandMacroArgs {
321 entries: Vec<CommandMacroEntry>,
322}
323impl TryFrom<TokenStream> for CommandMacroArgs {
324 type Error = Option<(String, Span)>;
325
326 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
327 let mut entries = vec![];
328 let mut tail3 = Vec::with_capacity(4);
330 for tt in macro_group_stream.into_iter() {
331 tail3.push(tt);
332 if tail3.len() > 3 {
333 tail3.remove(0);
334 match &tail3[..] {
335 [TokenTree::Ident(i0), TokenTree::Ident(id), TokenTree::Group(g)]
336 if i0 == "static" && matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket) =>
337 {
338 match CommandMacroEntry::try_from(g.stream()) {
339 Ok(mut entry) => {
340 entry.id.push('/');
341 entry.id.push_str(&id.to_string());
342 entries.push(entry);
343 }
344 Err(e) => {
345 if e.is_some() {
346 return Err(e);
347 }
348 }
349 }
350 }
351 _ => {}
352 }
353 }
354 }
355 if entries.is_empty() { Err(None) } else { Ok(Self { entries }) }
356 }
357}
358struct CommandMacroEntry {
359 id: String,
360 file_span: Span,
361 metadata: Vec<CommandMetaEntry>,
362}
363impl TryFrom<TokenStream> for CommandMacroEntry {
364 type Error = Option<(String, Span)>;
365
366 fn try_from(command_meta_group_stream: TokenStream) -> Result<Self, Self::Error> {
367 let mut tts = command_meta_group_stream.into_iter();
369
370 let mut r = CommandMacroEntry {
371 id: String::new(),
372 file_span: Span::call_site(),
373 metadata: vec![],
374 };
375
376 let mut buf: Vec<_> = (&mut tts).take(5).collect();
378 match &buf[..] {
379 [
380 TokenTree::Ident(i),
381 TokenTree::Punct(p0),
382 TokenTree::Punct(p1),
383 value,
384 TokenTree::Punct(p2),
385 ] if i == "l10n" && p0.as_char() == '!' && p1.as_char() == ':' && p2.as_char() == ',' => {
386 match litrs::Literal::try_from(value) {
387 Ok(litrs::Literal::String(str)) => {
388 r.id = str.into_value();
389 r.file_span = value.span();
390 }
391 Ok(litrs::Literal::Bool(b)) => {
392 if !b.value() {
393 return Err(None);
394 }
395 }
396 _ => {
397 return Err(Some((
398 "unexpected l10n: value, must be string or bool literal".to_owned(),
399 value.span(),
400 )));
401 }
402 }
403 }
404 _ => return Err(None),
405 }
406
407 buf.clear();
409 for tt in tts {
410 if buf.is_empty() && matches!(&tt, TokenTree::Punct(p) if p.as_char() == ',') {
411 continue;
412 }
413
414 buf.push(tt);
415 if buf.len() == 3 {
416 match &buf[..] {
417 [TokenTree::Ident(i), TokenTree::Punct(p), TokenTree::Literal(l)] if p.as_char() == ':' => {
418 if let Ok(s) = StringLit::try_from(l) {
419 r.metadata.push(CommandMetaEntry {
420 name: i.clone(),
421 value: s.into_value(),
422 value_span: l.span(),
423 })
424 }
425 }
426 _ => {}
427 }
428 buf.clear();
429 }
430 }
431
432 if r.metadata.is_empty() { Err(None) } else { Ok(r) }
433 }
434}
435struct CommandMetaEntry {
436 name: Ident,
437 value: String,
438 value_span: Span,
439}
440
441#[derive(Debug, Clone, PartialEq, Eq)]
443pub struct FluentNote {
444 pub file: String,
446
447 pub note: String,
449}
450
451#[derive(Debug, Clone, PartialEq, Eq)]
455pub struct FluentEntry {
456 pub section: Arc<String>,
458
459 pub comments: String,
461
462 pub file: String,
464 pub id: String,
466 pub attribute: String,
468
469 pub message: String,
471}
472
473#[derive(Default)]
477pub struct FluentTemplate {
478 pub notes: Vec<FluentNote>,
480
481 pub entries: Vec<FluentEntry>,
485}
486impl FluentTemplate {
487 pub fn extend(&mut self, other: Self) {
489 self.notes.extend(other.notes);
490 self.entries.extend(other.entries);
491 }
492
493 pub fn sort(&mut self) {
496 if self.entries.is_empty() {
497 return;
498 }
499
500 self.entries.sort_unstable_by(|a, b| {
502 match a.file.cmp(&b.file) {
503 core::cmp::Ordering::Equal => {}
504 ord => return ord,
505 }
506 match a.id.cmp(&b.id) {
507 core::cmp::Ordering::Equal => {}
508 ord => return ord,
509 }
510 a.attribute.cmp(&b.attribute)
511 });
512 let mut file = None;
514 let mut id = None;
515 let mut id_section = None;
516 for entry in &mut self.entries {
517 let f = Some(&entry.file);
518 let i = Some(&entry.id);
519
520 if (&file, &id) != (&f, &i) {
521 file = f;
522 id = i;
523 id_section = Some(&entry.section);
524 } else {
525 entry.section = Arc::clone(id_section.as_ref().unwrap());
526 }
527 }
528
529 let mut rmv_marker = None;
531 let mut id_start = 0;
532 for i in 1..self.entries.len() {
533 let prev = &self.entries[i - 1];
534 let e = &self.entries[i];
535
536 if e.id == prev.id && e.file == prev.file {
537 if let Some(already_i) = self.entries[id_start..i].iter().position(|s| s.attribute == e.attribute) {
538 let already_i = already_i + id_start;
539 self.entries[i].section = rmv_marker.get_or_insert_with(|| Arc::new(String::new())).clone();
543
544 let comment = mem::take(&mut self.entries[i].comments);
546 let c = &mut self.entries[already_i].comments;
547 if c.is_empty() {
548 *c = comment;
549 } else if !comment.is_empty() && !c.contains(&comment) {
550 c.push_str("\n\n");
551 c.push_str(&comment);
552 }
553
554 let prev = &self.entries[i - 1];
556 let e = &self.entries[i];
557 if prev.message != e.message {
558 let key = format_args!(
559 "{}/{}{}{}",
560 e.file,
561 e.id,
562 if e.attribute.is_empty() { "" } else { "." },
563 e.attribute
564 );
565 error!(
566 "{key} is declared multiple times with different messages\n{:?}\n{:?}",
567 prev.message, e.message
568 );
569 }
570 }
571 } else {
572 id_start = i;
573 }
574 }
575 if let Some(marker) = rmv_marker.take() {
576 let mut i = 0;
578 while i < self.entries.len() {
579 if Arc::ptr_eq(&marker, &self.entries[i].section) {
580 self.entries.swap_remove(i);
581 } else {
582 i += 1;
583 }
584 }
585 }
586
587 self.entries.sort_unstable_by(|a, b| {
589 match a.file.cmp(&b.file) {
590 core::cmp::Ordering::Equal => {}
591 ord => return ord,
592 }
593 match a.section.cmp(&b.section) {
594 core::cmp::Ordering::Equal => {}
595 ord => return ord,
596 }
597 match a.id.cmp(&b.id) {
598 core::cmp::Ordering::Equal => {}
599 ord => return ord,
600 }
601 a.attribute.cmp(&b.attribute)
602 });
603 }
604
605 pub const AUTO_GENERATED_HEADER: &str = "### Auto generated by `cargo zng l10n`\n\n";
606
607 pub fn write(&self, mut write_file: impl FnMut(&str, &str) -> io::Result<()> + Send + Sync) -> io::Result<()> {
616 let mut file = None;
617 let mut output = Self::AUTO_GENERATED_HEADER.to_owned();
618 let mut section = "";
619 let mut id = "";
620
621 for (i, entry) in self.entries.iter().enumerate() {
622 if file != Some(&entry.file) {
623 if let Some(prev) = &file {
624 write_file(prev, &output)?;
625 output.clear();
626 output.push_str(Self::AUTO_GENERATED_HEADER);
627 section = "";
628 id = "";
629 }
630 file = Some(&entry.file);
631
632 if !self.notes.is_empty() {
635 let mut any_note = false;
636 for n in &self.notes {
637 let matches_file = if n.file.contains('*') {
638 match globset::Glob::new(&n.file) {
639 Ok(b) => b.compile_matcher().is_match(&entry.file),
640 Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
641 }
642 } else {
643 n.file == entry.file
644 };
645
646 if matches_file {
647 writeln!(&mut output, "### {}", n.note).unwrap();
648 any_note = true;
649 }
650 }
651 if any_note {
652 writeln!(&mut output).unwrap();
653 }
654 }
655 }
656
657 if id != entry.id && !id.is_empty() {
658 writeln!(&mut output).unwrap();
659 }
660
661 if section != entry.section.as_str() {
662 for line in entry.section.lines() {
664 writeln!(&mut output, "## {line}").unwrap();
665 }
666 writeln!(&mut output).unwrap();
667 section = entry.section.as_str();
668 }
669
670 if id != entry.id {
684 id = &entry.id;
685
686 for entry in self.entries[i..].iter() {
687 if entry.id != id {
688 break;
689 }
690
691 if entry.comments.is_empty() {
692 continue;
693 }
694 let mut prefix = "";
695 if !entry.attribute.is_empty() {
696 writeln!(&mut output, "# {}:", entry.attribute).unwrap();
697 prefix = " ";
698 }
699 for line in entry.comments.lines() {
700 writeln!(&mut output, "# {prefix}{}", line.trim()).unwrap();
701 }
702 }
703
704 write!(&mut output, "{id} =").unwrap();
705 if entry.attribute.is_empty() {
706 let mut prefix = " ";
707
708 for line in entry.message.lines() {
709 writeln!(&mut output, "{prefix}{}", line.trim()).unwrap();
710 prefix = " ";
711 }
712 } else {
713 writeln!(&mut output).unwrap();
714 }
715 }
716 if !entry.attribute.is_empty() {
717 write!(&mut output, " .{} = ", entry.attribute).unwrap();
718 let mut prefix = "";
719 for line in entry.message.lines() {
720 writeln!(&mut output, "{prefix}{}", line.trim()).unwrap();
721 prefix = " ";
722 }
723 }
724 }
725
726 if let Some(prev) = &file {
727 write_file(prev, &output)?;
728 }
729
730 Ok(())
731 }
732}
733
734fn parse_validate_id(s: &str) -> Result<(String, String, String), String> {
736 let mut id = s;
737 let mut file = "";
738 let mut attribute = "";
739 if let Some((f, rest)) = id.rsplit_once('/') {
740 file = f;
741 id = rest;
742 }
743 if let Some((i, a)) = id.rsplit_once('.') {
744 id = i;
745 attribute = a;
746 }
747
748 if !file.is_empty() {
750 let mut first = true;
751 let mut valid = true;
752 let path: &std::path::Path = file.as_ref();
753 for c in path.components() {
754 if !first || !matches!(c, std::path::Component::Normal(_)) {
755 valid = false;
756 break;
757 }
758 first = false;
759 }
760 if !valid {
761 return Err(format!("invalid file {file:?}, must be a single file name"));
762 }
763 }
764
765 fn validate(value: &str) -> bool {
768 let mut first = true;
769 if !value.is_empty() {
770 for c in value.chars() {
771 if !first && (c == '_' || c == '-' || c.is_ascii_digit()) {
772 continue;
773 }
774 if !c.is_ascii_lowercase() && !c.is_ascii_uppercase() {
775 return false;
776 }
777
778 first = false;
779 }
780 } else {
781 return false;
782 }
783 true
784 }
785 if !validate(id) {
786 return Err(format!(
787 "invalid id {id:?}, must start with letter, followed by any letters, digits, `_` or `-`"
788 ));
789 }
790 if !attribute.is_empty() && !validate(attribute) {
791 return Err(format!(
792 "invalid id {attribute:?}, must start with letter, followed by any letters, digits, `_` or `-`"
793 ));
794 }
795
796 Ok((file.to_owned(), id.to_owned(), attribute.to_owned()))
797}