1use std::{fmt::Write as _, fs, io, mem, path::PathBuf, sync::Arc};
4
5use litrs::StringLit;
6use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree};
7use rayon::prelude::*;
8
9pub fn scrape_fluent_text(code_files_glob: &str, custom_macro_names: &[&str]) -> FluentTemplate {
22 let num_threads = rayon::max_num_threads();
23 let mut buf = Vec::with_capacity(num_threads);
24
25 let mut r = FluentTemplate::default();
26 for file in glob::glob(code_files_glob).unwrap_or_else(|e| fatal!("{e}")) {
27 let file = file.unwrap_or_else(|e| fatal!("{e}"));
28 if file.is_dir() {
29 continue;
30 }
31 buf.push(file);
32 if buf.len() == num_threads {
33 buf.sort();
34 r.extend(scrape_files(&mut buf, custom_macro_names));
35 }
36 }
37 if !buf.is_empty() {
38 buf.sort();
39 r.extend(scrape_files(&mut buf, custom_macro_names));
40 }
41
42 r
43}
44fn scrape_files(buf: &mut Vec<PathBuf>, custom_macro_names: &[&str]) -> FluentTemplate {
45 buf.par_drain(..).map(|f| scrape_file(f, custom_macro_names)).reduce(
46 || FluentTemplate {
47 notes: vec![],
48 entries: vec![],
49 },
50 |mut a, b| {
51 a.extend(b);
52 a
53 },
54 )
55}
56fn scrape_file(rs_file: PathBuf, custom_macro_names: &[&str]) -> FluentTemplate {
57 let mut r = FluentTemplate::default();
58
59 let file = fs::read_to_string(&rs_file).unwrap_or_else(|e| fatal!("cannot open `{}`, {e}", rs_file.display()));
60
61 if !["l10n!", "command!", "l10n-"]
62 .iter()
63 .chain(custom_macro_names)
64 .any(|h| file.contains(h))
65 {
66 return FluentTemplate::default();
67 }
68
69 let file = file.strip_prefix('\u{feff}').unwrap_or(file.as_str());
71 let file = if file.starts_with("#!") && !file.starts_with("#![") {
73 &file[file.find('\n').unwrap_or(file.len())..]
74 } else {
75 file
76 };
77
78 let mut sections = vec![(0, Arc::new(String::new()))];
79 let mut comments = vec![];
80
81 let mut str_lit = false;
83 for (ln, mut line) in file.lines().enumerate() {
84 if str_lit {
85 while let Some(i) = line.find('"') {
87 let str_end = i == 0 || !line[..i].ends_with('\\');
88 line = &line[i + 1..];
89 if str_end {
90 break;
91 }
92 }
93 }
94 let line = line.trim();
95 if let Some(line) = line.strip_prefix("//") {
96 let line = line.trim_start();
97 if let Some(c) = line.strip_prefix("l10n-") {
98 if let Some(i) = c.find("###") {
100 let file_name = c[..i].trim_end_matches('-');
101 let c = &c[i + "###".len()..];
102
103 r.notes.push(FluentNote {
104 file: file_name.to_owned(),
105 note: c.trim().to_owned(),
106 });
107 } else if let Some(c) = c.strip_prefix("##") {
108 sections.push((ln + 1, Arc::new(c.trim().to_owned())));
109 } else if let Some(c) = c.strip_prefix('#') {
110 comments.push((ln + 1, c.trim()));
111 }
112 }
113 } else {
114 let mut line = line;
115 while !line.is_empty() {
116 if let Some((code, comment)) = line.split_once("//") {
117 let mut escape = false;
118 for c in code.chars() {
119 if mem::take(&mut escape) {
120 continue;
121 }
122 match c {
123 '\\' => escape = true,
124 '"' => str_lit = !str_lit,
125 _ => {}
126 }
127 }
128 if str_lit {
129 line = comment;
130 } else {
131 if let Some(c) = comment.trim_start().strip_prefix("l10n-#")
132 && !c.starts_with('#')
133 {
134 comments.push((ln + 1, c.trim()));
135 }
136
137 break;
139 }
140 } else {
141 break;
143 }
144 }
145 }
146 }
147
148 let file: TokenStream = file.parse().unwrap_or_else(|e| fatal!("cannot parse `{}`, {e}", rs_file.display()));
149
150 let mut stream_stack = vec![file.into_iter()];
152 let next = |stack: &mut Vec<proc_macro2::token_stream::IntoIter>| {
153 while !stack.is_empty() {
154 let tt = stack.last_mut().unwrap().next();
155 if tt.is_some() {
156 return tt;
157 }
158 stack.pop();
159 }
160 None
161 };
162
163 let mut tail2 = Vec::with_capacity(2);
164 while let Some(tt) = next(&mut stream_stack) {
165 match tt {
166 TokenTree::Group(g) => {
167 if matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket)
168 && tail2.len() == 2
169 && matches!(&tail2[0], TokenTree::Punct(p) if p.as_char() == '!')
170 && matches!(&tail2[1], TokenTree::Ident(i) if ["l10n", "command"].iter().chain(custom_macro_names).any(|n| i == n))
171 {
172 let macro_ln = match &tail2[1] {
175 TokenTree::Ident(i) => i.span().start().line,
176 _ => unreachable!(),
177 };
178
179 tail2.clear();
180
181 if let Ok(args) = L10nMacroArgs::try_from(g.stream()) {
182 let (file, id, attribute) = match parse_validate_id(&args.id) {
183 Ok(t) => t,
184 Err(e) => {
185 let lc = args.id_span.start();
186 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
187 continue;
188 }
189 };
190
191 debug_assert!(!sections.is_empty()); let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
194 let section = sections[section - 1].1.clone();
195
196 let last_ln = g.span_close().end().line;
198 let mut t = String::new();
199 let mut sep = "";
200 for (l, c) in &comments {
201 if *l <= last_ln {
202 if (macro_ln - 1..=last_ln).contains(l) {
203 t.push_str(sep);
204 t.push_str(c);
205 sep = "\n";
206 }
207 } else {
208 break;
209 }
210 }
211
212 r.entries.push(FluentEntry {
213 section,
214 comments: t,
215 file,
216 id,
217 attribute,
218 message: args.msg,
219 })
220 } else {
221 match CommandMacroArgs::try_from(g.stream()) {
222 Ok(cmds) => {
223 for cmd in cmds.entries {
224 let (file, id, _attribute) = match parse_validate_id(&cmd.id) {
225 Ok(t) => t,
226 Err(e) => {
227 let lc = cmd.file_span.start();
228 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
229 continue;
230 }
231 };
232 debug_assert!(_attribute.is_empty());
233
234 let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
236 let section = sections[section - 1].1.clone();
237
238 for meta in cmd.metadata {
239 let ln = meta.name.span().start().line;
241 let last_ln = meta.value_span.end().line;
242
243 let mut t = String::new();
244 let mut sep = "";
245 for (l, c) in &comments {
246 if *l <= last_ln {
247 if (ln - 1..=last_ln).contains(l) {
248 t.push_str(sep);
249 t.push_str(c);
250 sep = "\n";
251 }
252 } else {
253 break;
254 }
255 }
256
257 r.entries.push(FluentEntry {
258 section: section.clone(),
259 comments: t,
260 file: file.clone(),
261 id: id.clone(),
262 attribute: meta.name.to_string(),
263 message: meta.value,
264 })
265 }
266 }
267 }
268 Err(e) => {
269 if let Some((e, span)) = e {
270 let lc = span.start();
271 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
272 }
273 stream_stack.push(g.stream().into_iter());
274 }
275 }
276 }
277 } else {
278 stream_stack.push(g.stream().into_iter());
279 }
280 }
281 tt => {
282 if tail2.len() == 2 {
283 tail2.pop();
284 }
285 tail2.insert(0, tt);
286 }
287 }
288 }
289
290 r
291}
292struct L10nMacroArgs {
293 id: String,
294 id_span: Span,
295 msg: String,
296}
297impl TryFrom<TokenStream> for L10nMacroArgs {
298 type Error = String;
299
300 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
301 let three: Vec<_> = macro_group_stream.into_iter().take(3).collect();
302 match &three[..] {
303 [TokenTree::Literal(l0), TokenTree::Punct(p), TokenTree::Literal(l1)] if p.as_char() == ',' => {
304 match (StringLit::try_from(l0), StringLit::try_from(l1)) {
305 (Ok(s0), Ok(s1)) => Ok(Self {
306 id: s0.into_value(),
307 id_span: l0.span(),
308 msg: s1.into_value(),
309 }),
310 _ => Err(String::new()),
311 }
312 }
313 _ => Err(String::new()),
314 }
315 }
316}
317
318struct CommandMacroArgs {
319 entries: Vec<CommandMacroEntry>,
320}
321impl TryFrom<TokenStream> for CommandMacroArgs {
322 type Error = Option<(String, Span)>;
323
324 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
325 let mut entries = vec![];
326 let mut tail4 = Vec::with_capacity(4);
328 for tt in macro_group_stream.into_iter() {
329 tail4.push(tt);
330 if tail4.len() > 4 {
331 tail4.remove(0);
332 match &tail4[..] {
333 [
334 TokenTree::Ident(i0),
335 TokenTree::Ident(id),
336 TokenTree::Punct(p0),
337 TokenTree::Group(g),
338 ] if i0 == "static"
339 && p0.as_char() == '='
340 && matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket) =>
341 {
342 match CommandMacroEntry::try_from(g.stream()) {
343 Ok(mut entry) => {
344 entry.id.push('/');
345 entry.id.push_str(&id.to_string());
346 entries.push(entry);
347 }
348 Err(e) => {
349 if e.is_some() {
350 return Err(e);
351 }
352 }
353 }
354 }
355 _ => {}
356 }
357 }
358 }
359 if entries.is_empty() { Err(None) } else { Ok(Self { entries }) }
360 }
361}
362struct CommandMacroEntry {
363 id: String,
364 file_span: Span,
365 metadata: Vec<CommandMetaEntry>,
366}
367impl TryFrom<TokenStream> for CommandMacroEntry {
368 type Error = Option<(String, Span)>;
369
370 fn try_from(command_meta_group_stream: TokenStream) -> Result<Self, Self::Error> {
371 let mut tts = command_meta_group_stream.into_iter();
373
374 let mut r = CommandMacroEntry {
375 id: String::new(),
376 file_span: Span::call_site(),
377 metadata: vec![],
378 };
379
380 let mut buf: Vec<_> = (&mut tts).take(5).collect();
382 match &buf[..] {
383 [
384 TokenTree::Ident(i),
385 TokenTree::Punct(p0),
386 TokenTree::Punct(p1),
387 value,
388 TokenTree::Punct(p2),
389 ] if i == "l10n" && p0.as_char() == '!' && p1.as_char() == ':' && p2.as_char() == ',' => {
390 match litrs::Literal::try_from(value) {
391 Ok(litrs::Literal::String(str)) => {
392 r.id = str.into_value();
393 r.file_span = value.span();
394 }
395 Ok(litrs::Literal::Bool(b)) => {
396 if !b.value() {
397 return Err(None);
398 }
399 }
400 _ => {
401 return Err(Some((
402 "unexpected l10n: value, must be string or bool literal".to_owned(),
403 value.span(),
404 )));
405 }
406 }
407 }
408 _ => return Err(None),
409 }
410
411 buf.clear();
413 for tt in tts {
414 if buf.is_empty() && matches!(&tt, TokenTree::Punct(p) if p.as_char() == ',') {
415 continue;
416 }
417
418 buf.push(tt);
419 if buf.len() == 3 {
420 match &buf[..] {
421 [TokenTree::Ident(i), TokenTree::Punct(p), TokenTree::Literal(l)] if p.as_char() == ':' => {
422 if let Ok(s) = StringLit::try_from(l) {
423 r.metadata.push(CommandMetaEntry {
424 name: i.clone(),
425 value: s.into_value(),
426 value_span: l.span(),
427 })
428 }
429 }
430 _ => {}
431 }
432 buf.clear();
433 }
434 }
435
436 if r.metadata.is_empty() { Err(None) } else { Ok(r) }
437 }
438}
439struct CommandMetaEntry {
440 name: Ident,
441 value: String,
442 value_span: Span,
443}
444
445#[derive(Debug, Clone, PartialEq, Eq)]
447pub struct FluentNote {
448 pub file: String,
450
451 pub note: String,
453}
454
455#[derive(Debug, Clone, PartialEq, Eq)]
459pub struct FluentEntry {
460 pub section: Arc<String>,
462
463 pub comments: String,
465
466 pub file: String,
468 pub id: String,
470 pub attribute: String,
472
473 pub message: String,
475}
476
477#[derive(Default)]
481pub struct FluentTemplate {
482 pub notes: Vec<FluentNote>,
484
485 pub entries: Vec<FluentEntry>,
489}
490impl FluentTemplate {
491 pub fn extend(&mut self, other: Self) {
493 self.notes.extend(other.notes);
494 self.entries.extend(other.entries);
495 }
496
497 pub fn sort(&mut self) {
500 if self.entries.is_empty() {
501 return;
502 }
503
504 self.entries.sort_unstable_by(|a, b| {
506 match a.file.cmp(&b.file) {
507 core::cmp::Ordering::Equal => {}
508 ord => return ord,
509 }
510 match a.id.cmp(&b.id) {
511 core::cmp::Ordering::Equal => {}
512 ord => return ord,
513 }
514 a.attribute.cmp(&b.attribute)
515 });
516 let mut file = None;
518 let mut id = None;
519 let mut id_section = None;
520 for entry in &mut self.entries {
521 let f = Some(&entry.file);
522 let i = Some(&entry.id);
523
524 if (&file, &id) != (&f, &i) {
525 file = f;
526 id = i;
527 id_section = Some(&entry.section);
528 } else {
529 entry.section = Arc::clone(id_section.as_ref().unwrap());
530 }
531 }
532
533 let mut rmv_marker = None;
535 let mut id_start = 0;
536 for i in 1..self.entries.len() {
537 let prev = &self.entries[i - 1];
538 let e = &self.entries[i];
539
540 if e.id == prev.id && e.file == prev.file {
541 if let Some(already_i) = self.entries[id_start..i].iter().position(|s| s.attribute == e.attribute) {
542 let already_i = already_i + id_start;
543 self.entries[i].section = rmv_marker.get_or_insert_with(|| Arc::new(String::new())).clone();
547
548 let comment = mem::take(&mut self.entries[i].comments);
550 let c = &mut self.entries[already_i].comments;
551 if c.is_empty() {
552 *c = comment;
553 } else if !comment.is_empty() && !c.contains(&comment) {
554 c.push_str("\n\n");
555 c.push_str(&comment);
556 }
557 }
558 } else {
559 id_start = i;
560 }
561 }
562 if let Some(marker) = rmv_marker.take() {
563 let mut i = 0;
565 while i < self.entries.len() {
566 if Arc::ptr_eq(&marker, &self.entries[i].section) {
567 self.entries.swap_remove(i);
568 } else {
569 i += 1;
570 }
571 }
572 }
573
574 self.entries.sort_unstable_by(|a, b| {
576 match a.file.cmp(&b.file) {
577 core::cmp::Ordering::Equal => {}
578 ord => return ord,
579 }
580 match a.section.cmp(&b.section) {
581 core::cmp::Ordering::Equal => {}
582 ord => return ord,
583 }
584 match a.id.cmp(&b.id) {
585 core::cmp::Ordering::Equal => {}
586 ord => return ord,
587 }
588 a.attribute.cmp(&b.attribute)
589 });
590 }
591
592 pub fn write(&self, write_file: impl Fn(&str, &str) -> io::Result<()> + Send + Sync) -> io::Result<()> {
601 let mut file = None;
602 let mut output = String::new();
603 let mut section = "";
604 let mut id = "";
605
606 for (i, entry) in self.entries.iter().enumerate() {
607 if file != Some(&entry.file) {
608 if let Some(prev) = &file {
609 write_file(prev, &output)?;
610 output.clear();
611 section = "";
612 id = "";
613 }
614 file = Some(&entry.file);
615
616 if !self.notes.is_empty() {
619 for n in &self.notes {
620 let matches_file = if n.file.contains('*') {
621 match glob::Pattern::new(&n.file) {
622 Ok(b) => b.matches(&entry.file),
623 Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
624 }
625 } else {
626 n.file == entry.file
627 };
628
629 if matches_file {
630 writeln!(&mut output, "### {}", n.note).unwrap();
631 }
632 }
633 writeln!(&mut output).unwrap();
634 }
635 }
636
637 if id != entry.id && !id.is_empty() {
638 writeln!(&mut output).unwrap();
639 }
640
641 if section != entry.section.as_str() {
642 for line in entry.section.lines() {
644 writeln!(&mut output, "## {line}").unwrap();
645 }
646 writeln!(&mut output).unwrap();
647 section = entry.section.as_str();
648 }
649
650 if id != entry.id {
664 id = &entry.id;
665
666 for entry in self.entries[i..].iter() {
667 if entry.id != id {
668 break;
669 }
670
671 if entry.comments.is_empty() {
672 continue;
673 }
674 let mut prefix = "";
675 if !entry.attribute.is_empty() {
676 writeln!(&mut output, "# {}:", entry.attribute).unwrap();
677 prefix = " ";
678 }
679 for line in entry.comments.lines() {
680 writeln!(&mut output, "# {prefix}{line}").unwrap();
681 }
682 }
683
684 write!(&mut output, "{id} =").unwrap();
685 if entry.attribute.is_empty() {
686 let mut prefix = " ";
687
688 for line in entry.message.lines() {
689 writeln!(&mut output, "{prefix}{line}").unwrap();
690 prefix = " ";
691 }
692 } else {
693 writeln!(&mut output).unwrap();
694 }
695 }
696 if !entry.attribute.is_empty() {
697 write!(&mut output, " .{} = ", entry.attribute).unwrap();
698 let mut prefix = "";
699 for line in entry.message.lines() {
700 writeln!(&mut output, "{prefix}{line}").unwrap();
701 prefix = " ";
702 }
703 }
704 }
705
706 if let Some(prev) = &file {
707 write_file(prev, &output)?;
708 }
709
710 Ok(())
711 }
712}
713
714fn parse_validate_id(s: &str) -> Result<(String, String, String), String> {
716 let mut id = s;
717 let mut file = "";
718 let mut attribute = "";
719 if let Some((f, rest)) = id.rsplit_once('/') {
720 file = f;
721 id = rest;
722 }
723 if let Some((i, a)) = id.rsplit_once('.') {
724 id = i;
725 attribute = a;
726 }
727
728 if !file.is_empty() {
730 let mut first = true;
731 let mut valid = true;
732 let path: &std::path::Path = file.as_ref();
733 for c in path.components() {
734 if !first || !matches!(c, std::path::Component::Normal(_)) {
735 valid = false;
736 break;
737 }
738 first = false;
739 }
740 if !valid {
741 return Err(format!("invalid file {file:?}, must be a single file name"));
742 }
743 }
744
745 fn validate(value: &str) -> bool {
748 let mut first = true;
749 if !value.is_empty() {
750 for c in value.chars() {
751 if !first && (c == '_' || c == '-' || c.is_ascii_digit()) {
752 continue;
753 }
754 if !c.is_ascii_lowercase() && !c.is_ascii_uppercase() {
755 return false;
756 }
757
758 first = false;
759 }
760 } else {
761 return false;
762 }
763 true
764 }
765 if !validate(id) {
766 return Err(format!(
767 "invalid id {id:?}, must start with letter, followed by any letters, digits, `_` or `-`"
768 ));
769 }
770 if !attribute.is_empty() && !validate(attribute) {
771 return Err(format!(
772 "invalid id {attribute:?}, must start with letter, followed by any letters, digits, `_` or `-`"
773 ));
774 }
775
776 Ok((file.to_owned(), id.to_owned(), attribute.to_owned()))
777}