1use std::{fmt::Write as _, fs, io, mem, path::PathBuf, sync::Arc};
4
5use litrs::StringLit;
6use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree};
7use rayon::prelude::*;
8
9pub fn scrape_fluent_text(code_files_glob: &str, custom_macro_names: &[&str]) -> FluentTemplate {
22 let num_threads = rayon::max_num_threads();
23 let mut buf = Vec::with_capacity(num_threads);
24
25 let mut r = FluentTemplate::default();
26 for file in glob::glob(code_files_glob).unwrap_or_else(|e| fatal!("{e}")) {
27 let file = file.unwrap_or_else(|e| fatal!("{e}"));
28 if file.is_dir() {
29 continue;
30 }
31 buf.push(file);
32 if buf.len() == num_threads {
33 buf.sort();
34 r.extend(scrape_files(&mut buf, custom_macro_names));
35 }
36 }
37 if !buf.is_empty() {
38 buf.sort();
39 r.extend(scrape_files(&mut buf, custom_macro_names));
40 }
41
42 r
43}
44fn scrape_files(buf: &mut Vec<PathBuf>, custom_macro_names: &[&str]) -> FluentTemplate {
45 buf.par_drain(..).map(|f| scrape_file(f, custom_macro_names)).reduce(
46 || FluentTemplate {
47 notes: vec![],
48 entries: vec![],
49 },
50 |mut a, b| {
51 a.extend(b);
52 a
53 },
54 )
55}
56fn scrape_file(rs_file: PathBuf, custom_macro_names: &[&str]) -> FluentTemplate {
57 let mut r = FluentTemplate::default();
58
59 let file = fs::read_to_string(&rs_file).unwrap_or_else(|e| fatal!("cannot open `{}`, {e}", rs_file.display()));
60
61 if !["l10n!", "command!", "l10n-"]
62 .iter()
63 .chain(custom_macro_names)
64 .any(|h| file.contains(h))
65 {
66 return FluentTemplate::default();
67 }
68
69 let file = file.strip_prefix('\u{feff}').unwrap_or(file.as_str());
71 let file = if file.starts_with("#!") && !file.starts_with("#![") {
73 &file[file.find('\n').unwrap_or(file.len())..]
74 } else {
75 file
76 };
77
78 let mut sections = vec![(0, Arc::new(String::new()))];
79 let mut comments = vec![];
80
81 let mut str_lit = false;
83 for (ln, mut line) in file.lines().enumerate() {
84 if str_lit {
85 while let Some(i) = line.find('"') {
87 let str_end = i == 0 || !line[..i].ends_with('\\');
88 line = &line[i + 1..];
89 if str_end {
90 break;
91 }
92 }
93 }
94 let line = line.trim();
95 if let Some(line) = line.strip_prefix("//") {
96 let line = line.trim_start();
97 if let Some(c) = line.strip_prefix("l10n-") {
98 if let Some(i) = c.find("###") {
100 let file_name = c[..i].trim_end_matches('-');
101 let c = &c[i + "###".len()..];
102
103 r.notes.push(FluentNote {
104 file: file_name.to_owned(),
105 note: c.trim().to_owned(),
106 });
107 } else if let Some(c) = c.strip_prefix("##") {
108 sections.push((ln + 1, Arc::new(c.trim().to_owned())));
109 } else if let Some(c) = c.strip_prefix('#') {
110 comments.push((ln + 1, c.trim()));
111 }
112 }
113 } else {
114 let mut line = line;
115 while !line.is_empty() {
116 if let Some((code, comment)) = line.split_once("//") {
117 let mut escape = false;
118 for c in code.chars() {
119 if mem::take(&mut escape) {
120 continue;
121 }
122 match c {
123 '\\' => escape = true,
124 '"' => str_lit = !str_lit,
125 _ => {}
126 }
127 }
128 if str_lit {
129 line = comment;
130 } else {
131 if let Some(c) = comment.trim_start().strip_prefix("l10n-#")
132 && !c.starts_with('#')
133 {
134 comments.push((ln + 1, c.trim()));
135 }
136
137 break;
139 }
140 } else {
141 break;
143 }
144 }
145 }
146 }
147
148 let file: TokenStream = file.parse().unwrap_or_else(|e| fatal!("cannot parse `{}`, {e}", rs_file.display()));
149
150 let mut stream_stack = vec![file.into_iter()];
152 let next = |stack: &mut Vec<proc_macro2::token_stream::IntoIter>| {
153 while !stack.is_empty() {
154 let tt = stack.last_mut().unwrap().next();
155 if tt.is_some() {
156 return tt;
157 }
158 stack.pop();
159 }
160 None
161 };
162
163 let mut tail2 = Vec::with_capacity(2);
164 while let Some(tt) = next(&mut stream_stack) {
165 match tt {
166 TokenTree::Group(g) => {
167 if matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket)
168 && tail2.len() == 2
169 && matches!(&tail2[0], TokenTree::Punct(p) if p.as_char() == '!')
170 && matches!(&tail2[1], TokenTree::Ident(i) if ["l10n", "command"].iter().chain(custom_macro_names).any(|n| i == n))
171 {
172 let macro_ln = match &tail2[1] {
175 TokenTree::Ident(i) => i.span().start().line,
176 _ => unreachable!(),
177 };
178
179 tail2.clear();
180
181 if let Ok(args) = L10nMacroArgs::try_from(g.stream()) {
182 let (file, id, attribute) = match parse_validate_id(&args.id) {
183 Ok(t) => t,
184 Err(e) => {
185 let lc = args.id_span.start();
186 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
187 continue;
188 }
189 };
190
191 debug_assert!(!sections.is_empty()); let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
194 let section = sections[section - 1].1.clone();
195
196 let last_ln = g.span_close().end().line;
198 let mut t = String::new();
199 let mut sep = "";
200 for (l, c) in &comments {
201 if *l <= last_ln {
202 if (macro_ln - 1..=last_ln).contains(l) {
203 t.push_str(sep);
204 t.push_str(c);
205 sep = "\n";
206 }
207 } else {
208 break;
209 }
210 }
211
212 r.entries.push(FluentEntry {
213 section,
214 comments: t,
215 file,
216 id,
217 attribute,
218 message: args.msg,
219 })
220 } else {
221 match CommandMacroArgs::try_from(g.stream()) {
222 Ok(cmds) => {
223 for cmd in cmds.entries {
224 let (file, id, _attribute) = match parse_validate_id(&cmd.id) {
225 Ok(t) => t,
226 Err(e) => {
227 let lc = cmd.file_span.start();
228 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
229 continue;
230 }
231 };
232 debug_assert!(_attribute.is_empty());
233
234 let section = sections.iter().position(|(l, _)| *l > macro_ln).unwrap_or(sections.len());
236 let section = sections[section - 1].1.clone();
237
238 for meta in cmd.metadata {
239 let ln = meta.name.span().start().line;
241 let last_ln = meta.value_span.end().line;
242
243 let mut t = String::new();
244 let mut sep = "";
245 for (l, c) in &comments {
246 if *l <= last_ln {
247 if (ln - 1..=last_ln).contains(l) {
248 t.push_str(sep);
249 t.push_str(c);
250 sep = "\n";
251 }
252 } else {
253 break;
254 }
255 }
256
257 r.entries.push(FluentEntry {
258 section: section.clone(),
259 comments: t,
260 file: file.clone(),
261 id: id.clone(),
262 attribute: meta.name.to_string(),
263 message: meta.value,
264 })
265 }
266 }
267 }
268 Err(e) => {
269 if let Some((e, span)) = e {
270 let lc = span.start();
271 error!("{e}\n {}:{}:{}", rs_file.display(), lc.line, lc.column);
272 }
273 stream_stack.push(g.stream().into_iter());
274 }
275 }
276 }
277 } else {
278 stream_stack.push(g.stream().into_iter());
279 }
280 }
281 tt => {
282 if tail2.len() == 2 {
283 tail2.pop();
284 }
285 tail2.insert(0, tt);
286 }
287 }
288 }
289
290 r
291}
292struct L10nMacroArgs {
293 id: String,
294 id_span: Span,
295 msg: String,
296}
297impl TryFrom<TokenStream> for L10nMacroArgs {
298 type Error = String;
299
300 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
301 let three: Vec<_> = macro_group_stream.into_iter().take(3).collect();
302 match &three[..] {
303 [TokenTree::Literal(l0), TokenTree::Punct(p), TokenTree::Literal(l1)] if p.as_char() == ',' => {
304 match (StringLit::try_from(l0), StringLit::try_from(l1)) {
305 (Ok(s0), Ok(s1)) => Ok(Self {
306 id: s0.into_value(),
307 id_span: l0.span(),
308 msg: s1.into_value(),
309 }),
310 _ => Err(String::new()),
311 }
312 }
313 _ => Err(String::new()),
314 }
315 }
316}
317
318struct CommandMacroArgs {
319 entries: Vec<CommandMacroEntry>,
320}
321impl TryFrom<TokenStream> for CommandMacroArgs {
322 type Error = Option<(String, Span)>;
323
324 fn try_from(macro_group_stream: TokenStream) -> Result<Self, Self::Error> {
325 let mut entries = vec![];
326 let mut tail3 = Vec::with_capacity(4);
328 for tt in macro_group_stream.into_iter() {
329 tail3.push(tt);
330 if tail3.len() > 3 {
331 tail3.remove(0);
332 match &tail3[..] {
333 [TokenTree::Ident(i0), TokenTree::Ident(id), TokenTree::Group(g)]
334 if i0 == "static" && matches!(g.delimiter(), Delimiter::Brace | Delimiter::Parenthesis | Delimiter::Bracket) =>
335 {
336 match CommandMacroEntry::try_from(g.stream()) {
337 Ok(mut entry) => {
338 entry.id.push('/');
339 entry.id.push_str(&id.to_string());
340 entries.push(entry);
341 }
342 Err(e) => {
343 if e.is_some() {
344 return Err(e);
345 }
346 }
347 }
348 }
349 _ => {}
350 }
351 }
352 }
353 if entries.is_empty() { Err(None) } else { Ok(Self { entries }) }
354 }
355}
356struct CommandMacroEntry {
357 id: String,
358 file_span: Span,
359 metadata: Vec<CommandMetaEntry>,
360}
361impl TryFrom<TokenStream> for CommandMacroEntry {
362 type Error = Option<(String, Span)>;
363
364 fn try_from(command_meta_group_stream: TokenStream) -> Result<Self, Self::Error> {
365 let mut tts = command_meta_group_stream.into_iter();
367
368 let mut r = CommandMacroEntry {
369 id: String::new(),
370 file_span: Span::call_site(),
371 metadata: vec![],
372 };
373
374 let mut buf: Vec<_> = (&mut tts).take(5).collect();
376 match &buf[..] {
377 [
378 TokenTree::Ident(i),
379 TokenTree::Punct(p0),
380 TokenTree::Punct(p1),
381 value,
382 TokenTree::Punct(p2),
383 ] if i == "l10n" && p0.as_char() == '!' && p1.as_char() == ':' && p2.as_char() == ',' => {
384 match litrs::Literal::try_from(value) {
385 Ok(litrs::Literal::String(str)) => {
386 r.id = str.into_value();
387 r.file_span = value.span();
388 }
389 Ok(litrs::Literal::Bool(b)) => {
390 if !b.value() {
391 return Err(None);
392 }
393 }
394 _ => {
395 return Err(Some((
396 "unexpected l10n: value, must be string or bool literal".to_owned(),
397 value.span(),
398 )));
399 }
400 }
401 }
402 _ => return Err(None),
403 }
404
405 buf.clear();
407 for tt in tts {
408 if buf.is_empty() && matches!(&tt, TokenTree::Punct(p) if p.as_char() == ',') {
409 continue;
410 }
411
412 buf.push(tt);
413 if buf.len() == 3 {
414 match &buf[..] {
415 [TokenTree::Ident(i), TokenTree::Punct(p), TokenTree::Literal(l)] if p.as_char() == ':' => {
416 if let Ok(s) = StringLit::try_from(l) {
417 r.metadata.push(CommandMetaEntry {
418 name: i.clone(),
419 value: s.into_value(),
420 value_span: l.span(),
421 })
422 }
423 }
424 _ => {}
425 }
426 buf.clear();
427 }
428 }
429
430 if r.metadata.is_empty() { Err(None) } else { Ok(r) }
431 }
432}
433struct CommandMetaEntry {
434 name: Ident,
435 value: String,
436 value_span: Span,
437}
438
439#[derive(Debug, Clone, PartialEq, Eq)]
441pub struct FluentNote {
442 pub file: String,
444
445 pub note: String,
447}
448
449#[derive(Debug, Clone, PartialEq, Eq)]
453pub struct FluentEntry {
454 pub section: Arc<String>,
456
457 pub comments: String,
459
460 pub file: String,
462 pub id: String,
464 pub attribute: String,
466
467 pub message: String,
469}
470
471#[derive(Default)]
475pub struct FluentTemplate {
476 pub notes: Vec<FluentNote>,
478
479 pub entries: Vec<FluentEntry>,
483}
484impl FluentTemplate {
485 pub fn extend(&mut self, other: Self) {
487 self.notes.extend(other.notes);
488 self.entries.extend(other.entries);
489 }
490
491 pub fn sort(&mut self) {
494 if self.entries.is_empty() {
495 return;
496 }
497
498 self.entries.sort_unstable_by(|a, b| {
500 match a.file.cmp(&b.file) {
501 core::cmp::Ordering::Equal => {}
502 ord => return ord,
503 }
504 match a.id.cmp(&b.id) {
505 core::cmp::Ordering::Equal => {}
506 ord => return ord,
507 }
508 a.attribute.cmp(&b.attribute)
509 });
510 let mut file = None;
512 let mut id = None;
513 let mut id_section = None;
514 for entry in &mut self.entries {
515 let f = Some(&entry.file);
516 let i = Some(&entry.id);
517
518 if (&file, &id) != (&f, &i) {
519 file = f;
520 id = i;
521 id_section = Some(&entry.section);
522 } else {
523 entry.section = Arc::clone(id_section.as_ref().unwrap());
524 }
525 }
526
527 let mut rmv_marker = None;
529 let mut id_start = 0;
530 for i in 1..self.entries.len() {
531 let prev = &self.entries[i - 1];
532 let e = &self.entries[i];
533
534 if e.id == prev.id && e.file == prev.file {
535 if let Some(already_i) = self.entries[id_start..i].iter().position(|s| s.attribute == e.attribute) {
536 let already_i = already_i + id_start;
537 self.entries[i].section = rmv_marker.get_or_insert_with(|| Arc::new(String::new())).clone();
541
542 let comment = mem::take(&mut self.entries[i].comments);
544 let c = &mut self.entries[already_i].comments;
545 if c.is_empty() {
546 *c = comment;
547 } else if !comment.is_empty() && !c.contains(&comment) {
548 c.push_str("\n\n");
549 c.push_str(&comment);
550 }
551 }
552 } else {
553 id_start = i;
554 }
555 }
556 if let Some(marker) = rmv_marker.take() {
557 let mut i = 0;
559 while i < self.entries.len() {
560 if Arc::ptr_eq(&marker, &self.entries[i].section) {
561 self.entries.swap_remove(i);
562 } else {
563 i += 1;
564 }
565 }
566 }
567
568 self.entries.sort_unstable_by(|a, b| {
570 match a.file.cmp(&b.file) {
571 core::cmp::Ordering::Equal => {}
572 ord => return ord,
573 }
574 match a.section.cmp(&b.section) {
575 core::cmp::Ordering::Equal => {}
576 ord => return ord,
577 }
578 match a.id.cmp(&b.id) {
579 core::cmp::Ordering::Equal => {}
580 ord => return ord,
581 }
582 a.attribute.cmp(&b.attribute)
583 });
584 }
585
586 pub const AUTO_GENERATED_HEADER: &str = "### Auto generated by `cargo zng l10n`\n\n";
587
588 pub fn write(&self, mut write_file: impl FnMut(&str, &str) -> io::Result<()> + Send + Sync) -> io::Result<()> {
597 let mut file = None;
598 let mut output = Self::AUTO_GENERATED_HEADER.to_owned();
599 let mut section = "";
600 let mut id = "";
601
602 for (i, entry) in self.entries.iter().enumerate() {
603 if file != Some(&entry.file) {
604 if let Some(prev) = &file {
605 write_file(prev, &output)?;
606 output.clear();
607 output.push_str(Self::AUTO_GENERATED_HEADER);
608 section = "";
609 id = "";
610 }
611 file = Some(&entry.file);
612
613 if !self.notes.is_empty() {
616 let mut any_note = false;
617 for n in &self.notes {
618 let matches_file = if n.file.contains('*') {
619 match glob::Pattern::new(&n.file) {
620 Ok(b) => b.matches(&entry.file),
621 Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidInput, e)),
622 }
623 } else {
624 n.file == entry.file
625 };
626
627 if matches_file {
628 writeln!(&mut output, "### {}", n.note).unwrap();
629 any_note = true;
630 }
631 }
632 if any_note {
633 writeln!(&mut output).unwrap();
634 }
635 }
636 }
637
638 if id != entry.id && !id.is_empty() {
639 writeln!(&mut output).unwrap();
640 }
641
642 if section != entry.section.as_str() {
643 for line in entry.section.lines() {
645 writeln!(&mut output, "## {line}").unwrap();
646 }
647 writeln!(&mut output).unwrap();
648 section = entry.section.as_str();
649 }
650
651 if id != entry.id {
665 id = &entry.id;
666
667 for entry in self.entries[i..].iter() {
668 if entry.id != id {
669 break;
670 }
671
672 if entry.comments.is_empty() {
673 continue;
674 }
675 let mut prefix = "";
676 if !entry.attribute.is_empty() {
677 writeln!(&mut output, "# {}:", entry.attribute).unwrap();
678 prefix = " ";
679 }
680 for line in entry.comments.lines() {
681 writeln!(&mut output, "# {prefix}{line}").unwrap();
682 }
683 }
684
685 write!(&mut output, "{id} =").unwrap();
686 if entry.attribute.is_empty() {
687 let mut prefix = " ";
688
689 for line in entry.message.lines() {
690 writeln!(&mut output, "{prefix}{line}").unwrap();
691 prefix = " ";
692 }
693 } else {
694 writeln!(&mut output).unwrap();
695 }
696 }
697 if !entry.attribute.is_empty() {
698 write!(&mut output, " .{} = ", entry.attribute).unwrap();
699 let mut prefix = "";
700 for line in entry.message.lines() {
701 writeln!(&mut output, "{prefix}{line}").unwrap();
702 prefix = " ";
703 }
704 }
705 }
706
707 if let Some(prev) = &file {
708 write_file(prev, &output)?;
709 }
710
711 Ok(())
712 }
713}
714
715fn parse_validate_id(s: &str) -> Result<(String, String, String), String> {
717 let mut id = s;
718 let mut file = "";
719 let mut attribute = "";
720 if let Some((f, rest)) = id.rsplit_once('/') {
721 file = f;
722 id = rest;
723 }
724 if let Some((i, a)) = id.rsplit_once('.') {
725 id = i;
726 attribute = a;
727 }
728
729 if !file.is_empty() {
731 let mut first = true;
732 let mut valid = true;
733 let path: &std::path::Path = file.as_ref();
734 for c in path.components() {
735 if !first || !matches!(c, std::path::Component::Normal(_)) {
736 valid = false;
737 break;
738 }
739 first = false;
740 }
741 if !valid {
742 return Err(format!("invalid file {file:?}, must be a single file name"));
743 }
744 }
745
746 fn validate(value: &str) -> bool {
749 let mut first = true;
750 if !value.is_empty() {
751 for c in value.chars() {
752 if !first && (c == '_' || c == '-' || c.is_ascii_digit()) {
753 continue;
754 }
755 if !c.is_ascii_lowercase() && !c.is_ascii_uppercase() {
756 return false;
757 }
758
759 first = false;
760 }
761 } else {
762 return false;
763 }
764 true
765 }
766 if !validate(id) {
767 return Err(format!(
768 "invalid id {id:?}, must start with letter, followed by any letters, digits, `_` or `-`"
769 ));
770 }
771 if !attribute.is_empty() && !validate(attribute) {
772 return Err(format!(
773 "invalid id {attribute:?}, must start with letter, followed by any letters, digits, `_` or `-`"
774 ));
775 }
776
777 Ok((file.to_owned(), id.to_owned(), attribute.to_owned()))
778}