1use std::{
8 cmp::Ordering,
9 collections::{HashMap, HashSet},
10 fmt::{self, Write as _},
11 fs,
12 io::{self, BufRead},
13 path::{Path, PathBuf},
14};
15
16use clap::*;
17
18use crate::{
19 l10n::scraper::FluentTemplate,
20 util::{self, glob_walker},
21};
22
23mod scraper;
24
25pub(crate) mod generate_util;
26mod pseudo;
27mod translate;
28
29#[derive(Args, Debug)]
30pub struct L10nArgs {
31 #[arg(short, long, default_value = "", value_name = "PATH", hide_default_value = true)]
33 input: String,
34
35 #[arg(short, long, default_value = "", value_name = "DIR", hide_default_value = true)]
37 output: String,
38
39 #[arg(short, long, default_value = "", hide_default_value = true)]
43 package: String,
44
45 #[arg(long, default_value = "", hide_default_value = true)]
49 manifest_path: String,
50
51 #[arg(long, action)]
55 no_deps: bool,
56
57 #[arg(long, action)]
61 no_local: bool,
62
63 #[arg(long, action)]
67 no_pkg: bool,
68
69 #[arg(long, action)]
71 clean_deps: bool,
72
73 #[arg(long, action)]
75 clean_template: bool,
76
77 #[arg(long, action)]
79 clean: bool,
80
81 #[arg(short, long, default_value = "", hide_default_value = true)]
83 macros: String,
84
85 #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
91 pseudo: String,
92 #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
94 pseudo_m: String,
95 #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
97 pseudo_w: String,
98
99 #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
104 release_langs: String,
105
106 #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
112 translate: String,
113
114 #[arg(long, default_value = "", value_name = "LANG", hide_default_value = true)]
118 translate_from: String,
119
120 #[arg(
124 long,
125 default_value = "ar,bg,ca,cs,da,de,el,en,es-419,es-ES,et,eu,fi,fr-CA,fr-FR,gl,he,hi,hr,hu,id,it,ja,ko,lt,lv,nb,nl,pl,pt-BR,pt-PT,ro,ru,sk,sl,sr-Latn,sv,th,tr,uk,vi,zh-Hans,zh-Hant",
126 value_name = "LANGS"
127 )]
128 translate_to: String,
129
130 #[arg(long, action)]
134 translate_replace: bool,
135
136 #[arg(long, action)]
142 translate_existing: bool,
143
144 #[arg(long, action)]
146 check: bool,
147
148 #[arg(long, action)]
150 check_strict: bool,
151}
152
153pub fn run(mut args: L10nArgs) {
154 if !args.package.is_empty() && !args.manifest_path.is_empty() {
155 fatal!("only one of --package --manifest-path must be set")
156 }
157
158 if args.check_strict {
159 args.check = true;
160 }
161
162 let mut input = String::new();
163 let mut output = args.output.replace('\\', "/");
164
165 if !args.input.is_empty() {
166 input = args.input.replace('\\', "/");
167
168 if !input.contains('*') && PathBuf::from(&input).is_dir() {
169 input = format!("{}/**/*.rs", input.trim_end_matches('/'));
170 }
171 }
172 if !args.package.is_empty() {
173 if let Some(m) = crate::util::manifest_path_from_package(&args.package) {
174 args.manifest_path = m;
175 } else {
176 fatal!("package `{}` not found in workspace", args.package);
177 }
178 }
179
180 if !args.manifest_path.is_empty() {
181 if !Path::new(&args.manifest_path).exists() {
182 fatal!("`{}` does not exist", args.manifest_path)
183 }
184
185 if let Some(path) = args.manifest_path.replace('\\', "/").strip_suffix("/Cargo.toml") {
186 if output.is_empty() {
187 output = format!("{path}/l10n");
188 }
189 if input.is_empty() {
190 input = format!("{path}/src/**/*.rs");
191 }
192 } else {
193 fatal!("expected path to Cargo.toml manifest file");
194 }
195 }
196
197 if args.check {
198 args.clean = false;
199 args.clean_deps = false;
200 args.clean_template = false;
201 } else if args.clean {
202 args.clean_deps = true;
203 args.clean_template = true;
204 }
205
206 println_v!(
207 "input: `{input}`\noutput: `{output}`\nclean_deps: {}\nclean_template: {}",
208 args.clean_deps,
209 args.clean_template
210 );
211
212 if input.is_empty() {
213 return run_others(&args);
214 }
215
216 if output.is_empty() {
217 fatal!("--output is required for --input")
218 }
219
220 let input = input;
221 let output = Path::new(&output);
222
223 let mut template = FluentTemplate::default();
224
225 check_scrap_package(&args, &input, output, &mut template);
226
227 if !template.entries.is_empty() || !template.notes.is_empty() {
228 if let Err(e) = util::check_or_create_dir_all(args.check, output) {
229 fatal!("cannot create dir `{}`, {e}", output.display());
230 }
231
232 let output = output.join("template");
233
234 if let Err(e) = util::check_or_create_dir_all(args.check, &output) {
235 fatal!("cannot create dir `{}`, {e}", output.display());
236 }
237
238 template.sort();
239
240 let mut clean_files = HashSet::new();
241
242 let r = template.write(|file, contents| {
243 let file = format!("{}.ftl", if file.is_empty() { "_" } else { file });
244 let output = output.join(&file);
245 clean_files.insert(file);
246 util::check_or_write(args.check, output, contents)
247 });
248 if let Err(e) = r {
249 fatal!("error writing template files, {e}");
250 }
251
252 if args.clean_template {
253 debug_assert!(!args.check);
254
255 let cleanup = || -> std::io::Result<()> {
256 for entry in std::fs::read_dir(&output)? {
257 let entry = entry?.path();
258 if entry.is_file() {
259 let name = entry.file_prefix().unwrap().to_string_lossy();
260 if name.ends_with(".ftl") && !clean_files.contains(&*name) {
261 let mut entry_file = std::fs::File::open(&entry)?;
262 if let Some(first_line) = std::io::BufReader::new(&mut entry_file).lines().next()
263 && first_line?.starts_with(FluentTemplate::AUTO_GENERATED_HEADER)
264 {
265 drop(entry_file);
266 std::fs::remove_file(entry)?;
267 }
268 }
269 }
270 }
271 Ok(())
272 };
273 if let Err(e) = cleanup() {
274 error!("failed template cleanup, {e}");
275 }
276 }
277 }
278
279 if args.check {
280 check_fluent_output(&args, output);
281 }
282
283 run_others(&args);
284}
285
286fn check_scrap_package(args: &L10nArgs, input: &str, output: &Path, template: &mut FluentTemplate) {
287 if !args.no_pkg {
289 if args.check {
290 println!(r#"checking "{input}".."#);
291 } else {
292 println!(r#"scraping "{input}".."#);
293 }
294
295 let custom_macro_names: Vec<&str> = args.macros.split(',').map(|n| n.trim()).collect();
296 let t = scraper::scrape_fluent_text(input, &custom_macro_names);
297 if !args.check {
298 match t.entries.len() {
299 0 => println!(" did not find any entry"),
300 1 => println!(" found 1 entry"),
301 n => println!(" found {n} entries"),
302 }
303 }
304 template.extend(t);
305 }
306
307 if args.clean_deps {
309 for entry in glob_walker(&format!("{}/*/deps", output.display()), false)
310 .unwrap_or_else(|e| fatal!("cannot cleanup deps in `{}`, {e}", output.display()))
311 {
312 let dir = entry.unwrap_or_else(|e| fatal!("cannot cleanup deps, {e}")).into_path();
313 println_v!("removing `{}` to clean dependencies", dir.display());
314
315 if let Err(e) = std::fs::remove_dir_all(&dir)
316 && !matches!(e.kind(), io::ErrorKind::NotFound)
317 {
318 error!("cannot remove `{}`, {e}", dir.display());
319 }
320 }
321 }
322
323 let mut local = vec![];
325 if !args.no_deps {
326 let mut count = 0;
327 let (workspace_root, deps) = util::dependencies(&args.manifest_path);
328 for dep in deps {
329 if dep.version.pre.as_str() == "local" && dep.manifest_path.starts_with(&workspace_root) {
330 local.push(dep);
331 continue;
332 }
333
334 let dep_l10n = dep.manifest_path.with_file_name("l10n");
335 let dep_l10n_reader = match fs::read_dir(&dep_l10n) {
336 Ok(d) => d,
337 Err(e) => {
338 if !matches!(e.kind(), io::ErrorKind::NotFound) {
339 error!("cannot read `{}`, {e}", dep_l10n.display());
340 }
341 continue;
342 }
343 };
344
345 let mut any = false;
346
347 let mut l10n_dir = |lang: Option<&std::ffi::OsStr>| {
349 any = true;
350 let dir = output.join(lang.unwrap()).join("deps");
351
352 let ignore_file = dir.join(".gitignore");
353
354 if !ignore_file.exists() {
355 (|| -> io::Result<()> {
357 util::check_or_create_dir_all(args.check, &dir)?;
358
359 let mut ignore = "# Dependency localization files\n".to_owned();
360
361 let output = Path::new(&output);
362 let custom_output = if output != Path::new(&args.manifest_path).with_file_name("l10n") {
363 format!(
364 " --output \"{}\"",
365 output.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(output).display()
366 )
367 .replace('\\', "/")
368 } else {
369 String::new()
370 };
371 if !args.package.is_empty() {
372 writeln!(
373 &mut ignore,
374 "# Call `cargo zng l10n --package {}{custom_output} --no-pkg --no-local --clean-deps` to update",
375 args.package
376 )
377 .unwrap();
378 } else {
379 let path = Path::new(&args.manifest_path);
380 let path = path.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(path);
381 writeln!(
382 &mut ignore,
383 "# Call `cargo zng l10n --manifest-path \"{}\" --no-pkg --no-local --clean-deps` to update",
384 path.display()
385 )
386 .unwrap();
387 }
388 writeln!(&mut ignore).unwrap();
389 writeln!(&mut ignore, "*").unwrap();
390 writeln!(&mut ignore, "!.gitignore").unwrap();
391
392 if let Err(e) = fs::write(&ignore_file, ignore.as_bytes()) {
393 fatal!("cannot write `{}`, {e}", ignore_file.display())
394 }
395
396 Ok(())
397 })()
398 .unwrap_or_else(|e| fatal!("cannot create `{}`, {e}", output.display()));
399 }
400
401 let dir = dir.join(&dep.name).join(dep.version.to_string());
402 let _ = util::check_or_create_dir_all(args.check, &dir);
403
404 dir
405 };
406
407 let mut reexport_deps = vec![];
409
410 for dep_l10n_entry in dep_l10n_reader {
411 let dep_l10n_entry = match dep_l10n_entry {
412 Ok(e) => e.path(),
413 Err(e) => {
414 error!("cannot read `{}` entry, {e}", dep_l10n.display());
415 continue;
416 }
417 };
418 if dep_l10n_entry.is_dir() {
419 let output_dir = l10n_dir(dep_l10n_entry.file_name());
421 let _ = util::check_or_create_dir_all(args.check, &output_dir);
422
423 let lang_dir_reader = match fs::read_dir(&dep_l10n_entry) {
424 Ok(d) => d,
425 Err(e) => {
426 error!("cannot read `{}`, {e}", dep_l10n_entry.display());
427 continue;
428 }
429 };
430
431 for lang_entry in lang_dir_reader {
432 let lang_entry = match lang_entry {
433 Ok(e) => e.path(),
434 Err(e) => {
435 error!("cannot read `{}` entry, {e}", dep_l10n_entry.display());
436 continue;
437 }
438 };
439
440 if lang_entry.is_dir() {
441 if lang_entry.file_name().map(|n| n == "deps").unwrap_or(false) {
442 reexport_deps.push((&dep, lang_entry));
443 }
444 } else if lang_entry.is_file() && lang_entry.extension().map(|e| e == "ftl").unwrap_or(false) {
445 let _ = util::check_or_create_dir_all(args.check, &output_dir);
446 let to = output_dir.join(lang_entry.file_name().unwrap());
447 if let Err(e) = util::check_or_copy(args.check, &lang_entry, &to) {
448 error!("cannot copy `{}` to `{}`, {e}", lang_entry.display(), to.display());
449 continue;
450 }
451 }
452 }
453 }
454 }
455
456 reexport_deps.sort_by(|a, b| match a.0.name.cmp(&b.0.name) {
457 Ordering::Equal => b.0.version.cmp(&a.0.version),
458 o => o,
459 });
460
461 for (_, deps) in reexport_deps {
462 let target = l10n_dir(deps.parent().and_then(|p| p.file_name()));
464
465 for entry in glob_walker(&deps.join("*/*/*.ftl").display().to_string(), false).unwrap() {
467 let entry = entry
468 .unwrap_or_else(|e| fatal!("cannot read `{}` entry, {e}", deps.display()))
469 .into_path();
470 let target = target.join(entry.strip_prefix(&deps).unwrap());
471 if !target.exists()
472 && entry.is_file()
473 && let Err(e) = util::check_or_copy(args.check, &entry, &target)
474 {
475 error!("cannot copy `{}` to `{}`, {e}", entry.display(), target.display());
476 }
477 }
478 }
479
480 count += any as u32;
481 }
482 println!("found {count} dependencies with localization");
483 }
484
485 if !args.no_local {
487 for dep in local {
488 let manifest_path = dep.manifest_path.display().to_string();
489 let input = manifest_path.replace('\\', "/");
490 let input = input.strip_suffix("/Cargo.toml").unwrap();
491 let input = format!("{input}/src/**/*.rs");
492 check_scrap_package(
493 &L10nArgs {
494 input: String::new(),
495 output: String::new(),
496 package: String::new(),
497 manifest_path,
498 no_deps: true,
499 no_local: true,
500 no_pkg: false,
501 clean_deps: false,
502 clean_template: false,
503 clean: false,
504 macros: args.macros.clone(),
505 pseudo: String::new(),
506 pseudo_m: String::new(),
507 pseudo_w: String::new(),
508 release_langs: String::new(),
509 translate: String::new(),
510 translate_from: String::new(),
511 translate_to: String::new(),
512 translate_replace: false,
513 translate_existing: false,
514 check: args.check,
515 check_strict: args.check_strict,
516 },
517 &input,
518 output,
519 template,
520 )
521 }
522 }
523}
524
525fn run_others(args: &L10nArgs) {
526 if !args.release_langs.is_empty() {
527 crate::res::built_in::release_langs(Path::new(&args.release_langs));
528 return;
529 }
530 if !args.pseudo.is_empty() {
531 pseudo::pseudo(&args.pseudo, args.check);
532 }
533 if !args.pseudo_m.is_empty() {
534 pseudo::pseudo_mirr(&args.pseudo_m, args.check);
535 }
536 if !args.pseudo_w.is_empty() {
537 pseudo::pseudo_wide(&args.pseudo_w, args.check);
538 }
539 if !args.translate.is_empty() {
540 translate::translate(
541 &args.translate,
542 &args.translate_from,
543 &args.translate_to,
544 args.translate_replace,
545 args.translate_existing,
546 args.check,
547 );
548 }
549}
550
551fn check_fluent_output(args: &L10nArgs, output: &Path) {
552 let read_dir = match fs::read_dir(output) {
553 Ok(d) => d,
554 Err(e) if matches!(e.kind(), io::ErrorKind::NotFound) => {
555 eprintln_v!("no fluent files to check, `{}` not found", output.display());
556 return;
557 }
558 Err(e) => fatal!("cannot read `{}`, {e}", output.display()),
559 };
560
561 let mut template = None;
563 let mut langs = vec![];
564 for lang_dir in read_dir {
565 let lang_dir = lang_dir
566 .unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", output.display()))
567 .path();
568 if lang_dir.is_dir() {
569 let mut files = vec![];
570
571 for file in fs::read_dir(&lang_dir).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())) {
572 let file = file.unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())).path();
573 if file.is_file() {
574 let content = fs::read_to_string(&file).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", file.display()));
575 let content = match fluent_syntax::parser::parse(content.as_str()) {
576 Ok(r) => r,
577 Err((_, errors)) => {
578 let e = FluentParserErrors(errors);
579 error!("cannot parse `{}`\n{e}", file.display());
580 continue;
581 }
582 };
583
584 let mut keys = vec![];
585 for entry in content.body {
586 if let fluent_syntax::ast::Entry::Message(m) = entry {
587 let key = m.id.name.to_owned();
588 keys.push((key, m.value.is_some()));
589 for attr in m.attributes {
590 keys.push((format!("{}.{}", m.id.name, attr.id.name), true));
591 }
592 }
593 }
594
595 files.push((file.file_name().unwrap().to_owned(), keys));
596 }
597 }
598
599 if lang_dir.file_name().unwrap() == "template" {
600 assert!(template.is_none());
601 template = Some(files);
602 } else {
603 langs.push((lang_dir, files));
604 }
605 }
606 }
607 if util::is_failed_run() {
608 return;
609 }
610
611 if let Some(template) = template {
613 if langs.is_empty() {
614 eprintln_v!("no fluent files to compare with template");
615 } else {
616 let template = template
618 .into_iter()
619 .map(|(k, v)| (k, v.into_iter().collect::<HashMap<_, _>>()))
620 .collect::<HashMap<_, _>>();
621
622 for (lang, files) in langs {
623 for (file, messages) in &files {
625 let mut errors = vec![];
626 if let Some(template_msgs) = template.get(file) {
627 for (id, has_value) in messages {
628 if let Some(template_has_value) = template_msgs.get(id) {
629 if has_value != template_has_value {
630 if *has_value {
631 errors.push(format!("unexpected value, `{id}` has no value in template"));
632 } else if args.check_strict {
633 errors.push(format!("missing value, `{id}` has value in template"));
634 }
635 }
636 } else {
637 errors.push(format!("unknown id, `{id}` not found in template file"));
638 }
639 }
640 if args.check_strict {
641 for template_id in template_msgs.keys() {
642 if !messages.iter().any(|(i, _)| i == template_id) {
643 errors.push(format!("missing id, `{template_id}` not found in localized file"));
644 }
645 }
646 }
647 } else {
648 errors.push("template file not found".to_owned());
649 }
650 if !errors.is_empty() {
651 let lang_path = Path::new(lang.file_name().unwrap()).join(file);
652 let template_path = Path::new("template").join(file);
653 let mut msg = format!("`{}` does not match `{}`\n", lang_path.display(), template_path.display());
654 for error in errors {
655 msg.push_str(" ");
656 msg.push_str(&error);
657 msg.push('\n');
658 }
659 error!("{msg}");
660 }
661 }
662 if args.check_strict {
663 for template_file in template.keys() {
664 if !files.iter().any(|(f, _)| f == template_file) {
665 let lang_path = Path::new(lang.file_name().unwrap()).join(template_file);
666 let template_path = Path::new("template").join(template_file);
667 error!(
668 "`{}` does not match `{}`\n localized file not found",
669 lang_path.display(),
670 template_path.display()
671 );
672 }
673 }
674 }
675 }
676 }
677 } else {
678 eprintln_v!("no template to compare, `{}` not found", output.join("template").display());
679 }
680}
681struct FluentParserErrors(Vec<fluent_syntax::parser::ParserError>);
682impl fmt::Display for FluentParserErrors {
683 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
684 let mut sep = "";
685 for e in &self.0 {
686 write!(f, " {sep}{e}")?;
687 sep = "\n";
688 }
689 Ok(())
690 }
691}