Skip to main content

cargo_zng/
l10n.rs

1//! Localization text scrapper.
2//!
3//! See the [`l10n!`] documentation for more details.
4//!
5//! [`l10n!`]: https://zng-ui.github.io/doc/zng/l10n/macro.l10n.html#scrap-template
6
7use std::{
8    cmp::Ordering,
9    collections::{HashMap, HashSet},
10    fmt::{self, Write as _},
11    fs,
12    io::{self, BufRead},
13    path::{Path, PathBuf},
14};
15
16use clap::*;
17
18use crate::{
19    l10n::scraper::FluentTemplate,
20    util::{self, glob_walker},
21};
22
23mod scraper;
24
25pub(crate) mod generate_util;
26mod pseudo;
27mod translate;
28
29#[derive(Args, Debug)]
30pub struct L10nArgs {
31    /// Rust files glob or directory
32    #[arg(short, long, default_value = "", value_name = "PATH", hide_default_value = true)]
33    input: String,
34
35    /// L10n resources dir
36    #[arg(short, long, default_value = "", value_name = "DIR", hide_default_value = true)]
37    output: String,
38
39    /// Package to scrap and copy dependencies
40    ///
41    /// If set the --input and --output default is src/**.rs and l10n/
42    #[arg(short, long, default_value = "", hide_default_value = true)]
43    package: String,
44
45    /// Path to Cargo.toml of crate to scrap and copy dependencies
46    ///
47    /// If set the --input and --output default to src/**.rs and l10n/
48    #[arg(long, default_value = "", hide_default_value = true)]
49    manifest_path: String,
50
51    /// Don't copy dependencies localization
52    ///
53    /// Use with --package or --manifest-path to not copy {dep-pkg}/l10n/*.ftl files
54    #[arg(long, action)]
55    no_deps: bool,
56
57    /// Don't scrap `#.#.#-local` dependencies
58    ///
59    /// Use with --package or --manifest-path to not scrap local dependencies.
60    #[arg(long, action)]
61    no_local: bool,
62
63    /// Don't scrap the target package.
64    ///
65    /// Use with --package or --manifest-path to only scrap dependencies.
66    #[arg(long, action)]
67    no_pkg: bool,
68
69    /// Remove all previously copied dependency localization files.
70    #[arg(long, action)]
71    clean_deps: bool,
72
73    /// Remove all previously scraped resources before scraping.
74    #[arg(long, action)]
75    clean_template: bool,
76
77    /// Same as --clean-deps --clean-template
78    #[arg(long, action)]
79    clean: bool,
80
81    /// Custom l10n macro names, comma separated
82    #[arg(short, long, default_value = "", hide_default_value = true)]
83    macros: String,
84
85    /// Generate pseudo locale from dir/lang
86    ///
87    /// EXAMPLE
88    ///
89    /// "l10n/en" generates pseudo from "l10n/en/**/*.ftl" to "l10n/pseudo"
90    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
91    pseudo: String,
92    /// Generate pseudo mirrored locale
93    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
94    pseudo_m: String,
95    /// Generate pseudo wide locale
96    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
97    pseudo_w: String,
98
99    /// Output comma separated list of langs that would be included by 'l10n
100    /// sourcing localization from the given PATH
101    ///
102    /// See cargo zng res --tool l10n for details
103    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
104    release_langs: String,
105
106    /// Machine translate locale from dir/lang
107    ///
108    /// EXAMPLE
109    ///
110    /// "l10n/template" translates from "l10n/template/**/*.ftl" to a folder for each --translate-to language
111    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
112    translate: String,
113
114    /// Explicit source language for --translate
115    ///
116    /// By default is the source folder name, or English for `template`
117    #[arg(long, default_value = "", value_name = "LANG", hide_default_value = true)]
118    translate_from: String,
119
120    /// Target languages for --translate
121    ///
122    /// The output folder for each is formatted $lang-machine
123    #[arg(
124        long,
125        default_value = "ar,bg,ca,cs,da,de,el,en,es-419,es-ES,et,eu,fi,fr-CA,fr-FR,gl,he,hi,hr,hu,id,it,ja,ko,lt,lv,nb,nl,pl,pt-BR,pt-PT,ro,ru,sk,sl,sr-Latn,sv,th,tr,uk,vi,zh-Hans,zh-Hant",
126        value_name = "LANGS"
127    )]
128    translate_to: String,
129
130    /// Replace all existing machine translations with --translate
131    ///
132    /// By default only replaces stale translations
133    #[arg(long, action)]
134    translate_replace: bool,
135
136    /// If should not skip a --translate-to lang that matches a non-machine translation
137    ///
138    /// By default only translates missing languages
139    ///
140    /// Note that the non-machine translation is not touched, a "$lang-machine" is created beside it
141    #[arg(long, action)]
142    translate_existing: bool,
143
144    /// Verify that packages are scrapped and validate Fluent files
145    #[arg(long, action)]
146    check: bool,
147
148    /// Require that all template keys be present in all localized files
149    #[arg(long, action)]
150    check_strict: bool,
151}
152
153pub fn run(mut args: L10nArgs) {
154    if !args.package.is_empty() && !args.manifest_path.is_empty() {
155        fatal!("only one of --package --manifest-path must be set")
156    }
157
158    if args.check_strict {
159        args.check = true;
160    }
161
162    let mut input = String::new();
163    let mut output = args.output.replace('\\', "/");
164
165    if !args.input.is_empty() {
166        input = args.input.replace('\\', "/");
167
168        if !input.contains('*') && PathBuf::from(&input).is_dir() {
169            input = format!("{}/**/*.rs", input.trim_end_matches('/'));
170        }
171    }
172    if !args.package.is_empty() {
173        if let Some(m) = crate::util::manifest_path_from_package(&args.package) {
174            args.manifest_path = m;
175        } else {
176            fatal!("package `{}` not found in workspace", args.package);
177        }
178    }
179
180    if !args.manifest_path.is_empty() {
181        if !Path::new(&args.manifest_path).exists() {
182            fatal!("`{}` does not exist", args.manifest_path)
183        }
184
185        if let Some(path) = args.manifest_path.replace('\\', "/").strip_suffix("/Cargo.toml") {
186            if output.is_empty() {
187                output = format!("{path}/l10n");
188            }
189            if input.is_empty() {
190                input = format!("{path}/src/**/*.rs");
191            }
192        } else {
193            fatal!("expected path to Cargo.toml manifest file");
194        }
195    }
196
197    if args.check {
198        args.clean = false;
199        args.clean_deps = false;
200        args.clean_template = false;
201    } else if args.clean {
202        args.clean_deps = true;
203        args.clean_template = true;
204    }
205
206    println_v!(
207        "input: `{input}`\noutput: `{output}`\nclean_deps: {}\nclean_template: {}",
208        args.clean_deps,
209        args.clean_template
210    );
211
212    if input.is_empty() {
213        return run_others(&args);
214    }
215
216    if output.is_empty() {
217        fatal!("--output is required for --input")
218    }
219
220    let input = input;
221    let output = Path::new(&output);
222
223    let mut template = FluentTemplate::default();
224
225    check_scrap_package(&args, &input, output, &mut template);
226
227    if !template.entries.is_empty() || !template.notes.is_empty() {
228        if let Err(e) = util::check_or_create_dir_all(args.check, output) {
229            fatal!("cannot create dir `{}`, {e}", output.display());
230        }
231
232        let output = output.join("template");
233
234        if let Err(e) = util::check_or_create_dir_all(args.check, &output) {
235            fatal!("cannot create dir `{}`, {e}", output.display());
236        }
237
238        template.sort();
239
240        let mut clean_files = HashSet::new();
241
242        let r = template.write(|file, contents| {
243            let file = format!("{}.ftl", if file.is_empty() { "_" } else { file });
244            let output = output.join(&file);
245            clean_files.insert(file);
246            util::check_or_write(args.check, output, contents)
247        });
248        if let Err(e) = r {
249            fatal!("error writing template files, {e}");
250        }
251
252        if args.clean_template {
253            debug_assert!(!args.check);
254
255            let cleanup = || -> std::io::Result<()> {
256                for entry in std::fs::read_dir(&output)? {
257                    let entry = entry?.path();
258                    if entry.is_file() {
259                        let name = entry.file_prefix().unwrap().to_string_lossy();
260                        if name.ends_with(".ftl") && !clean_files.contains(&*name) {
261                            let mut entry_file = std::fs::File::open(&entry)?;
262                            if let Some(first_line) = std::io::BufReader::new(&mut entry_file).lines().next()
263                                && first_line?.starts_with(FluentTemplate::AUTO_GENERATED_HEADER)
264                            {
265                                drop(entry_file);
266                                std::fs::remove_file(entry)?;
267                            }
268                        }
269                    }
270                }
271                Ok(())
272            };
273            if let Err(e) = cleanup() {
274                error!("failed template cleanup, {e}");
275            }
276        }
277    }
278
279    if args.check {
280        check_fluent_output(&args, output);
281    }
282
283    run_others(&args);
284}
285
286fn check_scrap_package(args: &L10nArgs, input: &str, output: &Path, template: &mut FluentTemplate) {
287    // scrap the target package
288    if !args.no_pkg {
289        if args.check {
290            println!(r#"checking "{input}".."#);
291        } else {
292            println!(r#"scraping "{input}".."#);
293        }
294
295        let custom_macro_names: Vec<&str> = args.macros.split(',').map(|n| n.trim()).collect();
296        let t = scraper::scrape_fluent_text(input, &custom_macro_names);
297        if !args.check {
298            match t.entries.len() {
299                0 => println!("  did not find any entry"),
300                1 => println!("  found 1 entry"),
301                n => println!("  found {n} entries"),
302            }
303        }
304        template.extend(t);
305    }
306
307    // cleanup dependencies
308    if args.clean_deps {
309        for entry in glob_walker(&format!("{}/*/deps", output.display()), false)
310            .unwrap_or_else(|e| fatal!("cannot cleanup deps in `{}`, {e}", output.display()))
311        {
312            let dir = entry.unwrap_or_else(|e| fatal!("cannot cleanup deps, {e}")).into_path();
313            println_v!("removing `{}` to clean dependencies", dir.display());
314
315            if let Err(e) = std::fs::remove_dir_all(&dir)
316                && !matches!(e.kind(), io::ErrorKind::NotFound)
317            {
318                error!("cannot remove `{}`, {e}", dir.display());
319            }
320        }
321    }
322
323    // collect dependencies
324    let mut local = vec![];
325    if !args.no_deps {
326        let mut count = 0;
327        let (workspace_root, deps) = util::dependencies(&args.manifest_path);
328        for dep in deps {
329            if dep.version.pre.as_str() == "local" && dep.manifest_path.starts_with(&workspace_root) {
330                local.push(dep);
331                continue;
332            }
333
334            let dep_l10n = dep.manifest_path.with_file_name("l10n");
335            let dep_l10n_reader = match fs::read_dir(&dep_l10n) {
336                Ok(d) => d,
337                Err(e) => {
338                    if !matches!(e.kind(), io::ErrorKind::NotFound) {
339                        error!("cannot read `{}`, {e}", dep_l10n.display());
340                    }
341                    continue;
342                }
343            };
344
345            let mut any = false;
346
347            // get l10n_dir/{lang}/deps/dep.name/dep.version/
348            let mut l10n_dir = |lang: Option<&std::ffi::OsStr>| {
349                any = true;
350                let dir = output.join(lang.unwrap()).join("deps");
351
352                let ignore_file = dir.join(".gitignore");
353
354                if !ignore_file.exists() {
355                    // create dir and .gitignore file
356                    (|| -> io::Result<()> {
357                        util::check_or_create_dir_all(args.check, &dir)?;
358
359                        let mut ignore = "# Dependency localization files\n".to_owned();
360
361                        let output = Path::new(&output);
362                        let custom_output = if output != Path::new(&args.manifest_path).with_file_name("l10n") {
363                            format!(
364                                " --output \"{}\"",
365                                output.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(output).display()
366                            )
367                            .replace('\\', "/")
368                        } else {
369                            String::new()
370                        };
371                        if !args.package.is_empty() {
372                            writeln!(
373                                &mut ignore,
374                                "# Call `cargo zng l10n --package {}{custom_output} --no-pkg --no-local --clean-deps` to update",
375                                args.package
376                            )
377                            .unwrap();
378                        } else {
379                            let path = Path::new(&args.manifest_path);
380                            let path = path.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(path);
381                            writeln!(
382                                &mut ignore,
383                                "# Call `cargo zng l10n --manifest-path \"{}\" --no-pkg --no-local --clean-deps` to update",
384                                path.display()
385                            )
386                            .unwrap();
387                        }
388                        writeln!(&mut ignore).unwrap();
389                        writeln!(&mut ignore, "*").unwrap();
390                        writeln!(&mut ignore, "!.gitignore").unwrap();
391
392                        if let Err(e) = fs::write(&ignore_file, ignore.as_bytes()) {
393                            fatal!("cannot write `{}`, {e}", ignore_file.display())
394                        }
395
396                        Ok(())
397                    })()
398                    .unwrap_or_else(|e| fatal!("cannot create `{}`, {e}", output.display()));
399                }
400
401                let dir = dir.join(&dep.name).join(dep.version.to_string());
402                let _ = util::check_or_create_dir_all(args.check, &dir);
403
404                dir
405            };
406
407            // [(exporter_dep, ".../{lang}?/deps")]
408            let mut reexport_deps = vec![];
409
410            for dep_l10n_entry in dep_l10n_reader {
411                let dep_l10n_entry = match dep_l10n_entry {
412                    Ok(e) => e.path(),
413                    Err(e) => {
414                        error!("cannot read `{}` entry, {e}", dep_l10n.display());
415                        continue;
416                    }
417                };
418                if dep_l10n_entry.is_dir() {
419                    // l10n/{lang}/deps/{dep.name}/{dep.version}
420                    let output_dir = l10n_dir(dep_l10n_entry.file_name());
421                    let _ = util::check_or_create_dir_all(args.check, &output_dir);
422
423                    let lang_dir_reader = match fs::read_dir(&dep_l10n_entry) {
424                        Ok(d) => d,
425                        Err(e) => {
426                            error!("cannot read `{}`, {e}", dep_l10n_entry.display());
427                            continue;
428                        }
429                    };
430
431                    for lang_entry in lang_dir_reader {
432                        let lang_entry = match lang_entry {
433                            Ok(e) => e.path(),
434                            Err(e) => {
435                                error!("cannot read `{}` entry, {e}", dep_l10n_entry.display());
436                                continue;
437                            }
438                        };
439
440                        if lang_entry.is_dir() {
441                            if lang_entry.file_name().map(|n| n == "deps").unwrap_or(false) {
442                                reexport_deps.push((&dep, lang_entry));
443                            }
444                        } else if lang_entry.is_file() && lang_entry.extension().map(|e| e == "ftl").unwrap_or(false) {
445                            let _ = util::check_or_create_dir_all(args.check, &output_dir);
446                            let to = output_dir.join(lang_entry.file_name().unwrap());
447                            if let Err(e) = util::check_or_copy(args.check, &lang_entry, &to) {
448                                error!("cannot copy `{}` to `{}`, {e}", lang_entry.display(), to.display());
449                                continue;
450                            }
451                        }
452                    }
453                }
454            }
455
456            reexport_deps.sort_by(|a, b| match a.0.name.cmp(&b.0.name) {
457                Ordering::Equal => b.0.version.cmp(&a.0.version),
458                o => o,
459            });
460
461            for (_, deps) in reexport_deps {
462                // dep/l10n/lang/deps/
463                let target = l10n_dir(deps.parent().and_then(|p| p.file_name()));
464
465                // deps/pkg-name/pkg-version/*.ftl
466                for entry in glob_walker(&deps.join("*/*/*.ftl").display().to_string(), false).unwrap() {
467                    let entry = entry
468                        .unwrap_or_else(|e| fatal!("cannot read `{}` entry, {e}", deps.display()))
469                        .into_path();
470                    let target = target.join(entry.strip_prefix(&deps).unwrap());
471                    if !target.exists()
472                        && entry.is_file()
473                        && let Err(e) = util::check_or_copy(args.check, &entry, &target)
474                    {
475                        error!("cannot copy `{}` to `{}`, {e}", entry.display(), target.display());
476                    }
477                }
478            }
479
480            count += any as u32;
481        }
482        println!("found {count} dependencies with localization");
483    }
484
485    // scrap local dependencies
486    if !args.no_local {
487        for dep in local {
488            let manifest_path = dep.manifest_path.display().to_string();
489            let input = manifest_path.replace('\\', "/");
490            let input = input.strip_suffix("/Cargo.toml").unwrap();
491            let input = format!("{input}/src/**/*.rs");
492            check_scrap_package(
493                &L10nArgs {
494                    input: String::new(),
495                    output: String::new(),
496                    package: String::new(),
497                    manifest_path,
498                    no_deps: true,
499                    no_local: true,
500                    no_pkg: false,
501                    clean_deps: false,
502                    clean_template: false,
503                    clean: false,
504                    macros: args.macros.clone(),
505                    pseudo: String::new(),
506                    pseudo_m: String::new(),
507                    pseudo_w: String::new(),
508                    release_langs: String::new(),
509                    translate: String::new(),
510                    translate_from: String::new(),
511                    translate_to: String::new(),
512                    translate_replace: false,
513                    translate_existing: false,
514                    check: args.check,
515                    check_strict: args.check_strict,
516                },
517                &input,
518                output,
519                template,
520            )
521        }
522    }
523}
524
525fn run_others(args: &L10nArgs) {
526    if !args.release_langs.is_empty() {
527        crate::res::built_in::release_langs(Path::new(&args.release_langs));
528        return;
529    }
530    if !args.pseudo.is_empty() {
531        pseudo::pseudo(&args.pseudo, args.check);
532    }
533    if !args.pseudo_m.is_empty() {
534        pseudo::pseudo_mirr(&args.pseudo_m, args.check);
535    }
536    if !args.pseudo_w.is_empty() {
537        pseudo::pseudo_wide(&args.pseudo_w, args.check);
538    }
539    if !args.translate.is_empty() {
540        translate::translate(
541            &args.translate,
542            &args.translate_from,
543            &args.translate_to,
544            args.translate_replace,
545            args.translate_existing,
546            args.check,
547        );
548    }
549}
550
551fn check_fluent_output(args: &L10nArgs, output: &Path) {
552    let read_dir = match fs::read_dir(output) {
553        Ok(d) => d,
554        Err(e) if matches!(e.kind(), io::ErrorKind::NotFound) => {
555            eprintln_v!("no fluent files to check, `{}` not found", output.display());
556            return;
557        }
558        Err(e) => fatal!("cannot read `{}`, {e}", output.display()),
559    };
560
561    // validate syntax of */*.ftl and collect entry keys
562    let mut template = None;
563    let mut langs = vec![];
564    for lang_dir in read_dir {
565        let lang_dir = lang_dir
566            .unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", output.display()))
567            .path();
568        if lang_dir.is_dir() {
569            let mut files = vec![];
570
571            for file in fs::read_dir(&lang_dir).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())) {
572                let file = file.unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())).path();
573                if file.is_file() {
574                    let content = fs::read_to_string(&file).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", file.display()));
575                    let content = match fluent_syntax::parser::parse(content.as_str()) {
576                        Ok(r) => r,
577                        Err((_, errors)) => {
578                            let e = FluentParserErrors(errors);
579                            error!("cannot parse `{}`\n{e}", file.display());
580                            continue;
581                        }
582                    };
583
584                    let mut keys = vec![];
585                    for entry in content.body {
586                        if let fluent_syntax::ast::Entry::Message(m) = entry {
587                            let key = m.id.name.to_owned();
588                            keys.push((key, m.value.is_some()));
589                            for attr in m.attributes {
590                                keys.push((format!("{}.{}", m.id.name, attr.id.name), true));
591                            }
592                        }
593                    }
594
595                    files.push((file.file_name().unwrap().to_owned(), keys));
596                }
597            }
598
599            if lang_dir.file_name().unwrap() == "template" {
600                assert!(template.is_none());
601                template = Some(files);
602            } else {
603                langs.push((lang_dir, files));
604            }
605        }
606    }
607    if util::is_failed_run() {
608        return;
609    }
610
611    // check
612    if let Some(template) = template {
613        if langs.is_empty() {
614            eprintln_v!("no fluent files to compare with template");
615        } else {
616            // faster template lookup
617            let template = template
618                .into_iter()
619                .map(|(k, v)| (k, v.into_iter().collect::<HashMap<_, _>>()))
620                .collect::<HashMap<_, _>>();
621
622            for (lang, files) in langs {
623                // match localized against template
624                for (file, messages) in &files {
625                    let mut errors = vec![];
626                    if let Some(template_msgs) = template.get(file) {
627                        for (id, has_value) in messages {
628                            if let Some(template_has_value) = template_msgs.get(id) {
629                                if has_value != template_has_value {
630                                    if *has_value {
631                                        errors.push(format!("unexpected value, `{id}` has no value in template"));
632                                    } else if args.check_strict {
633                                        errors.push(format!("missing value, `{id}` has value in template"));
634                                    }
635                                }
636                            } else {
637                                errors.push(format!("unknown id, `{id}` not found in template file"));
638                            }
639                        }
640                        if args.check_strict {
641                            for template_id in template_msgs.keys() {
642                                if !messages.iter().any(|(i, _)| i == template_id) {
643                                    errors.push(format!("missing id, `{template_id}` not found in localized file"));
644                                }
645                            }
646                        }
647                    } else {
648                        errors.push("template file not found".to_owned());
649                    }
650                    if !errors.is_empty() {
651                        let lang_path = Path::new(lang.file_name().unwrap()).join(file);
652                        let template_path = Path::new("template").join(file);
653                        let mut msg = format!("`{}` does not match `{}`\n", lang_path.display(), template_path.display());
654                        for error in errors {
655                            msg.push_str("  ");
656                            msg.push_str(&error);
657                            msg.push('\n');
658                        }
659                        error!("{msg}");
660                    }
661                }
662                if args.check_strict {
663                    for template_file in template.keys() {
664                        if !files.iter().any(|(f, _)| f == template_file) {
665                            let lang_path = Path::new(lang.file_name().unwrap()).join(template_file);
666                            let template_path = Path::new("template").join(template_file);
667                            error!(
668                                "`{}` does not match `{}`\n   localized file not found",
669                                lang_path.display(),
670                                template_path.display()
671                            );
672                        }
673                    }
674                }
675            }
676        }
677    } else {
678        eprintln_v!("no template to compare, `{}` not found", output.join("template").display());
679    }
680}
681struct FluentParserErrors(Vec<fluent_syntax::parser::ParserError>);
682impl fmt::Display for FluentParserErrors {
683    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
684        let mut sep = "";
685        for e in &self.0 {
686            write!(f, "  {sep}{e}")?;
687            sep = "\n";
688        }
689        Ok(())
690    }
691}