cargo_zng/
l10n.rs

1//! Localization text scrapper.
2//!
3//! See the [`l10n!`] documentation for more details.
4//!
5//! [`l10n!`]: https://zng-ui.github.io/doc/zng/l10n/macro.l10n.html#scrap-template
6
7use std::{
8    cmp::Ordering,
9    collections::{HashMap, HashSet},
10    fmt::{self, Write as _},
11    fs,
12    io::{self, BufRead},
13    path::{Path, PathBuf},
14};
15
16use clap::*;
17
18use crate::{l10n::scraper::FluentTemplate, util};
19
20mod scraper;
21
22mod generate_util;
23mod pseudo;
24#[derive(Args, Debug)]
25pub struct L10nArgs {
26    /// Rust files glob or directory
27    #[arg(short, long, default_value = "", value_name = "PATH", hide_default_value = true)]
28    input: String,
29
30    /// L10n resources dir
31    #[arg(short, long, default_value = "", value_name = "DIR", hide_default_value = true)]
32    output: String,
33
34    /// Package to scrap and copy dependencies
35    ///
36    /// If set the --input and --output default is src/**.rs and l10n/
37    #[arg(short, long, default_value = "", hide_default_value = true)]
38    package: String,
39
40    /// Path to Cargo.toml of crate to scrap and copy dependencies
41    ///
42    /// If set the --input and --output default to src/**.rs and l10n/
43    #[arg(long, default_value = "", hide_default_value = true)]
44    manifest_path: String,
45
46    /// Don't copy dependencies localization
47    ///
48    /// Use with --package or --manifest-path to not copy {dep-pkg}/l10n/*.ftl files
49    #[arg(long, action)]
50    no_deps: bool,
51
52    /// Don't scrap `#.#.#-local` dependencies
53    ///
54    /// Use with --package or --manifest-path to not scrap local dependencies.
55    #[arg(long, action)]
56    no_local: bool,
57
58    /// Don't scrap the target package.
59    ///
60    /// Use with --package or --manifest-path to only scrap dependencies.
61    #[arg(long, action)]
62    no_pkg: bool,
63
64    /// Remove all previously copied dependency localization files.
65    #[arg(long, action)]
66    clean_deps: bool,
67
68    /// Remove all previously scraped resources before scraping.
69    #[arg(long, action)]
70    clean_template: bool,
71
72    /// Same as --clean-deps --clean-template
73    #[arg(long, action)]
74    clean: bool,
75
76    /// Custom l10n macro names, comma separated
77    #[arg(short, long, default_value = "", hide_default_value = true)]
78    macros: String,
79
80    /// Generate pseudo locale from dir/lang
81    ///
82    /// EXAMPLE
83    ///
84    /// "l10n/en" generates pseudo from "l10n/en/**/*.ftl" to "l10n/pseudo"
85    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
86    pseudo: String,
87    /// Generate pseudo mirrored locale
88    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
89    pseudo_m: String,
90    /// Generate pseudo wide locale
91    #[arg(long, default_value = "", value_name = "PATH", hide_default_value = true)]
92    pseudo_w: String,
93
94    /// Verify that packages are scrapped and validate Fluent files
95    #[arg(long, action)]
96    check: bool,
97
98    /// Require that all template keys be present in all localized files
99    #[arg(long, action)]
100    check_strict: bool,
101
102    /// Use verbose output.
103    #[arg(short, long, action)]
104    verbose: bool,
105}
106
107pub fn run(mut args: L10nArgs) {
108    if !args.package.is_empty() && !args.manifest_path.is_empty() {
109        fatal!("only one of --package --manifest-path must be set")
110    }
111
112    if args.check_strict {
113        args.check = true;
114    }
115
116    let mut input = String::new();
117    let mut output = args.output.replace('\\', "/");
118
119    if !args.input.is_empty() {
120        input = args.input.replace('\\', "/");
121
122        if !input.contains('*') && PathBuf::from(&input).is_dir() {
123            input = format!("{}/**/*.rs", input.trim_end_matches('/'));
124        }
125    }
126    if !args.package.is_empty() {
127        if let Some(m) = crate::util::manifest_path_from_package(&args.package) {
128            args.manifest_path = m;
129        } else {
130            fatal!("package `{}` not found in workspace", args.package);
131        }
132    }
133
134    if !args.manifest_path.is_empty() {
135        if !Path::new(&args.manifest_path).exists() {
136            fatal!("`{}` does not exist", args.manifest_path)
137        }
138
139        if let Some(path) = args.manifest_path.replace('\\', "/").strip_suffix("/Cargo.toml") {
140            if output.is_empty() {
141                output = format!("{path}/l10n");
142            }
143            if input.is_empty() {
144                input = format!("{path}/src/**/*.rs");
145            }
146        } else {
147            fatal!("expected path to Cargo.toml manifest file");
148        }
149    }
150
151    if args.check {
152        args.clean = false;
153        args.clean_deps = false;
154        args.clean_template = false;
155    } else if args.clean {
156        args.clean_deps = true;
157        args.clean_template = true;
158    }
159
160    if args.verbose {
161        println!(
162            "input: `{input}`\noutput: `{output}`\nclean_deps: {}\nclean_template: {}",
163            args.clean_deps, args.clean_template
164        );
165    }
166
167    if input.is_empty() {
168        return run_generators(&args);
169    }
170
171    if output.is_empty() {
172        fatal!("--output is required for --input")
173    }
174
175    let input = input;
176    let output = Path::new(&output);
177
178    let mut template = FluentTemplate::default();
179
180    check_scrap_package(&args, &input, output, &mut template);
181
182    if !template.entries.is_empty() || !template.notes.is_empty() {
183        if let Err(e) = util::check_or_create_dir_all(args.check, output) {
184            fatal!("cannot create dir `{}`, {e}", output.display());
185        }
186
187        let output = output.join("template");
188
189        if let Err(e) = util::check_or_create_dir_all(args.check, &output) {
190            fatal!("cannot create dir `{}`, {e}", output.display());
191        }
192
193        template.sort();
194
195        let mut clean_files = HashSet::new();
196
197        let r = template.write(|file, contents| {
198            let file = format!("{}.ftl", if file.is_empty() { "_" } else { file });
199            let output = output.join(&file);
200            clean_files.insert(file);
201            util::check_or_write(args.check, output, contents, args.verbose)
202        });
203        if let Err(e) = r {
204            fatal!("error writing template files, {e}");
205        }
206
207        if args.clean_template {
208            debug_assert!(!args.check);
209
210            let cleanup = || -> std::io::Result<()> {
211                for entry in std::fs::read_dir(&output)? {
212                    let entry = entry?.path();
213                    if entry.is_file() {
214                        let name = entry.file_prefix().unwrap().to_string_lossy();
215                        if name.ends_with(".ftl") && !clean_files.contains(&*name) {
216                            let mut entry_file = std::fs::File::open(&entry)?;
217                            if let Some(first_line) = std::io::BufReader::new(&mut entry_file).lines().next()
218                                && first_line?.starts_with(FluentTemplate::AUTO_GENERATED_HEADER)
219                            {
220                                drop(entry_file);
221                                std::fs::remove_file(entry)?;
222                            }
223                        }
224                    }
225                }
226                Ok(())
227            };
228            if let Err(e) = cleanup() {
229                error!("failed template cleanup, {e}");
230            }
231        }
232    }
233
234    if args.check {
235        check_fluent_output(&args, output);
236    }
237
238    run_generators(&args);
239}
240
241fn check_scrap_package(args: &L10nArgs, input: &str, output: &Path, template: &mut FluentTemplate) {
242    // scrap the target package
243    if !args.no_pkg {
244        if args.check {
245            println!(r#"checking "{input}".."#);
246        } else {
247            println!(r#"scraping "{input}".."#);
248        }
249
250        let custom_macro_names: Vec<&str> = args.macros.split(',').map(|n| n.trim()).collect();
251        let t = scraper::scrape_fluent_text(input, &custom_macro_names);
252        if !args.check {
253            match t.entries.len() {
254                0 => println!("  did not find any entry"),
255                1 => println!("  found 1 entry"),
256                n => println!("  found {n} entries"),
257            }
258        }
259        template.extend(t);
260    }
261
262    // cleanup dependencies
263    if args.clean_deps {
264        for entry in glob::glob(&format!("{}/*/deps", output.display()))
265            .unwrap_or_else(|e| fatal!("cannot cleanup deps in `{}`, {e}", output.display()))
266        {
267            let dir = entry.unwrap_or_else(|e| fatal!("cannot cleanup deps, {e}"));
268            if args.verbose {
269                println!("removing `{}` to clean dependencies", dir.display());
270            }
271            if let Err(e) = std::fs::remove_dir_all(&dir)
272                && !matches!(e.kind(), io::ErrorKind::NotFound)
273            {
274                error!("cannot remove `{}`, {e}", dir.display());
275            }
276        }
277    }
278
279    // collect dependencies
280    let mut local = vec![];
281    if !args.no_deps {
282        let mut count = 0;
283        let (workspace_root, deps) = util::dependencies(&args.manifest_path);
284        for dep in deps {
285            if dep.version.pre.as_str() == "local" && dep.manifest_path.starts_with(&workspace_root) {
286                local.push(dep);
287                continue;
288            }
289
290            let dep_l10n = dep.manifest_path.with_file_name("l10n");
291            let dep_l10n_reader = match fs::read_dir(&dep_l10n) {
292                Ok(d) => d,
293                Err(e) => {
294                    if !matches!(e.kind(), io::ErrorKind::NotFound) {
295                        error!("cannot read `{}`, {e}", dep_l10n.display());
296                    }
297                    continue;
298                }
299            };
300
301            let mut any = false;
302
303            // get l10n_dir/{lang}/deps/dep.name/dep.version/
304            let mut l10n_dir = |lang: Option<&std::ffi::OsStr>| {
305                any = true;
306                let dir = output.join(lang.unwrap()).join("deps");
307
308                let ignore_file = dir.join(".gitignore");
309
310                if !ignore_file.exists() {
311                    // create dir and .gitignore file
312                    (|| -> io::Result<()> {
313                        util::check_or_create_dir_all(args.check, &dir)?;
314
315                        let mut ignore = "# Dependency localization files\n".to_owned();
316
317                        let output = Path::new(&output);
318                        let custom_output = if output != Path::new(&args.manifest_path).with_file_name("l10n") {
319                            format!(
320                                " --output \"{}\"",
321                                output.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(output).display()
322                            )
323                            .replace('\\', "/")
324                        } else {
325                            String::new()
326                        };
327                        if !args.package.is_empty() {
328                            writeln!(
329                                &mut ignore,
330                                "# Call `cargo zng l10n --package {}{custom_output} --no-pkg --no-local --clean-deps` to update",
331                                args.package
332                            )
333                            .unwrap();
334                        } else {
335                            let path = Path::new(&args.manifest_path);
336                            let path = path.strip_prefix(std::env::current_dir().unwrap()).unwrap_or(path);
337                            writeln!(
338                                &mut ignore,
339                                "# Call `cargo zng l10n --manifest-path \"{}\" --no-pkg --no-local --clean-deps` to update",
340                                path.display()
341                            )
342                            .unwrap();
343                        }
344                        writeln!(&mut ignore).unwrap();
345                        writeln!(&mut ignore, "*").unwrap();
346                        writeln!(&mut ignore, "!.gitignore").unwrap();
347
348                        if let Err(e) = fs::write(&ignore_file, ignore.as_bytes()) {
349                            fatal!("cannot write `{}`, {e}", ignore_file.display())
350                        }
351
352                        Ok(())
353                    })()
354                    .unwrap_or_else(|e| fatal!("cannot create `{}`, {e}", output.display()));
355                }
356
357                let dir = dir.join(&dep.name).join(dep.version.to_string());
358                let _ = util::check_or_create_dir_all(args.check, &dir);
359
360                dir
361            };
362
363            // [(exporter_dep, ".../{lang}?/deps")]
364            let mut reexport_deps = vec![];
365
366            for dep_l10n_entry in dep_l10n_reader {
367                let dep_l10n_entry = match dep_l10n_entry {
368                    Ok(e) => e.path(),
369                    Err(e) => {
370                        error!("cannot read `{}` entry, {e}", dep_l10n.display());
371                        continue;
372                    }
373                };
374                if dep_l10n_entry.is_dir() {
375                    // l10n/{lang}/deps/{dep.name}/{dep.version}
376                    let output_dir = l10n_dir(dep_l10n_entry.file_name());
377                    let _ = util::check_or_create_dir_all(args.check, &output_dir);
378
379                    let lang_dir_reader = match fs::read_dir(&dep_l10n_entry) {
380                        Ok(d) => d,
381                        Err(e) => {
382                            error!("cannot read `{}`, {e}", dep_l10n_entry.display());
383                            continue;
384                        }
385                    };
386
387                    for lang_entry in lang_dir_reader {
388                        let lang_entry = match lang_entry {
389                            Ok(e) => e.path(),
390                            Err(e) => {
391                                error!("cannot read `{}` entry, {e}", dep_l10n_entry.display());
392                                continue;
393                            }
394                        };
395
396                        if lang_entry.is_dir() {
397                            if lang_entry.file_name().map(|n| n == "deps").unwrap_or(false) {
398                                reexport_deps.push((&dep, lang_entry));
399                            }
400                        } else if lang_entry.is_file() && lang_entry.extension().map(|e| e == "ftl").unwrap_or(false) {
401                            let _ = util::check_or_create_dir_all(args.check, &output_dir);
402                            let to = output_dir.join(lang_entry.file_name().unwrap());
403                            if let Err(e) = util::check_or_copy(args.check, &lang_entry, &to, args.verbose) {
404                                error!("cannot copy `{}` to `{}`, {e}", lang_entry.display(), to.display());
405                                continue;
406                            }
407                        }
408                    }
409                }
410            }
411
412            reexport_deps.sort_by(|a, b| match a.0.name.cmp(&b.0.name) {
413                Ordering::Equal => b.0.version.cmp(&a.0.version),
414                o => o,
415            });
416
417            for (_, deps) in reexport_deps {
418                // dep/l10n/lang/deps/
419                let target = l10n_dir(deps.parent().and_then(|p| p.file_name()));
420
421                // deps/pkg-name/pkg-version/*.ftl
422                for entry in glob::glob(&deps.join("*/*/*.ftl").display().to_string()).unwrap() {
423                    let entry = entry.unwrap_or_else(|e| fatal!("cannot read `{}` entry, {e}", deps.display()));
424                    let target = target.join(entry.strip_prefix(&deps).unwrap());
425                    if !target.exists()
426                        && entry.is_file()
427                        && let Err(e) = util::check_or_copy(args.check, &entry, &target, args.verbose)
428                    {
429                        error!("cannot copy `{}` to `{}`, {e}", entry.display(), target.display());
430                    }
431                }
432            }
433
434            count += any as u32;
435        }
436        println!("found {count} dependencies with localization");
437    }
438
439    // scrap local dependencies
440    if !args.no_local {
441        for dep in local {
442            let manifest_path = dep.manifest_path.display().to_string();
443            let input = manifest_path.replace('\\', "/");
444            let input = input.strip_suffix("/Cargo.toml").unwrap();
445            let input = format!("{input}/src/**/*.rs");
446            check_scrap_package(
447                &L10nArgs {
448                    input: String::new(),
449                    output: String::new(),
450                    package: String::new(),
451                    manifest_path,
452                    no_deps: true,
453                    no_local: true,
454                    no_pkg: false,
455                    clean_deps: false,
456                    clean_template: false,
457                    clean: false,
458                    macros: args.macros.clone(),
459                    pseudo: String::new(),
460                    pseudo_m: String::new(),
461                    pseudo_w: String::new(),
462                    check: args.check,
463                    check_strict: args.check_strict,
464                    verbose: args.verbose,
465                },
466                &input,
467                output,
468                template,
469            )
470        }
471    }
472}
473
474fn run_generators(args: &L10nArgs) {
475    if !args.pseudo.is_empty() {
476        pseudo::pseudo(&args.pseudo, args.check, args.verbose);
477    }
478    if !args.pseudo_m.is_empty() {
479        pseudo::pseudo_mirr(&args.pseudo_m, args.check, args.verbose);
480    }
481    if !args.pseudo_w.is_empty() {
482        pseudo::pseudo_wide(&args.pseudo_w, args.check, args.verbose);
483    }
484}
485
486fn check_fluent_output(args: &L10nArgs, output: &Path) {
487    let read_dir = match fs::read_dir(output) {
488        Ok(d) => d,
489        Err(e) if matches!(e.kind(), io::ErrorKind::NotFound) => {
490            if args.verbose {
491                eprintln!("no fluent files to check, `{}` not found", output.display());
492            }
493            return;
494        }
495        Err(e) => fatal!("cannot read `{}`, {e}", output.display()),
496    };
497
498    // validate syntax of */*.ftl and collect entry keys
499    let mut template = None;
500    let mut langs = vec![];
501    for lang_dir in read_dir {
502        let lang_dir = lang_dir
503            .unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", output.display()))
504            .path();
505        if lang_dir.is_dir() {
506            let mut files = vec![];
507
508            for file in fs::read_dir(&lang_dir).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())) {
509                let file = file.unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", lang_dir.display())).path();
510                if file.is_file() {
511                    let content = fs::read_to_string(&file).unwrap_or_else(|e| fatal!("cannot read `{}`, {e}", file.display()));
512                    let content = match fluent_syntax::parser::parse(content.as_str()) {
513                        Ok(r) => r,
514                        Err((_, errors)) => {
515                            let e = FluentParserErrors(errors);
516                            error!("cannot parse `{}`\n{e}", file.display());
517                            continue;
518                        }
519                    };
520
521                    let mut keys = vec![];
522                    for entry in content.body {
523                        if let fluent_syntax::ast::Entry::Message(m) = entry {
524                            let key = m.id.name.to_owned();
525                            keys.push((key, m.value.is_some()));
526                            for attr in m.attributes {
527                                keys.push((format!("{}.{}", m.id.name, attr.id.name), true));
528                            }
529                        }
530                    }
531
532                    files.push((file.file_name().unwrap().to_owned(), keys));
533                }
534            }
535
536            if lang_dir.file_name().unwrap() == "template" {
537                assert!(template.is_none());
538                template = Some(files);
539            } else {
540                langs.push((lang_dir, files));
541            }
542        }
543    }
544    if util::is_failed_run() {
545        return;
546    }
547
548    // check
549    if let Some(template) = template {
550        if langs.is_empty() {
551            if args.verbose {
552                eprintln!("no fluent files to compare with template");
553            }
554        } else {
555            // faster template lookup
556            let template = template
557                .into_iter()
558                .map(|(k, v)| (k, v.into_iter().collect::<HashMap<_, _>>()))
559                .collect::<HashMap<_, _>>();
560
561            for (lang, files) in langs {
562                // match localized against template
563                for (file, messages) in &files {
564                    let mut errors = vec![];
565                    if let Some(template_msgs) = template.get(file) {
566                        for (id, has_value) in messages {
567                            if let Some(template_has_value) = template_msgs.get(id) {
568                                if has_value != template_has_value {
569                                    if *has_value {
570                                        errors.push(format!("unexpected value, `{id}` has no value in template"));
571                                    } else if args.check_strict {
572                                        errors.push(format!("missing value, `{id}` has value in template"));
573                                    }
574                                }
575                            } else {
576                                errors.push(format!("unknown id, `{id}` not found in template file"));
577                            }
578                        }
579                        if args.check_strict {
580                            for template_id in template_msgs.keys() {
581                                if !messages.iter().any(|(i, _)| i == template_id) {
582                                    errors.push(format!("missing id, `{template_id}` not found in localized file"));
583                                }
584                            }
585                        }
586                    } else {
587                        errors.push("template file not found".to_owned());
588                    }
589                    if !errors.is_empty() {
590                        let lang_path = Path::new(lang.file_name().unwrap()).join(file);
591                        let template_path = Path::new("template").join(file);
592                        let mut msg = format!("`{}` does not match `{}`\n", lang_path.display(), template_path.display());
593                        for error in errors {
594                            msg.push_str("  ");
595                            msg.push_str(&error);
596                            msg.push('\n');
597                        }
598                        error!("{msg}");
599                    }
600                }
601                if args.check_strict {
602                    for template_file in template.keys() {
603                        if !files.iter().any(|(f, _)| f == template_file) {
604                            let lang_path = Path::new(lang.file_name().unwrap()).join(template_file);
605                            let template_path = Path::new("template").join(template_file);
606                            error!(
607                                "`{}` does not match `{}`\n   localized file not found",
608                                lang_path.display(),
609                                template_path.display()
610                            );
611                        }
612                    }
613                }
614            }
615        }
616    } else if args.verbose {
617        eprintln!("no template to compare, `{}` not found", output.join("template").display());
618    }
619}
620struct FluentParserErrors(Vec<fluent_syntax::parser::ParserError>);
621impl fmt::Display for FluentParserErrors {
622    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
623        let mut sep = "";
624        for e in &self.0 {
625            write!(f, "  {sep}{e}")?;
626            sep = "\n";
627        }
628        Ok(())
629    }
630}
cargo_zng/l10n.rs

cargo_zng/
l10n.rs
