Skip to main content

cargo_zng/res/built_in/
l10n.rs

1use std::{
2    borrow::Cow,
3    collections::{HashMap, HashSet},
4};
5
6use once_cell::unsync::Lazy;
7
8use crate::util::glob_walker;
9
10use super::*;
11
12const L10N_HELP: &str = "
13Copy localization files (.ftl) and optimize for release
14
15The request file:
16  source/l10n 'l10n
17   | # comment
18   | path/dev-l10n
19
20Copies the `path/dev-l10n` dir to:
21  target/l10n
22
23Paths are relative to the Cargo workspace root
24
25Filter:
26
27Only localization files are included
28    **/*.ftl
29
30Development langs are excluded
31    !./pseudo*
32    !./template
33
34Only lang folders that have local translations are included
35    If ./{lang}/deps/** exists but no ./{lang}/*.ftl exists it is excluded
36
37Comments are stripped
38
39Subsetting:
40
41If a l10n subset profile is found is is applied to the dependency localization
42
43The subset profile is an allow list, see the docs `zng::l10n` for how to create one
44
45The subset profile is resolved in this order:
46
47ZNG_L10N_PROFILE_FILE env if is set
48    Must be set to a .subset file path, relative to the Cargo workspace root
49    If the file is a {name}.rec.subset auto includes a {name}.subset and vice versa 
50
51res/optimization-profiles/zng-ext-l10n.rec.subset
52    Default location, also includes zng-ext-l10n.subset if present
53
54{l10n-path}/*.subset
55    If multiple files match all are used
56
57";
58pub(super) fn l10n() {
59    help(L10N_HELP);
60
61    // read source
62    let source = read_path(&path(ZR_REQUEST)).unwrap_or_else(|e| fatal!("{e}"));
63    // target derived from the request file name
64    let mut target = path(ZR_TARGET);
65    // request without name "./'l10n", take name from source (this is deliberate not documented)
66    if target.ends_with("'l10n") || target.ends_with(" 'l10n") {
67        target = target.with_file_name(source.file_name().unwrap());
68    }
69
70    if source.is_dir() {
71        println!("{}", display_path(&target));
72        fs::create_dir(&target).unwrap_or_else(|e| {
73            if e.kind() != io::ErrorKind::AlreadyExists {
74                fatal!("{e}")
75            }
76        });
77
78        l10n_filter_copy(source, target);
79    } else if source.is_file() {
80        fatal!("expected l10n dir, '{}' is a file", source.display());
81    } else if source.is_symlink() {
82        symlink_warn(&source);
83    } else {
84        warn!("cannot copy l10n dir '{}', not found", source.display());
85    }
86}
87
88/// cargo zng l10n --release-langs PATH
89pub(crate) fn release_langs(path: &Path) {
90    let mut sep = "";
91    for from_lang in fs::read_dir(path).unwrap_or_else(|e| fatal!("cannot read {}, {}", path.display(), e)) {
92        let from_lang = from_lang
93            .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", path.display(), e))
94            .path();
95        if !from_lang.is_dir() {
96            continue;
97        }
98        let lang = match from_lang.file_name().and_then(|s| s.to_str()) {
99            Some(l) => l,
100            None => continue,
101        };
102        if lang.starts_with("pseudo") || lang == "template" {
103            continue;
104        }
105
106        let mut any_ftl = false;
107        for from_entry in fs::read_dir(&from_lang).unwrap_or_else(|e| fatal!("cannot read {}, {}", from_lang.display(), e)) {
108            let from_entry = from_entry
109                .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from_lang.display(), e))
110                .path();
111
112            if from_entry.is_file()
113                && let Some(ext) = from_entry.extension()
114                && ext.eq_ignore_ascii_case("ftl")
115            {
116                any_ftl = true;
117                break;
118            }
119        }
120        if any_ftl {
121            print!("{sep}{lang}");
122            sep = ",";
123        }
124    }
125    println!();
126}
127
128fn l10n_filter_copy(from: PathBuf, to: PathBuf) {
129    let subset = allow_subset(&from);
130
131    for from_lang in fs::read_dir(&from).unwrap_or_else(|e| fatal!("cannot read {}, {}", from.display(), e)) {
132        let from_lang = from_lang
133            .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from.display(), e))
134            .path();
135        if !from_lang.is_dir() {
136            continue;
137        }
138
139        // skip pseudo* and template
140        let lang = match from_lang.file_name().and_then(|s| s.to_str()) {
141            Some(l) => l,
142            None => continue,
143        };
144        if lang.starts_with("pseudo") || lang == "template" {
145            continue;
146        }
147
148        let to_lang = to.join(lang);
149
150        // copy *.ftl and collect ./deps
151        let mut any_ftl = false;
152        let mut from_deps = None;
153        for from_entry in fs::read_dir(&from_lang).unwrap_or_else(|e| fatal!("cannot read {}, {}", from_lang.display(), e)) {
154            let from_entry = from_entry
155                .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from_lang.display(), e))
156                .path();
157
158            if from_entry.is_file() {
159                if let Some(ext) = from_entry.extension()
160                    && ext.eq_ignore_ascii_case("ftl")
161                {
162                    if !any_ftl {
163                        fs::create_dir(&to_lang).unwrap_or_else(|e| fatal!("cannot create {}, {}", to_lang.display(), e));
164                    }
165                    any_ftl = true;
166
167                    let to_entry = to_lang.join(from_entry.file_name().unwrap());
168                    fs::copy(from_entry, &to_entry).unwrap_or_else(|e| fatal!("cannot copy to {}, {}", to_entry.display(), e));
169                }
170            } else if from_entry.is_dir()
171                && let Some(name) = from_entry.file_name()
172                && name == "deps"
173            {
174                from_deps = Some(from_entry);
175            }
176        }
177
178        // skip lang, no local translations
179        if !any_ftl {
180            continue;
181        }
182
183        let from_deps = match from_deps {
184            Some(p) => p,
185            None => continue,
186        };
187
188        macro_rules! lazy_path {
189            ($init:expr) => {
190                Lazy::<PathBuf, _>::new(|| {
191                    let p = $init;
192                    fs::create_dir(&p).unwrap_or_else(|e| fatal!("cannot create {}, {}", p.display(), e));
193                    p
194                })
195            };
196        }
197        let to_deps = lazy_path!(to_lang.join("deps"));
198
199        // copy ./deps/*/*/*.ftl
200        for from_pkg in fs::read_dir(&from_deps).unwrap_or_else(|e| fatal!("cannot read {}, {}", from_deps.display(), e)) {
201            let from_pkg = from_pkg
202                .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from_deps.display(), e))
203                .path();
204            if !from_pkg.is_dir() {
205                continue;
206            }
207
208            let pkg = match from_pkg.file_name().and_then(|p| p.to_str()) {
209                Some(p) => p,
210                None => continue,
211            };
212            let subset_pkg = match subset.get(pkg) {
213                Some(m) => Cow::Borrowed(m),
214                None => {
215                    // subset filter
216                    if !subset.is_empty() {
217                        continue;
218                    }
219                    // no filter
220                    Cow::Owned(HashMap::new())
221                }
222            };
223
224            let to_pkg = lazy_path!(to_deps.join(pkg));
225
226            for from_ver in fs::read_dir(&from_pkg).unwrap_or_else(|e| fatal!("cannot read {}, {}", from_pkg.display(), e)) {
227                let from_ver = from_ver
228                    .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from_pkg.display(), e))
229                    .path();
230                if !from_ver.is_dir() {
231                    continue;
232                }
233
234                let to_ver = lazy_path!(to_pkg.join(from_ver.file_name().unwrap()));
235
236                for from_entry in fs::read_dir(&from_ver).unwrap_or_else(|e| fatal!("cannot read {}, {}", from_ver.display(), e)) {
237                    let from_entry = from_entry
238                        .unwrap_or_else(|e| fatal!("cannot read {} entry, {}", from_ver.display(), e))
239                        .path();
240                    if !from_entry.is_file() || !matches!(from_entry.extension(), Some(ext) if ext.eq_ignore_ascii_case("ftl")) {
241                        continue;
242                    }
243
244                    let file = match from_entry.file_name().unwrap().to_str() {
245                        Some(f) => f,
246                        None => continue,
247                    };
248                    let subset_file = match subset_pkg.get(file) {
249                        Some(m) => Cow::Borrowed(m),
250                        None => {
251                            if !subset_pkg.is_empty() {
252                                continue;
253                            }
254                            Cow::Owned(HashMap::new())
255                        }
256                    };
257
258                    let to_entry = to_ver.join(file);
259
260                    let ok = crate::l10n::generate_util::transform_file(
261                        &from_entry,
262                        &to_entry,
263                        "",
264                        &|id, attr| match subset_file.get(id) {
265                            Some(attrs) => attr.is_empty() || attrs.contains(attr),
266                            None => subset_file.is_empty(), // allow if no filter
267                        },
268                        &|s| Cow::Borrowed(s),
269                        false,
270                    );
271                    if !ok {
272                        fatal!("cannot optimize {}", from_entry.display());
273                    }
274                }
275            }
276        }
277    }
278}
279
280// [package => [file => [id => [attribute]]]]
281type SubsetMap = HashMap<String, HashMap<String, HashMap<String, HashSet<String>>>>;
282
283fn allow_subset(from: &Path) -> SubsetMap {
284    let mut out = SubsetMap::new();
285
286    if let Ok(path) = env::var("ZNG_L10N_PROFILE_FILE")
287        && !path.is_empty()
288    {
289        if !path.ends_with(".subset") {
290            fatal!("ZNG_L10N_PROFILE_FILE must be a .subset file");
291        }
292        read_subset(Path::new(&path), true, &mut out);
293    } else {
294        let default_profile = Path::new("res/optimization-profiles/zng-ext-l10n.rec.subset");
295        if default_profile.exists() {
296            read_subset(default_profile, true, &mut out);
297        } else {
298            let default_profile_pair = Path::new("res/optimization-profiles/zng-ext-l10n.subset");
299            if default_profile_pair.exists() {
300                read_subset(default_profile, false, &mut out);
301            } else {
302                for file in glob_walker(&format!("{}/*.subset", from.display()), false).unwrap() {
303                    let file = file.unwrap_or_else(|e| fatal!("cannot read {}, {}", from.display(), e)).into_path();
304                    read_subset(&file, false, &mut out);
305                }
306            }
307        }
308    }
309
310    out
311}
312fn read_subset(path: &Path, try_pair: bool, out: &mut SubsetMap) {
313    let file = fs::File::open(path).unwrap_or_else(|e| fatal!("cannot  read {}, {}", path.display(), e));
314    let file = io::BufReader::new(file);
315    for (i, line) in file.lines().enumerate() {
316        let line = line.unwrap_or_else(|e| fatal!("cannot read {}, {}", path.display(), e));
317        let line = line.trim();
318        if line.starts_with('#') || line.is_empty() {
319            continue;
320        }
321
322        let (dependency, mut key) = match line.split_once("//") {
323            Some((d, k)) if !d.is_empty() && !k.is_empty() => (d, k),
324            _ => fatal!("unexpected line {}:{}", path.display(), i + 1),
325        };
326        let file = match key.split_once('/') {
327            Some((f, k)) => {
328                key = k;
329                // add .ftl so we can match the file_name directly
330                if f == "_" || f.is_empty() {
331                    Cow::Borrowed("_.ftl")
332                } else {
333                    Cow::Owned(format!("{f}.ftl"))
334                }
335            }
336            None => Cow::Borrowed("_.ftl"),
337        };
338        let (id, attribute) = match key.split_once('.') {
339            Some((id, a)) => {
340                if id.is_empty() || a.is_empty() {
341                    fatal!("unexpected line {}:{}", path.display(), i + 1);
342                }
343                (id, a)
344            }
345            None => (key, ""),
346        };
347
348        let dep_map = match out.get_mut(dependency) {
349            Some(m) => m,
350            None => out.entry(dependency.to_owned()).or_default(),
351        };
352        let file_map = match dep_map.get_mut(&*file) {
353            Some(m) => m,
354            None => dep_map.entry(file.into_owned()).or_default(),
355        };
356        let id_map = match file_map.get_mut(id) {
357            Some(m) => m,
358            None => file_map.entry(id.to_owned()).or_default(),
359        };
360        if !attribute.is_empty() && !id_map.contains(attribute) {
361            id_map.insert(attribute.to_owned());
362        }
363    }
364
365    if try_pair {
366        let name = path.file_name().unwrap().to_str().unwrap();
367        let pair_name = if let Some(n) = name.strip_suffix(".rec.subset") {
368            format!("{n}.subset")
369        } else if let Some(n) = name.strip_suffix(".subset") {
370            format!("{n}.rec.subset")
371        } else {
372            return;
373        };
374        let pair_path = path.parent().unwrap().join(pair_name);
375        if pair_path.exists() {
376            read_subset(&pair_path, false, out);
377        }
378    }
379}