Skip to main content

cargo_zng/l10n/
translate.rs

1use std::{
2    fs,
3    io::{self, BufRead, Read, Write},
4    path::{Path, PathBuf},
5    time::Duration,
6};
7
8use leaky_bucket_lite::sync_threadsafe::LeakyBucket;
9use sha2::Digest;
10use unic_langid::LanguageIdentifier;
11
12use crate::util::{self, glob_walker};
13
14pub fn translate(dir: &str, from: &str, to: &str, replace: bool, existing: bool, check: bool) {
15    let dir_path = Path::new(dir);
16    if !dir_path.exists() {
17        fatal!("dir `{dir}` does not exist")
18    }
19
20    let from = if from.is_empty() {
21        let name = dir_path.file_name().unwrap().to_string_lossy();
22        let name = &*name;
23        if name == "template" {
24            println_v!("from language set to `en`, default language for `template` folder");
25            LanguageIdentifier::from_bytes(b"en").unwrap()
26        } else {
27            let name = name.strip_suffix("-machine").unwrap_or(name);
28            match LanguageIdentifier::from_bytes(name.as_bytes()) {
29                Ok(l) => {
30                    println_v!("from language language set to `{l}`, derived from `{name}` folder");
31                    l
32                }
33                Err(e) => fatal!("cannot define from language, `{name}` is not a language, {e}\nset --translate-from to resolve"),
34            }
35        }
36    } else {
37        match LanguageIdentifier::from_bytes(from.as_bytes()) {
38            Ok(l) => l,
39            Err(e) => fatal!("cannot translate from `{from}`, {e}"),
40        }
41    };
42    let to = {
43        let mut t = vec![];
44        for l_str in to.split(',') {
45            let l_str = l_str.trim();
46            if l_str.is_empty() {
47                continue;
48            }
49            let l_str = l_str.strip_suffix("-machine").unwrap_or(l_str);
50            match LanguageIdentifier::from_bytes(l_str.as_bytes()) {
51                Ok(l) => t.push(l),
52                Err(e) => fatal!("cannot translate to `{l_str}`, {e}"),
53            }
54        }
55        t
56    };
57    if to.is_empty() {
58        fatal!("--translate-to requires at least one language")
59    }
60
61    let pattern = dir_path.join("**/*.ftl");
62    let mut files = vec![];
63    for entry in glob_walker(&pattern.display().to_string(), true).unwrap_or_else(|e| fatal!("cannot read `{dir}`, {e}")) {
64        let entry = entry.unwrap_or_else(|e| fatal!("cannot read `{dir}` entry, {e}")).into_path();
65        let relative_entry = entry.strip_prefix(dir_path).unwrap();
66        let file = match std::fs::read_to_string(&entry) {
67            Ok(f) => f,
68            Err(e) => fatal!("cannot read `{}`, {}", entry.display(), e),
69        };
70        let mut hasher = sha2::Sha256::new();
71        hasher.update(file.as_bytes());
72        let hash = format!("{:x}", base16ct::HexDisplay(&hasher.finalize()));
73        files.push((relative_entry.to_owned(), file, hash));
74    }
75    let files = files;
76
77    println_v!("collected {} source files", files.len());
78
79    let dir_path = dunce::canonicalize(dir_path).unwrap();
80    let dir_parent = dir_path.parent().unwrap();
81
82    let translator = Translator::default();
83    println_v!("using `{}`, RPM: {}", translator.path.display(), translator.rpm);
84
85    for to in to {
86        if to == from {
87            continue;
88        }
89
90        if !existing {
91            let non_machine_dir = dir_parent.join(to.to_string());
92            if non_machine_dir.exists() {
93                continue;
94            }
95        }
96        println!("translating to {to}-machine");
97
98        let dir_to = dir_parent.join(format!("{to}-machine"));
99
100        for (relative_path, file, hash) in &files {
101            let file_to = dir_to.join(relative_path);
102            println_v!("  {}", file_to.display());
103
104            let _ = util::check_or_create_dir_all(check, file_to.parent().unwrap());
105
106            const HEADER_PREFIX: &str = "### Machine translated by `cargo zng l10n`, ";
107
108            let mut stale = false;
109            match fs::File::open(&file_to) {
110                Ok(f) => {
111                    if !replace {
112                        let mut f = io::BufReader::new(f);
113                        let mut header = String::new();
114                        let _ = f.read_line(&mut header);
115                        if let Some(h) = header.strip_prefix(HEADER_PREFIX) {
116                            if hash == h.trim_end() {
117                                // translation is up-to-date
118                                continue;
119                            }
120                            stale = true;
121                        } else {
122                            // translation not generated by us, assume is up-to-date
123                            continue;
124                        }
125                    }
126                }
127                Err(_) => {
128                    if check {
129                        fatal!("expected `{}`", file_to.display());
130                    }
131                }
132            }
133
134            if !check && (replace || stale || !file_to.exists()) {
135                let r = translator.translate(&from, &to, file);
136                let write = || -> io::Result<()> {
137                    let mut f = fs::File::create(&file_to)?;
138                    f.write_all(HEADER_PREFIX.as_bytes())?;
139                    f.write_all(hash.as_bytes())?;
140                    f.write_all("\n\n".as_bytes())?;
141                    f.write_all(r.as_bytes())?;
142                    Ok(())
143                };
144                if let Err(e) = write() {
145                    fatal!("cannot write `{}`, {}", file_to.display(), e);
146                }
147            }
148        }
149    }
150}
151
152struct Translator {
153    path: PathBuf,
154    rpm: u64,
155    limiter: LeakyBucket,
156}
157impl Default for Translator {
158    fn default() -> Self {
159        let install_dir = std::env::current_exe().unwrap();
160        let install_dir = install_dir.parent().unwrap();
161
162        // find translator
163        let mut t = if let Ok(translator) = std::env::var("ZNG_L10N_TRANSLATOR") {
164            let p = if translator.contains('/') || translator.contains('\\') {
165                PathBuf::from(&translator)
166            } else {
167                install_dir.join(format!("zng-l10n-translator-{translator}{}", std::env::consts::EXE_SUFFIX))
168            };
169            if !p.exists() {
170                fatal!("cannot find translator `{translator}`");
171            }
172            Translator {
173                path: p,
174                rpm: 0,
175                limiter: LeakyBucket::builder().build(),
176            }
177        } else {
178            let translators_pattern = install_dir.join("zng-l10n-translator-*").display().to_string();
179            let mut options = vec![];
180            for opt in glob_walker(&translators_pattern, false).unwrap() {
181                options.push(opt.unwrap().into_path());
182            }
183            if options.is_empty() {
184                fatal!("no translator installed\n   install a zng-l10n-translator-* crate\nor set ZNG_L10N_TRANSLATOR to a path")
185            } else if options.len() > 1 {
186                let mut names: Vec<_> = options
187                    .iter()
188                    .map(|p| {
189                        p.file_name()
190                            .unwrap()
191                            .to_string_lossy()
192                            .trim_start_matches("zng-l10n-translator-")
193                            .trim_end_matches(std::env::consts::EXE_SUFFIX)
194                            .to_owned()
195                    })
196                    .collect();
197                names.sort();
198                let names = names.join(", ");
199                fatal!("multiple translators installed\n    set ZNG_L10N_TRANSLATOR to one of: {names}");
200            }
201            Translator {
202                path: options.remove(0),
203                rpm: 0,
204                limiter: LeakyBucket::builder().build(),
205            }
206        };
207
208        // request limits
209        t.read_limits();
210
211        t
212    }
213}
214impl Translator {
215    fn read_limits(&mut self) {
216        #[derive(serde::Deserialize)]
217        struct Limits {
218            #[serde(rename = "requests-per-minute")]
219            rpm: u64,
220        }
221
222        let output = std::process::Command::new(&self.path).arg("--limits").output().unwrap();
223        if output.status.success() {
224            match serde_json::from_slice::<Limits>(&output.stdout) {
225                Ok(l) => {
226                    self.rpm = l.rpm;
227                    let t: u32 = self.rpm.try_into().unwrap();
228                    self.limiter = LeakyBucket::builder()
229                        .max(t)
230                        .tokens(t)
231                        .refill_amount(t)
232                        .refill_interval(Duration::from_mins(1))
233                        .build();
234                }
235                Err(e) => {
236                    let limits = String::from_utf8_lossy(&output.stdout);
237                    fatal!("invalid response to --limits, {e}\n{limits}")
238                }
239            }
240        } else {
241            let error = String::from_utf8_lossy(&output.stderr);
242            fatal!(
243                "{error}\n{} exited with code {}",
244                self.path.file_name().unwrap().to_string_lossy(),
245                output.status.code().unwrap_or(0)
246            );
247        }
248    }
249
250    fn translate(&self, from: &LanguageIdentifier, to: &LanguageIdentifier, file: &str) -> String {
251        self.limiter.acquire_one();
252
253        let mut cmd = std::process::Command::new(&self.path)
254            .arg("--from-lang")
255            .arg(from.to_string())
256            .arg("--to-lang")
257            .arg(to.to_string())
258            .stdin(std::process::Stdio::piped())
259            .stdout(std::process::Stdio::piped())
260            .stderr(std::process::Stdio::piped())
261            .spawn()
262            .unwrap();
263
264        let mut stdin = cmd.stdin.take().unwrap();
265        stdin.write_all(file.as_bytes()).unwrap();
266        stdin.flush().unwrap();
267        drop(stdin);
268
269        let s = cmd.wait().unwrap();
270        if !s.success() {
271            let mut error = String::new();
272            cmd.stderr.unwrap().read_to_string(&mut error).unwrap();
273            fatal!(
274                "{error}\n{} exited with code {}",
275                self.path.file_name().unwrap().to_string_lossy(),
276                s.code().unwrap_or(0)
277            );
278        }
279        let mut out = String::new();
280        cmd.stdout.unwrap().read_to_string(&mut out).unwrap();
281
282        if let Err(e) = fluent_syntax::parser::parse(out.as_str()) {
283            fatal!("translator output is not valid Fluent, {}", &e.1[0]);
284        }
285
286        out
287    }
288}