cargo_zng/l10n/
translate.rs1use std::{
2 fs,
3 io::{self, BufRead, Read, Write},
4 path::{Path, PathBuf},
5 time::Duration,
6};
7
8use leaky_bucket_lite::sync_threadsafe::LeakyBucket;
9use sha2::Digest;
10use unic_langid::LanguageIdentifier;
11
12use crate::util::{self, glob_walker};
13
14pub fn translate(dir: &str, from: &str, to: &str, replace: bool, existing: bool, check: bool) {
15 let dir_path = Path::new(dir);
16 if !dir_path.exists() {
17 fatal!("dir `{dir}` does not exist")
18 }
19
20 let from = if from.is_empty() {
21 let name = dir_path.file_name().unwrap().to_string_lossy();
22 let name = &*name;
23 if name == "template" {
24 println_v!("from language set to `en`, default language for `template` folder");
25 LanguageIdentifier::from_bytes(b"en").unwrap()
26 } else {
27 let name = name.strip_suffix("-machine").unwrap_or(name);
28 match LanguageIdentifier::from_bytes(name.as_bytes()) {
29 Ok(l) => {
30 println_v!("from language language set to `{l}`, derived from `{name}` folder");
31 l
32 }
33 Err(e) => fatal!("cannot define from language, `{name}` is not a language, {e}\nset --translate-from to resolve"),
34 }
35 }
36 } else {
37 match LanguageIdentifier::from_bytes(from.as_bytes()) {
38 Ok(l) => l,
39 Err(e) => fatal!("cannot translate from `{from}`, {e}"),
40 }
41 };
42 let to = {
43 let mut t = vec![];
44 for l_str in to.split(',') {
45 let l_str = l_str.trim();
46 if l_str.is_empty() {
47 continue;
48 }
49 let l_str = l_str.strip_suffix("-machine").unwrap_or(l_str);
50 match LanguageIdentifier::from_bytes(l_str.as_bytes()) {
51 Ok(l) => t.push(l),
52 Err(e) => fatal!("cannot translate to `{l_str}`, {e}"),
53 }
54 }
55 t
56 };
57 if to.is_empty() {
58 fatal!("--translate-to requires at least one language")
59 }
60
61 let pattern = dir_path.join("**/*.ftl");
62 let mut files = vec![];
63 for entry in glob_walker(&pattern.display().to_string(), true).unwrap_or_else(|e| fatal!("cannot read `{dir}`, {e}")) {
64 let entry = entry.unwrap_or_else(|e| fatal!("cannot read `{dir}` entry, {e}")).into_path();
65 let relative_entry = entry.strip_prefix(dir_path).unwrap();
66 let file = match std::fs::read_to_string(&entry) {
67 Ok(f) => f,
68 Err(e) => fatal!("cannot read `{}`, {}", entry.display(), e),
69 };
70 let mut hasher = sha2::Sha256::new();
71 hasher.update(file.as_bytes());
72 let hash = format!("{:x}", base16ct::HexDisplay(&hasher.finalize()));
73 files.push((relative_entry.to_owned(), file, hash));
74 }
75 let files = files;
76
77 println_v!("collected {} source files", files.len());
78
79 let dir_path = dunce::canonicalize(dir_path).unwrap();
80 let dir_parent = dir_path.parent().unwrap();
81
82 let translator = Translator::default();
83 println_v!("using `{}`, RPM: {}", translator.path.display(), translator.rpm);
84
85 for to in to {
86 if to == from {
87 continue;
88 }
89
90 if !existing {
91 let non_machine_dir = dir_parent.join(to.to_string());
92 if non_machine_dir.exists() {
93 continue;
94 }
95 }
96 println!("translating to {to}-machine");
97
98 let dir_to = dir_parent.join(format!("{to}-machine"));
99
100 for (relative_path, file, hash) in &files {
101 let file_to = dir_to.join(relative_path);
102 println_v!(" {}", file_to.display());
103
104 let _ = util::check_or_create_dir_all(check, file_to.parent().unwrap());
105
106 const HEADER_PREFIX: &str = "### Machine translated by `cargo zng l10n`, ";
107
108 let mut stale = false;
109 match fs::File::open(&file_to) {
110 Ok(f) => {
111 if !replace {
112 let mut f = io::BufReader::new(f);
113 let mut header = String::new();
114 let _ = f.read_line(&mut header);
115 if let Some(h) = header.strip_prefix(HEADER_PREFIX) {
116 if hash == h.trim_end() {
117 continue;
119 }
120 stale = true;
121 } else {
122 continue;
124 }
125 }
126 }
127 Err(_) => {
128 if check {
129 fatal!("expected `{}`", file_to.display());
130 }
131 }
132 }
133
134 if !check && (replace || stale || !file_to.exists()) {
135 let r = translator.translate(&from, &to, file);
136 let write = || -> io::Result<()> {
137 let mut f = fs::File::create(&file_to)?;
138 f.write_all(HEADER_PREFIX.as_bytes())?;
139 f.write_all(hash.as_bytes())?;
140 f.write_all("\n\n".as_bytes())?;
141 f.write_all(r.as_bytes())?;
142 Ok(())
143 };
144 if let Err(e) = write() {
145 fatal!("cannot write `{}`, {}", file_to.display(), e);
146 }
147 }
148 }
149 }
150}
151
152struct Translator {
153 path: PathBuf,
154 rpm: u64,
155 limiter: LeakyBucket,
156}
157impl Default for Translator {
158 fn default() -> Self {
159 let install_dir = std::env::current_exe().unwrap();
160 let install_dir = install_dir.parent().unwrap();
161
162 let mut t = if let Ok(translator) = std::env::var("ZNG_L10N_TRANSLATOR") {
164 let p = if translator.contains('/') || translator.contains('\\') {
165 PathBuf::from(&translator)
166 } else {
167 install_dir.join(format!("zng-l10n-translator-{translator}{}", std::env::consts::EXE_SUFFIX))
168 };
169 if !p.exists() {
170 fatal!("cannot find translator `{translator}`");
171 }
172 Translator {
173 path: p,
174 rpm: 0,
175 limiter: LeakyBucket::builder().build(),
176 }
177 } else {
178 let translators_pattern = install_dir.join("zng-l10n-translator-*").display().to_string();
179 let mut options = vec![];
180 for opt in glob_walker(&translators_pattern, false).unwrap() {
181 options.push(opt.unwrap().into_path());
182 }
183 if options.is_empty() {
184 fatal!("no translator installed\n install a zng-l10n-translator-* crate\nor set ZNG_L10N_TRANSLATOR to a path")
185 } else if options.len() > 1 {
186 let mut names: Vec<_> = options
187 .iter()
188 .map(|p| {
189 p.file_name()
190 .unwrap()
191 .to_string_lossy()
192 .trim_start_matches("zng-l10n-translator-")
193 .trim_end_matches(std::env::consts::EXE_SUFFIX)
194 .to_owned()
195 })
196 .collect();
197 names.sort();
198 let names = names.join(", ");
199 fatal!("multiple translators installed\n set ZNG_L10N_TRANSLATOR to one of: {names}");
200 }
201 Translator {
202 path: options.remove(0),
203 rpm: 0,
204 limiter: LeakyBucket::builder().build(),
205 }
206 };
207
208 t.read_limits();
210
211 t
212 }
213}
214impl Translator {
215 fn read_limits(&mut self) {
216 #[derive(serde::Deserialize)]
217 struct Limits {
218 #[serde(rename = "requests-per-minute")]
219 rpm: u64,
220 }
221
222 let output = std::process::Command::new(&self.path).arg("--limits").output().unwrap();
223 if output.status.success() {
224 match serde_json::from_slice::<Limits>(&output.stdout) {
225 Ok(l) => {
226 self.rpm = l.rpm;
227 let t: u32 = self.rpm.try_into().unwrap();
228 self.limiter = LeakyBucket::builder()
229 .max(t)
230 .tokens(t)
231 .refill_amount(t)
232 .refill_interval(Duration::from_mins(1))
233 .build();
234 }
235 Err(e) => {
236 let limits = String::from_utf8_lossy(&output.stdout);
237 fatal!("invalid response to --limits, {e}\n{limits}")
238 }
239 }
240 } else {
241 let error = String::from_utf8_lossy(&output.stderr);
242 fatal!(
243 "{error}\n{} exited with code {}",
244 self.path.file_name().unwrap().to_string_lossy(),
245 output.status.code().unwrap_or(0)
246 );
247 }
248 }
249
250 fn translate(&self, from: &LanguageIdentifier, to: &LanguageIdentifier, file: &str) -> String {
251 self.limiter.acquire_one();
252
253 let mut cmd = std::process::Command::new(&self.path)
254 .arg("--from-lang")
255 .arg(from.to_string())
256 .arg("--to-lang")
257 .arg(to.to_string())
258 .stdin(std::process::Stdio::piped())
259 .stdout(std::process::Stdio::piped())
260 .stderr(std::process::Stdio::piped())
261 .spawn()
262 .unwrap();
263
264 let mut stdin = cmd.stdin.take().unwrap();
265 stdin.write_all(file.as_bytes()).unwrap();
266 stdin.flush().unwrap();
267 drop(stdin);
268
269 let s = cmd.wait().unwrap();
270 if !s.success() {
271 let mut error = String::new();
272 cmd.stderr.unwrap().read_to_string(&mut error).unwrap();
273 fatal!(
274 "{error}\n{} exited with code {}",
275 self.path.file_name().unwrap().to_string_lossy(),
276 s.code().unwrap_or(0)
277 );
278 }
279 let mut out = String::new();
280 cmd.stdout.unwrap().read_to_string(&mut out).unwrap();
281
282 if let Err(e) = fluent_syntax::parser::parse(out.as_str()) {
283 fatal!("translator output is not valid Fluent, {}", &e.1[0]);
284 }
285
286 out
287 }
288}