zng_ext_l10n/sources/
tar.rs

1use std::{
2    borrow::Cow,
3    collections::HashMap,
4    fmt,
5    io::Read as _,
6    path::{Path, PathBuf},
7    str::FromStr as _,
8    sync::Arc,
9};
10
11use semver::Version;
12use zng_clone_move::clmv;
13use zng_txt::Txt;
14use zng_var::{ArcEq, Var, WeakVar, const_var, var, weak_var};
15
16use crate::{FluentParserErrors, L10nSource, Lang, LangFilePath, LangMap, LangResourceStatus};
17
18/// Represents localization resources loaded from a `.tar`, `.tar.gz` or `.tar.zst` container.
19///
20/// The expected container layout is `root_dir/{lang}/{file}.ftl` app files and `root_dir/{lang}/deps/{pkg-name}/{pkg-version}/{file}.ftl`
21/// for dependencies, same as [`L10nDir`], `root_dir` can have any name.
22///
23/// [`L10nDir`]: crate::L10nDir
24pub struct L10nTar {
25    data: L10nTarData,
26    available_langs: Var<Arc<LangMap<HashMap<LangFilePath, PathBuf>>>>,
27    available_langs_status: Var<LangResourceStatus>,
28    res: HashMap<(Lang, LangFilePath), L10nEntry>,
29}
30impl L10nTar {
31    /// Load from TAR data.
32    pub fn load(data: impl Into<L10nTarData>) -> Self {
33        Self::load_impl(data.into())
34    }
35    fn load_impl(data: L10nTarData) -> Self {
36        let r = Self {
37            data,
38            available_langs: var(Arc::new(LangMap::new())),
39            available_langs_status: var(LangResourceStatus::Loading),
40            res: HashMap::default(),
41        };
42        r.load_available_langs();
43        r
44    }
45    fn load_available_langs(&self) {
46        let status = self.available_langs_status.clone();
47        let map = self.available_langs.clone();
48        let data = self.data.clone();
49        zng_task::spawn_wait(move || {
50            let r = (|| -> std::io::Result<_> {
51                let mut set: LangMap<HashMap<LangFilePath, PathBuf>> = LangMap::new();
52                let mut errors: Vec<Arc<dyn std::error::Error + Send + Sync>> = vec![];
53                // resource_var expects the "fatal" errors here to not insert in map
54                let data = data.decode_bytes()?;
55                let data: &[u8] = &data;
56                let mut archive = tar::Archive::new(std::io::Cursor::new(data));
57                let entries = archive.entries_with_seek()?;
58                for entry in entries {
59                    let entry = entry?;
60                    let ty = entry.header().entry_type();
61                    let entry = entry.path()?;
62
63                    const EXT: unicase::Ascii<&'static str> = unicase::Ascii::new("ftl");
64
65                    let is_ftl = ty.is_file()
66                        && entry
67                            .file_name()
68                            .and_then(|s| s.to_str())
69                            .and_then(|n| n.rsplit_once('.'))
70                            .map(|(_, ext)| ext.is_ascii() && unicase::Ascii::new(ext) == EXT)
71                            .unwrap_or(false);
72
73                    if !is_ftl {
74                        continue;
75                    }
76
77                    let utf8_path: Vec<_> = entry.iter().take(6).map(|s| s.to_str().unwrap_or("")).collect();
78                    let utf8_path = &utf8_path[1..];
79
80                    let (lang, mut file) = match utf8_path.len() {
81                        // lang/file.ftl
82                        2 => {
83                            let lang = utf8_path[0];
84                            let file_str = utf8_path[1].rsplit_once('.').unwrap().0;
85                            let file = Txt::from_str(if file_str == "_" { "" } else { file_str });
86                            (lang, LangFilePath::current_app(file))
87                        }
88                        // lang/deps/pkg-name/pkg-version/file.ftl
89                        5 => {
90                            if utf8_path[1] != "deps" {
91                                continue;
92                            }
93                            let lang = utf8_path[0];
94                            let pkg_name = Txt::from_str(utf8_path[2]);
95                            let pkg_version: Version = match utf8_path[3].parse() {
96                                Ok(v) => v,
97                                Err(e) => {
98                                    errors.push(Arc::new(e));
99                                    continue;
100                                }
101                            };
102                            let file_str = utf8_path[4].rsplit_once('.').unwrap().0;
103                            let file = Txt::from_str(if file_str == "_" { "" } else { file_str });
104
105                            (lang, LangFilePath::new(pkg_name, pkg_version, file))
106                        }
107                        _ => {
108                            continue;
109                        }
110                    };
111
112                    let lang = match Lang::from_str(lang) {
113                        Ok(l) => l,
114                        Err(e) => {
115                            errors.push(Arc::new(e));
116                            continue;
117                        }
118                    };
119
120                    if file.file == "_" {
121                        file.file = "".into();
122                    }
123
124                    set.get_exact_or_insert(lang, Default::default)
125                        .insert(file, entry.as_ref().to_owned());
126                }
127                for m in set.values_mut() {
128                    m.shrink_to_fit();
129                }
130                set.shrink_to_fit();
131                map.set(set);
132                Ok(errors)
133            })();
134            match r {
135                Ok(e) => {
136                    if e.is_empty() {
137                        status.set(LangResourceStatus::Loaded)
138                    } else {
139                        let e = LangResourceStatus::Errors(e);
140                        tracing::error!("'loading available' {e}");
141                        status.set(e)
142                    }
143                }
144                Err(e) => {
145                    tracing::error!("failed to load tar, {e}");
146                    status.set(LangResourceStatus::Errors(vec![Arc::new(e)]))
147                }
148            }
149        });
150    }
151}
152impl L10nSource for L10nTar {
153    fn available_langs(&mut self) -> Var<Arc<LangMap<HashMap<LangFilePath, PathBuf>>>> {
154        self.available_langs.read_only()
155    }
156
157    fn available_langs_status(&mut self) -> Var<LangResourceStatus> {
158        self.available_langs_status.read_only()
159    }
160
161    fn lang_resource(&mut self, lang: Lang, file: LangFilePath) -> Var<Option<ArcEq<fluent::FluentResource>>> {
162        match self.res.entry((lang, file)) {
163            std::collections::hash_map::Entry::Occupied(mut e) => {
164                if let Some(out) = e.get().res.upgrade() {
165                    out.read_only()
166                } else {
167                    let (lang, file) = e.key();
168                    let out = resource_var(
169                        self.data.clone(),
170                        &self.available_langs,
171                        e.get().status.clone(),
172                        lang.clone(),
173                        file.clone(),
174                    );
175                    e.get_mut().res = out.downgrade();
176                    out
177                }
178            }
179            std::collections::hash_map::Entry::Vacant(e) => {
180                let mut f = L10nEntry::new();
181                let (lang, file) = e.key();
182                let out = resource_var(
183                    self.data.clone(),
184                    &self.available_langs,
185                    f.status.clone(),
186                    lang.clone(),
187                    file.clone(),
188                );
189                f.res = out.downgrade();
190                e.insert(f);
191                out
192            }
193        }
194    }
195
196    fn lang_resource_status(&mut self, lang: Lang, file: LangFilePath) -> Var<LangResourceStatus> {
197        self.res.entry((lang, file)).or_insert_with(L10nEntry::new).status.read_only()
198    }
199}
200
201/// TAR data for [`L10nTar`].
202#[derive(Clone, PartialEq, Eq)]
203pub enum L10nTarData {
204    /// Embedded data.
205    Static(&'static [u8]),
206    /// Loaded data.
207    Arc(Arc<Vec<u8>>),
208}
209impl fmt::Debug for L10nTarData {
210    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
211        match self {
212            Self::Static(_) => f.debug_tuple("Static").finish_non_exhaustive(),
213            Self::Arc(_) => f.debug_tuple("Arc").finish_non_exhaustive(),
214        }
215    }
216}
217impl From<&'static [u8]> for L10nTarData {
218    fn from(value: &'static [u8]) -> Self {
219        L10nTarData::Static(value)
220    }
221}
222impl From<Arc<Vec<u8>>> for L10nTarData {
223    fn from(value: Arc<Vec<u8>>) -> Self {
224        L10nTarData::Arc(value)
225    }
226}
227impl From<Vec<u8>> for L10nTarData {
228    fn from(value: Vec<u8>) -> Self {
229        L10nTarData::Arc(Arc::new(value))
230    }
231}
232impl L10nTarData {
233    /// Reference the data.
234    pub fn bytes(&self) -> &[u8] {
235        match self {
236            L10nTarData::Static(b) => b,
237            L10nTarData::Arc(b) => b,
238        }
239    }
240
241    /// Check if [`bytes`] have the GZIP magic number.
242    ///
243    /// [`bytes`]: Self::bytes
244    pub fn is_gz(&self) -> bool {
245        let bytes = self.bytes();
246        bytes.len() > 2 && bytes[0..2] == [0x1F, 0x8B]
247    }
248
249    /// Check if [`bytes`] have the ZSTD magic number.
250    ///
251    /// [`bytes`]: Self::bytes
252    pub fn is_zst(&self) -> bool {
253        let bytes = self.bytes();
254        bytes.len() > 4 && bytes[0..4] == [0x28, 0xB5, 0x2F, 0xFD]
255    }
256
257    /// Decompress bytes.
258    pub fn decode_bytes(&self) -> std::io::Result<Cow<'_, [u8]>> {
259        if self.is_gz() {
260            let bytes = self.bytes();
261            let mut data = vec![];
262            let mut decoder = flate2::read::GzDecoder::new(bytes);
263            decoder.read_to_end(&mut data)?;
264            Ok(Cow::Owned(data))
265        } else if self.is_zst() {
266            let bytes = self.bytes();
267            let mut data = vec![];
268            let mut decoder = zstd::Decoder::new(bytes)?;
269            decoder.read_to_end(&mut data)?;
270            Ok(Cow::Owned(data))
271        } else {
272            Ok(Cow::Borrowed(self.bytes()))
273        }
274    }
275
276    /// Extract data to a directory.
277    ///
278    /// The extracted directory can be used as a l10n source that can be live edited.
279    ///
280    /// Note that [`L10nTar`] does not use this, it references the TAR data in memory directly for better performance.
281    pub fn extract(&self, path: impl AsRef<Path>) -> std::io::Result<()> {
282        self.extract_impl(path.as_ref())
283    }
284    fn extract_impl(&self, path: &Path) -> std::io::Result<()> {
285        let data = self.decode_bytes()?;
286        let mut data = tar::Archive::new(&*data);
287        data.unpack(path)
288    }
289}
290
291struct L10nEntry {
292    res: WeakVar<Option<ArcEq<fluent::FluentResource>>>,
293    status: Var<LangResourceStatus>,
294}
295impl L10nEntry {
296    fn new() -> Self {
297        Self {
298            res: weak_var(),
299            status: var(LangResourceStatus::Loading),
300        }
301    }
302}
303
304fn resource_var(
305    data: L10nTarData,
306    available_langs: &Var<Arc<LangMap<HashMap<LangFilePath, PathBuf>>>>,
307    status: Var<LangResourceStatus>,
308    lang: Lang,
309    file: LangFilePath,
310) -> Var<Option<ArcEq<fluent::FluentResource>>> {
311    available_langs
312        .map(move |w| w.get_file(&lang, &file).cloned())
313        .flat_map(move |p| match p {
314            Some(p) => {
315                status.set(LangResourceStatus::Loading);
316                let rsp = zng_task::wait_respond(clmv!(p, status, data, || {
317                    const E: &str = "already decoded ok once to get entries";
318                    let data = data.decode_bytes().expect(E);
319                    let data: &[u8] = &data;
320                    let mut archive = tar::Archive::new(std::io::Cursor::new(data));
321                    for entry in archive.entries_with_seek().expect(E) {
322                        let mut entry = entry.expect(E);
323                        if entry.path().map(|ep| ep == p).unwrap_or(false) {
324                            let mut flt = String::new();
325                            if let Err(e) = entry.read_to_string(&mut flt) {
326                                tracing::error!("error reading fluent resource, {e}");
327                                status.set(LangResourceStatus::Errors(vec![Arc::new(e)]));
328                            } else {
329                                match fluent::FluentResource::try_new(flt) {
330                                    Ok(flt) => {
331                                        // ok
332                                        // Loaded set by `r` to avoid race condition in waiter.
333                                        return Some(ArcEq::new(flt));
334                                    }
335                                    Err(e) => {
336                                        let e = FluentParserErrors(e.1);
337                                        tracing::error!("error parsing fluent resource, {e}");
338                                        status.set(LangResourceStatus::Errors(vec![Arc::new(e)]));
339                                    }
340                                }
341                            }
342                            return None;
343                        }
344                    }
345                    status.set(LangResourceStatus::NotAvailable);
346                    None
347                }));
348                rsp.bind_filter_map(&status, |r| r.done().and_then(|r| r.as_ref()).map(|_| LangResourceStatus::Loaded))
349                    .perm();
350                rsp.map(|r| r.done().cloned().flatten())
351            }
352            None => const_var(None),
353        })
354}
zng_ext_l10n/sources/tar.rs

zng_ext_l10n/sources/
tar.rs
