Skip to main content

cargo_zng/res/built_in/
tar.rs

1use std::{collections::HashSet, io::Write, rc::Rc};
2
3use lzma_rust2::filter::bcj::BcjWriter;
4use serde::Deserialize;
5
6use crate::util::glob_walker;
7
8use super::*;
9
10const TAR_HELP: &str = r#"
11Pack files and dirs into a TAR container with optional compression
12
13The request file:
14  source/data.tar.zst 'tar
15   | [[entry]]
16   | path = "res/bin/*"
17   | name = "bin/*"
18   |
19   | [[entry]]
20   | path = "README.md"
21   | name = "docs/README.md"
22   |
23   | # Optional compression
24   | [zstd]
25   | level = 19
26
27Packs entries into a TAR, compresses it with ZSTD.
28
29The syntax is a TOML file, the tables are:
30
31[[entry]] — Array of entries to pack.
32path — Path or glob pattern, relative to the workspace root. Required.
33name — Optional name of the file on the TAR container. Optional.
34
35Each entry can be a file, directory or glob selection. Only file and directory 
36entries are supported by this tool, other tar archive entries are not supported.
37
38If 'path' matches a directory all contained files and sub directories are packed. 
39
40Matching is case insensitive in all platforms. See 'glob tool for glob syntax.
41
42If 'name' is not set it is path relative to workspace root. 
43
44If 'name' is set it must not contain "/..".
45
46If 'name' ends with "/*" the selected files and dirs are named in this TAR dir.
47
48All 'name' paths are relative to the TAR root, "/foo" is packed as "foo".
49
50[filter] — Optional lossless transform to apply to the TAR
51bcj — Branch/Call/Jump filter that optimizes compression of binary code files.
52    Values: "x86", "arm", "arm64", "arm-thumb", "ppc", "sparc", "ia64", "riscv"
53    Note that decompressor must revert the filter.
54
55The decompressor must undo these changes before reading the TAR. The 'sfx supports decoding BCJ. 
56
57[zstd] — Optional ZStandard compression
58level — Compression level, -131072..=22, 0 means no compression, default is 19.
59
60Compress the TAR, after [filter] is applied, using ZStandard. The "contentsize" field of zstd header
61is correctly set, so this is fully compatible with 'sfx that uses this field to report response stream length.
62
63[gzip] — Optional GZip compression
64level — Compression level, 0..=9, 0 means no compression, default is 8.
65
66"#;
67pub(super) fn tar() {
68    help(TAR_HELP);
69
70    let request = std::fs::read_to_string(path(ZR_REQUEST)).unwrap_or_else(|e| fatal!("{e}"));
71    let request: Request = toml::from_str(&request).unwrap_or_else(|e| fatal!("{e}"));
72
73    let target = path(ZR_TARGET);
74
75    let target = fs::File::create(&target).unwrap_or_else(|e| fatal!("{e}"));
76    let mut target: Box<dyn WriteFinish> = Box::new(target);
77
78    let tar_entries = collect_tar_entries(request.entries);
79
80    if let Some(zstd) = request.zstd
81        && zstd.level != 0
82    {
83        let range = zstd::compression_level_range();
84        let level = zstd.level.clamp(*range.start(), *range.end());
85        let mut encoder = zstd::Encoder::new(target, level).unwrap_or_else(|e| fatal!("{e}"));
86
87        let len = sum_tar_len(&tar_entries);
88        encoder.set_pledged_src_size(Some(len)).unwrap_or_else(|e| fatal!("{e}"));
89        encoder.include_contentsize(true).unwrap_or_else(|e| fatal!("{e}"));
90        target = Box::new(encoder);
91    } else if let Some(gzip) = request.gzip
92        && gzip.level != 0
93    {
94        let level = flate2::Compression::new(gzip.level.clamp(0, 9));
95        let encoder = flate2::write::GzEncoder::new(target, level);
96        target = Box::new(encoder);
97    }
98
99    if !request.filter.bcj.is_empty() {
100        let bcj = match request.filter.bcj.as_str() {
101            "x86" => BcjWriter::new_x86(target, 0),
102            "arm" => BcjWriter::new_arm(target, 0),
103            "arm64" => BcjWriter::new_arm64(target, 0),
104            "arm-thumb" => BcjWriter::new_arm_thumb(target, 0),
105            "ppc" => BcjWriter::new_ppc(target, 0),
106            "sparc" => BcjWriter::new_sparc(target, 0),
107            "ia64" => BcjWriter::new_ia64(target, 0),
108            "riscv" => BcjWriter::new_riscv(target, 0),
109            _ => fatal!("unknown bcj filter"),
110        };
111        target = Box::new(bcj)
112    }
113
114    let mut target = ::tar::Builder::new(target);
115    // all symlinks are skipped anyway, this is just an optimization
116    // avoids some IO in the builder
117    target.follow_symlinks(false);
118
119    for entry in tar_entries {
120        println!("{}", entry.name);
121        if matches!(entry.header.entry_type(), ::tar::EntryType::Directory) {
122            const NONE: &[u8] = &[];
123            target.append(&entry.header, NONE).unwrap_or_else(|e| fatal!("{e}"));
124        } else {
125            let file = std::fs::File::open(&entry.source).unwrap_or_else(|e| fatal!("{e}"));
126            target.append(&entry.header, file).unwrap_or_else(|e| fatal!("{e}"));
127        }
128    }
129
130    target.finish().unwrap_or_else(|e| fatal!("{e}"));
131    target.into_inner().unwrap().finish().unwrap_or_else(|e| fatal!("{e}"));
132}
133
134struct TarEntry {
135    source: PathBuf,
136    name: Rc<String>,
137    header: ::tar::Header,
138}
139fn collect_tar_entries(entries: Vec<Entry>) -> Vec<TarEntry> {
140    let mut tar = Vec::with_capacity(entries.len());
141    let mut names = HashSet::new();
142    for entry in entries {
143        let mut name = entry.name.as_str();
144
145        println_v!("name={name:?}");
146
147        if name.contains("/..") {
148            error!("name cannot contain /..");
149            continue;
150        }
151        if name.contains('\\') {
152            error!("name must use / slashes");
153            continue;
154        }
155        let name_is_prefix = name.ends_with("/*");
156        if name_is_prefix {
157            name = &name[..name.len() - "/*".len()];
158        }
159        if name.starts_with('/') {
160            name = &name["/".len()..];
161        }
162        if name.contains('*') {
163            error!("name can only contain * in suffix /*");
164            continue;
165        }
166
167        let mut any = false;
168        for glob_entry in glob_walker(&entry.path, true).unwrap_or_else(|e| fatal!("{e}")) {
169            let glob_entry = glob_entry.unwrap_or_else(|e| fatal!("{e}")).into_path();
170            let entry_name = if let Some(n) = glob_entry.file_name()
171                && let Some(n) = n.to_str()
172            {
173                n
174            } else {
175                continue;
176            };
177            if glob_entry.is_file() {
178                let name = if name_is_prefix {
179                    if name.is_empty() {
180                        entry_name.to_owned()
181                    } else {
182                        format!("{name}/{entry_name}")
183                    }
184                } else {
185                    if name.ends_with('/') {
186                        error!("matched file, but name ends with /");
187                        continue;
188                    }
189                    name.to_owned()
190                };
191                let name = Rc::new(name);
192
193                let meta = std::fs::metadata(&glob_entry).unwrap_or_else(|e| fatal!("{e}"));
194                let mut header = ::tar::Header::new_gnu();
195                header.set_metadata(&meta);
196                if let Err(e) = header.set_path(name.as_str()) {
197                    error!("name {name:?} is not a valid TAR path, {e}");
198                    continue;
199                }
200                header.set_cksum();
201
202                if !names.insert(name.clone()) {
203                    warn!("name {name:?} already defined, entry overwritten");
204                    let i = tar.iter().position(|t: &TarEntry| t.name == name).unwrap();
205                    tar.swap_remove(i);
206                }
207
208                tar.push(TarEntry {
209                    source: glob_entry,
210                    name,
211                    header,
212                });
213            } else if glob_entry.is_dir() {
214                let dir_prefix = if name_is_prefix {
215                    glob_entry.parent().unwrap_or_else(|| Path::new(""))
216                } else {
217                    &glob_entry
218                };
219
220                for dir_entry in walkdir::WalkDir::new(&glob_entry).follow_links(false) {
221                    let dir_entry = dir_entry.unwrap_or_else(|e| fatal!("{e}"));
222                    let dir_entry = dir_entry.path();
223
224                    if dir_entry.is_file() || dir_entry.is_dir() {
225                        let name = Path::new(&name)
226                            .join(dir_entry.strip_prefix(dir_prefix).unwrap())
227                            .display()
228                            .to_string()
229                            .replace('\\', "/")
230                            .to_owned();
231                        let name = Rc::new(name);
232
233                        let meta = std::fs::metadata(dir_entry).unwrap_or_else(|e| fatal!("{e}"));
234                        let mut header = ::tar::Header::new_gnu();
235                        header.set_metadata(&meta);
236                        if let Err(e) = header.set_path(name.as_str()) {
237                            error!("name {name:?} is not a valid TAR path, {e}");
238                            continue;
239                        }
240                        header.set_cksum();
241
242                        if !names.insert(name.clone()) {
243                            warn!("name already defined, entry overwritten");
244                            let i = tar.iter().position(|t: &TarEntry| t.name == name).unwrap();
245                            tar.swap_remove(i);
246                        }
247
248                        tar.push(TarEntry {
249                            source: dir_entry.to_path_buf(),
250                            name,
251                            header,
252                        });
253                    }
254                }
255            } else {
256                continue;
257            }
258            any = true;
259        }
260        if !any {
261            if entry.path.contains('*') {
262                warn!("no matches")
263            } else {
264                error!("no matches")
265            }
266        }
267    }
268    tar.sort_by(|a, b| a.name.cmp(&b.name));
269    tar
270}
271
272fn sum_tar_len(entries: &[TarEntry]) -> u64 {
273    struct SumWriter(u64);
274    impl io::Write for SumWriter {
275        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
276            self.0 += buf.len() as u64;
277            Ok(buf.len())
278        }
279
280        fn flush(&mut self) -> io::Result<()> {
281            Ok(())
282        }
283    }
284    let mut sum = SumWriter(0);
285    {
286        let mut tar = ::tar::Builder::new(&mut sum);
287        tar.follow_symlinks(false);
288
289        for entry in entries {
290            struct LenRead(u64);
291            impl io::Read for LenRead {
292                fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
293                    let n = self.0.min(buf.len() as u64) as usize;
294                    buf[..n].fill(0);
295                    self.0 -= n as u64;
296                    Ok(n)
297                }
298            }
299            let len = entry.header.size().unwrap();
300            tar.append(&entry.header, LenRead(len)).unwrap();
301        }
302    }
303    sum.0
304}
305
306#[derive(Deserialize)]
307struct Request {
308    #[serde(default, rename = "entry")]
309    entries: Vec<Entry>,
310    #[serde(default)]
311    filter: Filter,
312    #[serde(default)]
313    zstd: Option<ZStd>,
314    #[serde(default)]
315    gzip: Option<GZip>,
316}
317#[derive(Deserialize)]
318struct Entry {
319    path: String,
320    #[serde(default)]
321    name: String,
322}
323#[derive(Deserialize, Default)]
324struct Filter {
325    #[serde(default)]
326    bcj: String,
327}
328#[derive(Deserialize, Default)]
329struct ZStd {
330    #[serde(default = "default_zstd_level")]
331    level: i32,
332}
333fn default_zstd_level() -> i32 {
334    19
335}
336#[derive(Deserialize, Default)]
337struct GZip {
338    #[serde(default = "default_gzip_level")]
339    level: u32,
340}
341fn default_gzip_level() -> u32 {
342    7
343}
344
345/// BcjWriter and zstd::Encoder need to write a "footer" after data write
346trait WriteFinish: io::Write {
347    fn finish(self: Box<Self>) -> io::Result<()>;
348}
349impl WriteFinish for fs::File {
350    fn finish(mut self: Box<Self>) -> io::Result<()> {
351        self.flush()
352    }
353}
354impl WriteFinish for BcjWriter<Box<dyn WriteFinish>> {
355    fn finish(self: Box<Self>) -> io::Result<()> {
356        (*self).finish()?.finish()
357    }
358}
359impl WriteFinish for zstd::Encoder<'static, Box<dyn WriteFinish>> {
360    fn finish(self: Box<Self>) -> io::Result<()> {
361        (*self).finish()?.finish()
362    }
363}
364impl WriteFinish for flate2::write::GzEncoder<Box<dyn WriteFinish>> {
365    fn finish(self: Box<Self>) -> io::Result<()> {
366        (*self).finish()?.finish()
367    }
368}