cargo_zng/res/built_in/
tar.rs1use std::{collections::HashSet, io::Write, rc::Rc};
2
3use lzma_rust2::filter::bcj::BcjWriter;
4use serde::Deserialize;
5
6use crate::util::glob_walker;
7
8use super::*;
9
10const TAR_HELP: &str = r#"
11Pack files and dirs into a TAR container with optional compression
12
13The request file:
14 source/data.tar.zst 'tar
15 | [[entry]]
16 | path = "res/bin/*"
17 | name = "bin/*"
18 |
19 | [[entry]]
20 | path = "README.md"
21 | name = "docs/README.md"
22 |
23 | # Optional compression
24 | [zstd]
25 | level = 19
26
27Packs entries into a TAR, compresses it with ZSTD.
28
29The syntax is a TOML file, the tables are:
30
31[[entry]] — Array of entries to pack.
32path — Path or glob pattern, relative to the workspace root. Required.
33name — Optional name of the file on the TAR container. Optional.
34
35Each entry can be a file, directory or glob selection. Only file and directory
36entries are supported by this tool, other tar archive entries are not supported.
37
38If 'path' matches a directory all contained files and sub directories are packed.
39
40Matching is case insensitive in all platforms. See 'glob tool for glob syntax.
41
42If 'name' is not set it is path relative to workspace root.
43
44If 'name' is set it must not contain "/..".
45
46If 'name' ends with "/*" the selected files and dirs are named in this TAR dir.
47
48All 'name' paths are relative to the TAR root, "/foo" is packed as "foo".
49
50[filter] — Optional lossless transform to apply to the TAR
51bcj — Branch/Call/Jump filter that optimizes compression of binary code files.
52 Values: "x86", "arm", "arm64", "arm-thumb", "ppc", "sparc", "ia64", "riscv"
53 Note that decompressor must revert the filter.
54
55The decompressor must undo these changes before reading the TAR. The 'sfx supports decoding BCJ.
56
57[zstd] — Optional ZStandard compression
58level — Compression level, -131072..=22, 0 means no compression, default is 19.
59
60Compress the TAR, after [filter] is applied, using ZStandard. The "contentsize" field of zstd header
61is correctly set, so this is fully compatible with 'sfx that uses this field to report response stream length.
62
63[gzip] — Optional GZip compression
64level — Compression level, 0..=9, 0 means no compression, default is 8.
65
66"#;
67pub(super) fn tar() {
68 help(TAR_HELP);
69
70 let request = std::fs::read_to_string(path(ZR_REQUEST)).unwrap_or_else(|e| fatal!("{e}"));
71 let request: Request = toml::from_str(&request).unwrap_or_else(|e| fatal!("{e}"));
72
73 let target = path(ZR_TARGET);
74
75 let target = fs::File::create(&target).unwrap_or_else(|e| fatal!("{e}"));
76 let mut target: Box<dyn WriteFinish> = Box::new(target);
77
78 let tar_entries = collect_tar_entries(request.entries);
79
80 if let Some(zstd) = request.zstd
81 && zstd.level != 0
82 {
83 let range = zstd::compression_level_range();
84 let level = zstd.level.clamp(*range.start(), *range.end());
85 let mut encoder = zstd::Encoder::new(target, level).unwrap_or_else(|e| fatal!("{e}"));
86
87 let len = sum_tar_len(&tar_entries);
88 encoder.set_pledged_src_size(Some(len)).unwrap_or_else(|e| fatal!("{e}"));
89 encoder.include_contentsize(true).unwrap_or_else(|e| fatal!("{e}"));
90 target = Box::new(encoder);
91 } else if let Some(gzip) = request.gzip
92 && gzip.level != 0
93 {
94 let level = flate2::Compression::new(gzip.level.clamp(0, 9));
95 let encoder = flate2::write::GzEncoder::new(target, level);
96 target = Box::new(encoder);
97 }
98
99 if !request.filter.bcj.is_empty() {
100 let bcj = match request.filter.bcj.as_str() {
101 "x86" => BcjWriter::new_x86(target, 0),
102 "arm" => BcjWriter::new_arm(target, 0),
103 "arm64" => BcjWriter::new_arm64(target, 0),
104 "arm-thumb" => BcjWriter::new_arm_thumb(target, 0),
105 "ppc" => BcjWriter::new_ppc(target, 0),
106 "sparc" => BcjWriter::new_sparc(target, 0),
107 "ia64" => BcjWriter::new_ia64(target, 0),
108 "riscv" => BcjWriter::new_riscv(target, 0),
109 _ => fatal!("unknown bcj filter"),
110 };
111 target = Box::new(bcj)
112 }
113
114 let mut target = ::tar::Builder::new(target);
115 target.follow_symlinks(false);
118
119 for entry in tar_entries {
120 println!("{}", entry.name);
121 if matches!(entry.header.entry_type(), ::tar::EntryType::Directory) {
122 const NONE: &[u8] = &[];
123 target.append(&entry.header, NONE).unwrap_or_else(|e| fatal!("{e}"));
124 } else {
125 let file = std::fs::File::open(&entry.source).unwrap_or_else(|e| fatal!("{e}"));
126 target.append(&entry.header, file).unwrap_or_else(|e| fatal!("{e}"));
127 }
128 }
129
130 target.finish().unwrap_or_else(|e| fatal!("{e}"));
131 target.into_inner().unwrap().finish().unwrap_or_else(|e| fatal!("{e}"));
132}
133
134struct TarEntry {
135 source: PathBuf,
136 name: Rc<String>,
137 header: ::tar::Header,
138}
139fn collect_tar_entries(entries: Vec<Entry>) -> Vec<TarEntry> {
140 let mut tar = Vec::with_capacity(entries.len());
141 let mut names = HashSet::new();
142 for entry in entries {
143 let mut name = entry.name.as_str();
144
145 println_v!("name={name:?}");
146
147 if name.contains("/..") {
148 error!("name cannot contain /..");
149 continue;
150 }
151 if name.contains('\\') {
152 error!("name must use / slashes");
153 continue;
154 }
155 let name_is_prefix = name.ends_with("/*");
156 if name_is_prefix {
157 name = &name[..name.len() - "/*".len()];
158 }
159 if name.starts_with('/') {
160 name = &name["/".len()..];
161 }
162 if name.contains('*') {
163 error!("name can only contain * in suffix /*");
164 continue;
165 }
166
167 let mut any = false;
168 for glob_entry in glob_walker(&entry.path, true).unwrap_or_else(|e| fatal!("{e}")) {
169 let glob_entry = glob_entry.unwrap_or_else(|e| fatal!("{e}")).into_path();
170 let entry_name = if let Some(n) = glob_entry.file_name()
171 && let Some(n) = n.to_str()
172 {
173 n
174 } else {
175 continue;
176 };
177 if glob_entry.is_file() {
178 let name = if name_is_prefix {
179 if name.is_empty() {
180 entry_name.to_owned()
181 } else {
182 format!("{name}/{entry_name}")
183 }
184 } else {
185 if name.ends_with('/') {
186 error!("matched file, but name ends with /");
187 continue;
188 }
189 name.to_owned()
190 };
191 let name = Rc::new(name);
192
193 let meta = std::fs::metadata(&glob_entry).unwrap_or_else(|e| fatal!("{e}"));
194 let mut header = ::tar::Header::new_gnu();
195 header.set_metadata(&meta);
196 if let Err(e) = header.set_path(name.as_str()) {
197 error!("name {name:?} is not a valid TAR path, {e}");
198 continue;
199 }
200 header.set_cksum();
201
202 if !names.insert(name.clone()) {
203 warn!("name {name:?} already defined, entry overwritten");
204 let i = tar.iter().position(|t: &TarEntry| t.name == name).unwrap();
205 tar.swap_remove(i);
206 }
207
208 tar.push(TarEntry {
209 source: glob_entry,
210 name,
211 header,
212 });
213 } else if glob_entry.is_dir() {
214 let dir_prefix = if name_is_prefix {
215 glob_entry.parent().unwrap_or_else(|| Path::new(""))
216 } else {
217 &glob_entry
218 };
219
220 for dir_entry in walkdir::WalkDir::new(&glob_entry).follow_links(false) {
221 let dir_entry = dir_entry.unwrap_or_else(|e| fatal!("{e}"));
222 let dir_entry = dir_entry.path();
223
224 if dir_entry.is_file() || dir_entry.is_dir() {
225 let name = Path::new(&name)
226 .join(dir_entry.strip_prefix(dir_prefix).unwrap())
227 .display()
228 .to_string()
229 .replace('\\', "/")
230 .to_owned();
231 let name = Rc::new(name);
232
233 let meta = std::fs::metadata(dir_entry).unwrap_or_else(|e| fatal!("{e}"));
234 let mut header = ::tar::Header::new_gnu();
235 header.set_metadata(&meta);
236 if let Err(e) = header.set_path(name.as_str()) {
237 error!("name {name:?} is not a valid TAR path, {e}");
238 continue;
239 }
240 header.set_cksum();
241
242 if !names.insert(name.clone()) {
243 warn!("name already defined, entry overwritten");
244 let i = tar.iter().position(|t: &TarEntry| t.name == name).unwrap();
245 tar.swap_remove(i);
246 }
247
248 tar.push(TarEntry {
249 source: dir_entry.to_path_buf(),
250 name,
251 header,
252 });
253 }
254 }
255 } else {
256 continue;
257 }
258 any = true;
259 }
260 if !any {
261 if entry.path.contains('*') {
262 warn!("no matches")
263 } else {
264 error!("no matches")
265 }
266 }
267 }
268 tar.sort_by(|a, b| a.name.cmp(&b.name));
269 tar
270}
271
272fn sum_tar_len(entries: &[TarEntry]) -> u64 {
273 struct SumWriter(u64);
274 impl io::Write for SumWriter {
275 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
276 self.0 += buf.len() as u64;
277 Ok(buf.len())
278 }
279
280 fn flush(&mut self) -> io::Result<()> {
281 Ok(())
282 }
283 }
284 let mut sum = SumWriter(0);
285 {
286 let mut tar = ::tar::Builder::new(&mut sum);
287 tar.follow_symlinks(false);
288
289 for entry in entries {
290 struct LenRead(u64);
291 impl io::Read for LenRead {
292 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
293 let n = self.0.min(buf.len() as u64) as usize;
294 buf[..n].fill(0);
295 self.0 -= n as u64;
296 Ok(n)
297 }
298 }
299 let len = entry.header.size().unwrap();
300 tar.append(&entry.header, LenRead(len)).unwrap();
301 }
302 }
303 sum.0
304}
305
306#[derive(Deserialize)]
307struct Request {
308 #[serde(default, rename = "entry")]
309 entries: Vec<Entry>,
310 #[serde(default)]
311 filter: Filter,
312 #[serde(default)]
313 zstd: Option<ZStd>,
314 #[serde(default)]
315 gzip: Option<GZip>,
316}
317#[derive(Deserialize)]
318struct Entry {
319 path: String,
320 #[serde(default)]
321 name: String,
322}
323#[derive(Deserialize, Default)]
324struct Filter {
325 #[serde(default)]
326 bcj: String,
327}
328#[derive(Deserialize, Default)]
329struct ZStd {
330 #[serde(default = "default_zstd_level")]
331 level: i32,
332}
333fn default_zstd_level() -> i32 {
334 19
335}
336#[derive(Deserialize, Default)]
337struct GZip {
338 #[serde(default = "default_gzip_level")]
339 level: u32,
340}
341fn default_gzip_level() -> u32 {
342 7
343}
344
345trait WriteFinish: io::Write {
347 fn finish(self: Box<Self>) -> io::Result<()>;
348}
349impl WriteFinish for fs::File {
350 fn finish(mut self: Box<Self>) -> io::Result<()> {
351 self.flush()
352 }
353}
354impl WriteFinish for BcjWriter<Box<dyn WriteFinish>> {
355 fn finish(self: Box<Self>) -> io::Result<()> {
356 (*self).finish()?.finish()
357 }
358}
359impl WriteFinish for zstd::Encoder<'static, Box<dyn WriteFinish>> {
360 fn finish(self: Box<Self>) -> io::Result<()> {
361 (*self).finish()?.finish()
362 }
363}
364impl WriteFinish for flate2::write::GzEncoder<Box<dyn WriteFinish>> {
365 fn finish(self: Box<Self>) -> io::Result<()> {
366 (*self).finish()?.finish()
367 }
368}