Skip to main content

cargo_zng/res/built_in/
sfx.rs

1use std::{
2    fmt::Write as _,
3    io::{Read, Seek},
4};
5
6use indexmap::IndexMap;
7use lzma_rust2::filter::bcj::BcjWriter;
8use serde::Deserialize;
9
10use crate::util::unix_path;
11
12use super::*;
13
14const SFX_HELP: &str = r#"
15Compile a self-extracting executable
16
17The request file:
18  source/sfx-package 'sfx
19   | [sfx]
20   | # executable to run, required
21   | run = "target/release/run"
22   |
23   | # Embedded icon for the sfx executable (Windows only)
24   | icon = "res/sfx.ico"
25   |
26   | # optional args for 'run'
27   | args = ["--foo"]
28   | # optional extra env for 'run'
29   | env = {
30   |     FOO = "bar",
31   | }
32   |
33   | # rustc target triple, default is the host triple
34   | # rustc-target = "x86_64-pc-windows-msvc"
35   | # build a console exe on Windows, default is true (build a GUI exe)
36   | windows-subsystem = false
37   |
38   | # compression to use for 'run', default is "zstd-bcj"
39   | # compress = "none"
40   |
41   | # data the sfx can serve the 'run'
42   | [[data]]
43   | # name must be unique and not include ':' or '\n', default is "", for single data
44   | name = "payload"
45   | # compress data on build, default is "zstd"
46   | compress = "zstd"
47   | # file to include
48   | file = "./data.tar"
49   | 
50   | [sign]
51   | # optional, code sign the sfx exe
52   | tool = "signtool sign /v /f $PFX /tr http://timestamp.sectigo.com /td SHA256 /fd SHA256 $SIGN_TARGET"
53   | # only sign the sfx exe, default 'false' signs the 'run' exe too
54   | # sfx-only = true
55
56Compiles and signs a 'sfx-package.exe' with custom icon on Windows, or a 'sfx-package' on Unix.
57
58Run:
59
60When sfx runs it extracts the 'run' executable to a temp dir and runs it.
61
62The optional 'env' variables override the system env. The SFX_ARGS var is always set. 
63
64The SFX_ARGS is set to the sfx command line args, '\n' separated. The first arg is the path to the sfx exe.
65
66On build, also searches for "$run.exe" if "$run" is not found and has no extension.
67
68Data:
69
70To read data the 'run' exe must spawn another instance of the sfx with the "SFX_GET_DATA" set
71to the entry name. It will serve the data to stdout. The data may be decompressed on demand.
72
73To get a list of data names and decompressed lengths run with "SFX_GET_MANIFEST", each stdout
74line is <name>:<len>, <len> is an u64 or "unknown".
75
76The `zng::setup::SfxClient` can also be used to connect and get data.
77
78File Paths:
79
80Paths are relative to the Cargo workspace root, you can also use 'rp to select files in the
81resource target dir.
82
83This request file:
84  source/sfx-package 'rp'sfxf
85   | [[data]]
86   | file = "${ZR_TARGET_DD}/res.txt"
87
88Compiles a 'sfx-package' that includes the 'res.txt' copied to the target dir by `cargo zng res`.
89
90Compress:
91
92The sfx exe includes a zstd decompressor that is used to extract the 'run' exe.
93
94The decompressor code can be used to read data too. The 'compress' field values are:
95
96"none" — No compression on build. Data is served as is.
97"zstd" — Compress on build unless file is already zstd (magic number check). Decompress on demand while reading.
98"zstd-[filter]" — Transform data to improve compression, unless file is already zstd. Reverses
99  transform on demand while reading.
100
101Sfx is optimized for small number of large data entries. Use a container format to
102package many small entries.
103
104Filter:
105
106Currently only BCJ (Branch/Call/Jump) filters are supported, identified by CPU instruction set:
107
108"zstd-bcj-[set]" where [set] is: "x86", "arm", "arm64", "arm-thumb", "ppc", "sparc", "ia64", "riscv".
109"zstd-bcj" — Select filter from 'rustc-target' arch, or zstd unfiltered for no matches.
110
111The target file must be a binary (exe or lib) or a container (like tar) with only binary entries. The filters
112are non-destructive but if the wrong filter is selected it will have negative impact on the compression level.
113
114Signing:
115
116Code signing must be applied to both the run exe and sfx exe, to facilitate this you can set the 'sign.tool'.
117
118The sign-tool command will run twice, with $SIGN_TARGET set to "./run.exe" and "package.exe".
119
120In the example above The $PFX var is an example of how to set the the private key. 
121Keep the private key file outside the repository and set an env var to it. In CI use
122secure variables.
123
124Icon:
125
126On Windows the sfx executable icon can be set with 'icon' field. Note that this requires the build
127to run on a Windows machine with MSVC Toolkit installed. Cross-compilation from other systems will not work.
128
129"#;
130pub(super) fn sfx() {
131    help(SFX_HELP);
132
133    let request = std::fs::read_to_string(path(ZR_REQUEST)).unwrap_or_else(|e| fatal!("{e}"));
134    let request: Request = toml::from_str(&request).unwrap_or_else(|e| fatal!("{e}"));
135
136    // make target-temp/src
137    let target = path(ZR_TARGET);
138    let tmp = target.with_file_name(format!("{}-temp", target.file_name().unwrap().display()));
139    fs::create_dir(&tmp).unwrap_or_else(|e| fatal!("cannot create {}, {e}", tmp.display()));
140    let src = tmp.join("src");
141    fs::create_dir(&src).unwrap_or_else(|e| fatal!("cannot create src, {e}"));
142
143    let mut icon = request.sfx.icon;
144    if icon.is_some() && (!cfg!(windows) || !request.sfx.rustc_target.contains("windows")) {
145        warn!("ignoring icon, can only build for Windows on Windows");
146        icon = None;
147    }
148
149    // write target-temp/Cargo.toml
150    let cargo = sfx_cargo(icon.is_some());
151    let manifest = tmp.join("Cargo.toml");
152    fs::write(&manifest, cargo.as_bytes()).unwrap_or_else(|e| fatal!("cannot create Cargo.toml, {e}"));
153
154    // write target-temp/build.rs if needed
155    if let Some(build_rs) = sfx_build(icon) {
156        let build = tmp.join("build.rs");
157        fs::write(&build, build_rs.as_bytes()).unwrap_or_else(|e| fatal!("cannot create build.rs, {e}"));
158    }
159
160    let mut run = request.sfx.run;
161    if !run.exists() {
162        if run.extension().is_none() && run.set_extension("exe") {
163            if !run.exists() {
164                run.set_extension("");
165                fatal!(
166                    "cannot find 'run' executable\n    {}\n    also tried {}.exe",
167                    unix_path(&run),
168                    run.file_name().unwrap().display()
169                );
170            }
171        } else {
172            fatal!("cannot find 'run' executable\n    {}", unix_path(&run));
173        }
174    }
175
176    if let Some(tool) = &request.sign.tool
177        && !request.sign.only_sfx
178    {
179        let sr = tmp.join("signed-run");
180        fs::copy(&run, &sr).unwrap_or_else(|e| fatal!("cannot copy {}, {e}", run.display()));
181        run = sr.clone();
182        // SAFETY: tools run single threaded
183        unsafe { std::env::set_var("SIGN_TARGET", &*unix_path(&sr)) };
184        super::sh_run(tool.clone(), false, None).unwrap_or_else(|e| fatal!("cannot sign run, {e}"));
185    }
186
187    let mut data = vec![];
188    let mut run_compression = parse_compress(&request.sfx.rustc_target, &request.sfx.compress);
189    let run_parts = prepare_data(&tmp, 0, &mut run_compression, &run).unwrap_or_else(|e| fatal!("cannot compress run, {e}"));
190    data.push((":run", run_compression, run_parts));
191    for (id, d) in request.data.iter().enumerate() {
192        if d.name.contains(':') || d.name.contains('\n') {
193            fatal!("data name cannot contain ':' or '\n'");
194        }
195        let mut compression = parse_compress(&request.sfx.rustc_target, &d.compress);
196        let file = d.file.as_path();
197
198        let parts = prepare_data(&tmp, id + 1, &mut compression, file)
199            .unwrap_or_else(|e| fatal!("cannot process file, {e}\n    file: {}", unix_path(file)));
200        data.push((d.name.as_str(), compression, parts));
201    }
202    let sfx_main = sfx_main(request.sfx.windows_subsystem, &request.sfx.args, &request.sfx.env, &data);
203    fs::write(src.join("main.rs"), sfx_main.as_bytes()).unwrap_or_else(|e| fatal!("cannot create main.rs, {e}"));
204
205    let r = std::process::Command::new("cargo")
206        .arg("build")
207        .arg("--release")
208        .arg("--target")
209        .arg(&request.sfx.rustc_target)
210        .arg("--manifest-path")
211        .arg(manifest)
212        .status()
213        .unwrap_or_else(|e| fatal!("{e}"));
214    assert!(r.success());
215
216    let output = tmp.join(format!(
217        "target/{}/release/zng-res-sfx{}",
218        request.sfx.rustc_target,
219        std::env::consts::EXE_SUFFIX
220    ));
221    if let Some(tool) = request.sign.tool {
222        // SAFETY: tools run single threaded
223        unsafe { std::env::set_var("SIGN_TARGET", &*unix_path(&output)) };
224        super::sh_run(tool, false, None).unwrap_or_else(|e| fatal!("cannot sign sfx, {e}"));
225    }
226
227    let mut target = target;
228    target.set_extension(output.extension().unwrap_or_default());
229    fs::rename(output, target).unwrap_or_else(|e| fatal!("cannot finalize build, {e}"));
230
231    fs::remove_dir_all(tmp).unwrap_or_else(|e| fatal!("cannot cleanup, {e}"));
232}
233
234fn parse_compress(rustc_target: &str, compress: &str) -> Compression {
235    if let Some(filter) = compress.strip_prefix("zstd-") {
236        match filter {
237            "bcj-x86" => Compression::ZstdBcj(BcjFilter::X86),
238            "bcj-arm" => Compression::ZstdBcj(BcjFilter::Arm),
239            "bcj-arm64" => Compression::ZstdBcj(BcjFilter::Arm64),
240            "bcj-arm-thumb" => Compression::ZstdBcj(BcjFilter::ArmThumb),
241            "bcj-arm-ppc" => Compression::ZstdBcj(BcjFilter::Ppc),
242            "bcj-arm-sparc" => Compression::ZstdBcj(BcjFilter::Sparc),
243            "bcj-arm-ia64" => Compression::ZstdBcj(BcjFilter::Ia64),
244            "bcj-arm-riscv" => Compression::ZstdBcj(BcjFilter::Riscv),
245            "bcj" => bcj_from_triple(rustc_target).map(Compression::ZstdBcj).unwrap_or(Compression::Zstd),
246            unk => fatal!("unknown filter {unk:?}"),
247        }
248    } else {
249        match compress {
250            "none" => Compression::None,
251            "zstd" => Compression::Zstd,
252            unk => fatal!("unknown compression {unk:?}"),
253        }
254    }
255}
256
257#[derive(Deserialize)]
258struct Request {
259    sfx: Sfx,
260    #[serde(default)]
261    data: Vec<Data>,
262    #[serde(default = "default_sign")]
263    sign: Sign,
264}
265fn default_sign() -> Sign {
266    Sign {
267        tool: None,
268        only_sfx: false,
269    }
270}
271
272#[derive(Deserialize)]
273struct Sfx {
274    run: PathBuf,
275    icon: Option<PathBuf>,
276    #[serde(default)]
277    args: Vec<String>,
278    #[serde(default)]
279    env: indexmap::IndexMap<String, String>,
280    #[serde(rename = "rustc-target")]
281    #[serde(default = "rustc_host_triple")]
282    rustc_target: String,
283    #[serde(rename = "windows-subsystem")]
284    #[serde(default = "default_windows_subsystem")]
285    windows_subsystem: bool,
286    #[serde(default = "default_run_compress")]
287    compress: String,
288}
289fn default_windows_subsystem() -> bool {
290    true
291}
292fn default_run_compress() -> String {
293    "zstd-bcj".to_owned()
294}
295
296#[derive(Deserialize)]
297struct Data {
298    name: String,
299    #[serde(default = "default_compress")]
300    compress: String,
301    file: PathBuf,
302}
303fn default_compress() -> String {
304    "zstd".to_owned()
305}
306
307#[derive(Deserialize)]
308struct Sign {
309    tool: Option<String>,
310    #[serde(default)]
311    only_sfx: bool,
312}
313
314fn prepare_data(tmp: &Path, data_id: usize, decompress: &mut Compression, file: &Path) -> io::Result<Vec<PathBuf>> {
315    let file_path = file;
316    let mut file = fs::File::open(file)?;
317
318    // 200MB
319    //
320    // Difficult to find exact limits for `include_bytes!`, Windows maybe has a 2GB limit per object.
321    //
322    // Static data is memory mapped to RAM on demand, so we depend on the system pagination to keep RAM
323    // usage down. The split into 200MB parts here is to hopefully provide extra clear hinting that an
324    // object is no longer needed as the sfx iterates over parts.
325    const PART_MAX: u64 = 200u64 * 2u64.pow(20);
326
327    let len = file.metadata()?.len();
328    if len < 32 {
329        // not worth compressing, zstd header alone is ~18 bytes
330        *decompress = Compression::None;
331    }
332
333    let mut compress = *decompress;
334
335    if matches!(compress, Compression::Zstd) {
336        let mut magic = [0u8; 4];
337        file.read_exact(&mut magic)?;
338        file.seek(io::SeekFrom::Start(0))?;
339        if magic == [0x28, 0xB5, 0x2F, 0xFD] {
340            // already zstd
341            compress = Compression::None;
342        }
343    }
344
345    match compress {
346        Compression::None => {
347            println!("preparing {}", unix_path(file_path));
348
349            if len <= PART_MAX {
350                return Ok(vec![file_path.to_owned()]);
351            }
352
353            let mut len = len;
354
355            let mut file = io::BufReader::new(file);
356            let mut parts = vec![];
357            loop {
358                let part_path = tmp.join(format!("d{data_id}-p{}", parts.len()));
359                let mut part = fs::File::create_new(&part_path)?;
360                parts.push(part_path);
361                if PART_MAX > len {
362                    part.set_len(PART_MAX)?;
363                    io::copy(&mut (&mut file).take(PART_MAX), &mut part)?;
364                    len -= PART_MAX;
365                } else {
366                    part.set_len(len)?;
367                    io::copy(&mut file, &mut part)?;
368                    // len = 0;
369                    break;
370                }
371            }
372            Ok(parts)
373        }
374        Compression::Zstd => {
375            println!("compressing {}", unix_path(file_path));
376
377            // 19 is the maximum non-ultra compression
378            // zstd creates an optimal BufReader
379            let mut file = zstd::stream::read::Encoder::new(file, 19)?;
380
381            file.set_pledged_src_size(Some(len))?;
382            file.include_contentsize(true)?;
383
384            let mut parts = vec![];
385            loop {
386                let part_path = tmp.join(format!("d{data_id}-p{}", parts.len()));
387                let mut part = fs::File::create_new(&part_path)?;
388                parts.push(part_path);
389
390                let part_len = io::copy(&mut (&mut file).take(PART_MAX), &mut part)?;
391                if part_len < PART_MAX {
392                    break;
393                }
394            }
395
396            Ok(parts)
397        }
398        Compression::ZstdBcj(bcj) => {
399            println!("compressing {}", unix_path(file_path));
400
401            // There is no pull-based bcj encoder so the parts swap needs to happen inside
402            struct PartsWriter<'a> {
403                tmp: &'a Path,
404                data_id: usize,
405                parts: &'a mut Vec<PathBuf>,
406                part: Option<fs::File>,
407                left: usize,
408            }
409            impl<'a> PartsWriter<'a> {
410                fn next_part(&mut self) -> io::Result<()> {
411                    let part_path = self.tmp.join(format!("d{}-p{}", self.data_id, self.parts.len()));
412                    self.part = Some(fs::File::create_new(&part_path)?);
413                    self.parts.push(part_path);
414                    self.left = PART_MAX as usize;
415                    Ok(())
416                }
417            }
418            impl<'a> io::Write for PartsWriter<'a> {
419                fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
420                    if buf.is_empty() {
421                        return Ok(0);
422                    }
423                    if self.left == 0 {
424                        self.next_part()?;
425                    }
426                    if let Some(f) = &mut self.part {
427                        let max_len = self.left.min(buf.len());
428                        let buf = &buf[..max_len];
429                        let written = f.write(buf)?;
430                        self.left -= written;
431                        Ok(written)
432                    } else {
433                        Ok(0)
434                    }
435                }
436
437                fn flush(&mut self) -> io::Result<()> {
438                    if let Some(f) = &mut self.part { f.flush() } else { Ok(()) }
439                }
440            }
441
442            let mut parts = vec![];
443            let out = PartsWriter {
444                tmp,
445                data_id,
446                parts: &mut parts,
447                part: None,
448                left: 0,
449            };
450            // 22 is the maximum ultra compression (high CPU and RAM usage)
451            let mut encoder = zstd::stream::write::Encoder::new(out, 22)?;
452            encoder.set_pledged_src_size(Some(len))?;
453            encoder.include_contentsize(true)?;
454
455            let out = &mut encoder;
456            let mut bcj_filter = match bcj {
457                BcjFilter::X86 => BcjWriter::new_x86(out, 0),
458                BcjFilter::Arm => BcjWriter::new_arm(out, 0),
459                BcjFilter::Arm64 => BcjWriter::new_arm64(out, 0),
460                BcjFilter::ArmThumb => BcjWriter::new_arm_thumb(out, 0),
461                BcjFilter::Ppc => BcjWriter::new_ppc(out, 0),
462                BcjFilter::Sparc => BcjWriter::new_sparc(out, 0),
463                BcjFilter::Ia64 => BcjWriter::new_ia64(out, 0),
464                BcjFilter::Riscv => BcjWriter::new_riscv(out, 0),
465            };
466
467            let mut file = file;
468            io::copy(&mut file, &mut bcj_filter)?;
469            bcj_filter.finish()?;
470            encoder.finish()?;
471
472            Ok(parts)
473        }
474    }
475}
476
477fn bcj_from_triple(target: &str) -> Option<BcjFilter> {
478    let arch = target.split('-').next()?;
479
480    match arch {
481        "i386" | "i486" | "i586" | "i686" | "x86_64" => Some(BcjFilter::X86),
482        "arm" | "armv4t" | "armv5te" | "armv7" | "armv7a" | "armv7r" | "armv7s" | "armebv7r" => Some(BcjFilter::Arm),
483        "thumbv6m" | "thumbv7m" | "thumbv7em" | "thumbv7neon" | "thumbv8m.base" | "thumbv8m.main" => Some(BcjFilter::ArmThumb),
484        "aarch64" | "aarch64_be" => Some(BcjFilter::Arm64),
485        "powerpc" | "powerpc64" | "powerpc64le" => Some(BcjFilter::Ppc),
486        "sparc" | "sparcv9" | "sparc64" => Some(BcjFilter::Sparc),
487        "ia64" => Some(BcjFilter::Ia64),
488        "riscv32" | "riscv64" => Some(BcjFilter::Riscv),
489
490        _ => None,
491    }
492}
493
494// enable rust-analyzer
495#[path = "sfx_res/sfx_main.rs"]
496#[allow(unused)]
497mod sfx_main;
498use sfx_main::{BcjFilter, Compression};
499
500fn sfx_main(
501    windows_subsystem: bool,
502    args: &[String],
503    env: &IndexMap<String, String>,
504    data: &[(&str, Compression, Vec<PathBuf>)],
505) -> String {
506    let main = include_str!("sfx_res/sfx_main.rs");
507
508    const DATA: &str = "static DATA: &[(&str, Compression, &[&[u8]])] = &[";
509    let mut out_data = String::new();
510    out_data.push_str(DATA);
511    out_data.push('\n');
512    for (name, compression, parts) in data {
513        let (compression, filter) = match compression {
514            Compression::None => ("None", ""),
515            Compression::Zstd => ("Zstd", ""),
516            Compression::ZstdBcj(f) => {
517                let f = match f {
518                    BcjFilter::X86 => "(BcjFilter::X86)",
519                    BcjFilter::Arm => "(BcjFilter::Arm)",
520                    BcjFilter::Arm64 => "(BcjFilter::Arm64)",
521                    BcjFilter::ArmThumb => "(BcjFilter::ArmThumb)",
522                    BcjFilter::Ppc => "(BcjFilter::Ppc)",
523                    BcjFilter::Sparc => "(BcjFilter::Sparc)",
524                    BcjFilter::Ia64 => "(BcjFilter::Ia64)",
525                    BcjFilter::Riscv => "(BcjFilter::Riscv)",
526                };
527                ("ZstdBcj", f)
528            }
529        };
530        write!(&mut out_data, "  ({name:?}, Compression::{compression}{filter}, &[").unwrap();
531        for part in parts {
532            write!(&mut out_data, "include_bytes!(\"{}\"), ", unix_path(part)).unwrap();
533        }
534        writeln!(&mut out_data, "]),").unwrap();
535    }
536    out_data.push_str("\n];");
537
538    const ARGS: &str = "static ARGS: &[&str] = &[";
539    let mut out_args = ARGS.to_owned();
540    for arg in args {
541        write!(&mut out_args, "{arg:?}, ").unwrap();
542    }
543    out_args.push_str("];");
544
545    const ENV: &str = "static ENV: &[(&str, &str)] = &[";
546    let mut out_env = ENV.to_owned();
547    out_env.push('\n');
548    for (key, value) in env {
549        writeln!(&mut out_env, "  ({key:?}, {value:?}),").unwrap();
550    }
551    out_env.push_str("\n];");
552
553    let mut replaces = vec![(DATA, out_data), (ARGS, out_args), (ENV, out_env)];
554
555    let mut out_main = String::new();
556    let mut region = "";
557    for line in main.lines() {
558        // conditional regions
559        if let Some(r) = line.trim_start().strip_prefix("// </")
560            && let Some(r) = r.strip_suffix('>')
561        {
562            assert_eq!(region, r);
563            region = "";
564            continue;
565        }
566        if let Some(r) = line.trim_start().strip_prefix("// <")
567            && let Some(r) = r.strip_suffix('>')
568        {
569            region = r;
570            continue;
571        }
572        let keep_line = match region {
573            "windows-subsystem" => windows_subsystem,
574            "" => true,
575            unk => panic!("unknown region {unk:?}"),
576        };
577        if !keep_line {
578            continue;
579        }
580
581        if let Some(i) = replaces.iter().position(|(k, _)| line.starts_with(k)) {
582            let (_, value) = replaces.swap_remove(i);
583            out_main.push_str(&value);
584        } else {
585            out_main.push_str(line);
586        }
587
588        if replaces.is_empty() {
589            let i = line.as_ptr() as usize - main.as_ptr() as usize + line.len();
590            out_main.push_str(&main[i..]);
591            break;
592        } else {
593            out_main.push('\n');
594        }
595    }
596
597    out_main
598}
599
600fn sfx_build(icon: Option<PathBuf>) -> Option<String> {
601    let ico = icon?;
602    let build = include_str!("sfx_res/build_icon.rs");
603    let build = build.replace("{icon-path}", &ico.display().to_string());
604    Some(build)
605}
606
607fn sfx_cargo(has_icon: bool) -> String {
608    let cargo = include_str!("sfx_res/sfx_cargo.toml");
609    if has_icon {
610        cargo.to_owned()
611    } else {
612        let (before, after) = cargo.split_once("# <has-icon>\n").unwrap();
613        let (_, after) = after.split_once("\n# </has-icon>\n").unwrap();
614        format!("{before}{after}")
615    }
616}
617
618fn rustc_host_triple() -> String {
619    let o = std::process::Command::new("rustc")
620        .arg("--version")
621        .arg("--verbose")
622        .output()
623        .unwrap_or_else(|e| fatal!("cannot find host triple, {e}"));
624
625    if !o.status.success() {
626        fatal!("cannot find host triple, exit code: {:?}", o.status.code().unwrap_or(0))
627    }
628
629    let stdout = str::from_utf8(&o.stdout).unwrap_or_else(|e| fatal!("cannot find host triple, {e}"));
630
631    for line in stdout.lines() {
632        if let Some(triple) = line.strip_prefix("host: ") {
633            return triple.to_owned();
634        }
635    }
636
637    fatal!("cannot find host triple")
638}