diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..609e4c4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,237 @@ +name: Release + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build ${{ matrix.platform.target }} + strategy: + fail-fast: false + matrix: + platform: + - target: x86_64-apple-darwin + os: macos-latest + python-architecture: x64 + archive: diffenator3-x86_64-apple-darwin.tar.gz + - target: x86_64-pc-windows-msvc + os: windows-latest + python-architecture: x64 + archive: diffenator3-x86_64-pc-windows-msvc.zip + - target: i686-pc-windows-msvc + os: windows-latest + python-architecture: x86 + archive: diffenator3-i686-pc-windows-msvc.zip + runs-on: ${{ matrix.platform.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + architecture: ${{ matrix.platform.python-architecture }} + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.platform.target }} + + - name: Install protoc for google-fonts-languages + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # Install gnu-tar because BSD tar is buggy + # https://github.com/actions/cache/issues/403 + - name: Install GNU tar (macOS) + if: matrix.platform.os == 'macos-latest' + run: | + brew install gnu-tar + echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH + + - name: Build wheel (macOS universal2) and sdist + if: matrix.platform.target == 'x86_64-apple-darwin' + uses: PyO3/maturin-action@v1 + with: + target: universal2-apple-darwin + args: --release -o dist --sdist + + - name: Build wheel (without sdist) + if: matrix.platform.target != 'x86_64-apple-darwin' + uses: PyO3/maturin-action@v1 + with: + target: ${{matrix.platform.target}} + args: --release -o dist + + - name: Install wheel + shell: bash + run: | + pip install --no-index --find-links dist/ --force-reinstall diffenator3 + which diffenator3 + diffenator3 --version + + - name: Check sdist metadata + if: matrix.platform.target == 'x86_64-apple-darwin' + run: pipx run twine check dist/*.tar.gz + + - name: Archive binary + if: matrix.platform.os != 'windows-latest' + run: | + cd target/${{ matrix.platform.target }}/release + tar czvf ../../../${{ matrix.platform.archive }} diffenator3 diff3proof + cd - + + - name: Archive binary (windows) + if: matrix.platform.os == 'windows-latest' + run: | + cd target/${{ matrix.platform.target }}/release + 7z a ../../../${{ matrix.platform.archive }} diffenator3.exe diff3proof.exe + cd - + + - name: Archive binary (macOS aarch64) + if: matrix.platform.os == 'macos-latest' + run: | + cd target/aarch64-apple-darwin/release + tar czvf ../../../diffenator3-aarch64-apple-darwin.tar.gz diffenator3 diff3proof + cd - + + - name: Upload wheel artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform.target }} + path: dist + + - name: Upload binary artifacts + uses: actions/upload-artifact@v4 + with: + name: binaries-${{ matrix.platform.target }} + path: | + *.tar.gz + *.zip + + build_linux: + name: Build ${{ matrix.platform.target }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - target: "x86_64-unknown-linux-musl" + image_tag: "x86_64-musl" + compatibility: "manylinux2010 musllinux_1_1" + - target: "aarch64-unknown-linux-musl" + image_tag: "aarch64-musl" + compatibility: "manylinux2014 musllinux_1_1" + container: + image: docker://ghcr.io/rust-cross/rust-musl-cross:${{ matrix.platform.image_tag }} + steps: + - uses: actions/checkout@v3 + + - name: Build Wheels + uses: PyO3/maturin-action@main + with: + target: ${{ matrix.platform.target }} + manylinux: ${{ matrix.platform.compatibility }} + container: off + args: --release -o dist + + - name: Install x86_64 wheel + if: startsWith(matrix.platform.target, 'x86_64') + run: | + /usr/bin/python3 -m pip install --no-index --find-links dist/ --force-reinstall diffenator3 + which diffenator3 + diffenator3 --version + + - name: Archive binary + run: tar czvf target/release/diffenator3-${{ matrix.platform.target }}.tar.gz -C target/${{ matrix.platform.target }}/release diffenator3 diff3proof + + - name: Upload wheel artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform.target }} + path: dist + + - name: Upload binary artifacts + uses: actions/upload-artifact@v4 + with: + name: binaries-${{ matrix.platform.target }} + path: target/release/diffenator3-${{ matrix.platform.target }}.tar.gz + + release-pypi: + name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build_linux] + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true + + release-github: + permissions: + contents: write + name: Publish to GitHub releases + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build_linux] + steps: + - uses: actions/download-artifact@v4 + with: + pattern: binaries-* + merge-multiple: true + + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: wheels + + - name: Generate requirements.txt with SHA256 hashes + run: | + pipx install pip-tools + pipx runpip pip-tools install 'pip==25.0.1' + echo diffenator3 | pip-compile - --no-index --find-links wheels/ --no-emit-find-links --generate-hashes --pip-args '--only-binary=:all:' --no-annotate --no-header --output-file requirements.txt + + - name: Compute checksums of release assets + run: | + sha256sum *.tar.gz *.zip requirements.txt > checksums.txt + + - name: Detect if tag is a pre-release + id: before_release + env: + PRERELEASE_TAG_PATTERN: "v[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+([ab]|rc)[[:digit:]]+" + run: | + TAG_NAME="${GITHUB_REF##*/}" + if egrep -q "$PRERELEASE_TAG_PATTERN" <<< "$TAG_NAME"; then + echo "Tag ${TAG_NAME} contains a pre-release suffix" + echo "is_prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "Tag ${TAG_NAME} does not contain a pre-release suffix" + echo "is_prerelease=false" >> "$GITHUB_OUTPUT" + fi + echo "release_title=${TAG_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: Release + uses: softprops/action-gh-release@v2 + with: + name: ${{ steps.before_release.outputs.release_title }} + files: | + *.tar.gz + *.zip + checksums.txt + requirements.txt + prerelease: ${{ steps.before_release.outputs.is_prerelease }} + generate_release_notes: true diff --git a/Cargo.lock b/Cargo.lock index dee1622..b3ec564 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,6 +571,8 @@ dependencies = [ "colored", "diffenator3-lib", "env_logger", + "fancy-regex", + "google-fonts-languages", "indexmap 1.9.3", "itertools 0.13.0", "log", @@ -579,6 +581,7 @@ dependencies = [ "serde_json", "shaperglot", "skrifa", + "tera", "tabled", "ttj", "typescript-type-def", diff --git a/diffenator3-cli/Cargo.toml b/diffenator3-cli/Cargo.toml index 955f0ce..01a97b3 100644 --- a/diffenator3-cli/Cargo.toml +++ b/diffenator3-cli/Cargo.toml @@ -12,6 +12,9 @@ license = "Apache-2.0" name = "diffenator3" path = "src/main.rs" +[[bin]] +name = "diff3proof" +path = "src/bin/diff3proof.rs" [lib] crate-type = ["cdylib", "rlib"] path = "src/lib.rs" @@ -34,6 +37,11 @@ colored = "2.1.0" clap = { version = "4.5.9", features = ["derive"] } itertools = "0.13.0" env_logger = "0.11" + +# diff3proof dependencies +google-fonts-languages = "0" +tera = "1" +fancy-regex = "0.13" log = { workspace = true } shaperglot = { workspace = true } tabled = "0.20.0" diff --git a/diffenator3-cli/src/bin/diff3proof.rs b/diffenator3-cli/src/bin/diff3proof.rs new file mode 100644 index 0000000..a1374b7 --- /dev/null +++ b/diffenator3-cli/src/bin/diff3proof.rs @@ -0,0 +1,213 @@ +use std::collections::HashMap; +use std::path::Path; +use std::{collections::HashSet, path::PathBuf}; + +/// Create before/after HTML proofs of two fonts +// In a way this is not related to the core goal of diffenator3, but +// at the same time, we happen to have all the moving parts required +// to make this, and it would be a shame not to use them. +use clap::Parser; +use diffenator3_lib::dfont::{shared_axes, DFont}; +use diffenator3_lib::html::{gen_html, template_engine}; +use env_logger::Env; +use google_fonts_languages::{SampleTextProto, LANGUAGES, SCRIPTS}; +use serde_json::json; + +#[derive(Parser, Debug, clap::ValueEnum, Clone, PartialEq)] +enum SampleMode { + /// Sample text emphasises real language input + Context, + /// Sample text optimizes for codepoint coverage + Cover, +} + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Cli { + /// Output directory for HTML + #[clap(long = "output", default_value = "out")] + output: String, + + /// Directory for custom templates + #[clap(long = "templates")] + templates: Option, + + /// Update diffenator3's stock templates + #[clap(long = "update-templates")] + update_templates: bool, + + /// Point size for sample text in pixels + #[clap(long = "point-size", default_value = "25")] + point_size: u32, + + /// Choice of sample text + #[clap(long = "sample-mode", default_value = "context")] + sample_mode: SampleMode, + + /// Update + /// The first font file to compare + font1: PathBuf, + /// The second font file to compare + font2: Option, +} + +fn main() { + let cli = Cli::parse(); + env_logger::Builder::from_env(Env::default().default_filter_or("warn")).init(); + + let font_binary_a = std::fs::read(&cli.font1).expect("Couldn't open file"); + + let tera = template_engine(cli.templates.as_ref(), cli.update_templates); + let font_a = DFont::new(&font_binary_a); + + let (shared_codepoints, axes, instances) = if let Some(font2) = &cli.font2 { + let font_binary_b = std::fs::read(&font2).expect("Couldn't open file"); + let font_b = DFont::new(&font_binary_b); + + let shared_codepoints: HashSet = font_a + .codepoints + .intersection(&font_b.codepoints) + .copied() + .collect(); + let (axes, instances) = shared_axes(&font_a, &font_b); + (shared_codepoints, axes, instances) + } else { + let shared_codepoints = font_a.codepoints.clone(); + let (axes, instances) = shared_axes(&font_a, &font_a); + (shared_codepoints, axes, instances) + }; + + let axes_instances = serde_json::to_string(&json!({ + "axes": axes, + "instances": instances + })) + .unwrap(); + + let mut variables = serde_json::Map::new(); + variables.insert("axes_instances".to_string(), axes_instances.into()); + match cli.sample_mode { + SampleMode::Context => { + let sample_texts = language_sample_texts(&shared_codepoints); + variables.insert("language_samples".to_string(), json!(sample_texts)); + } + SampleMode::Cover => { + let sample_text = cover_sample_texts(&shared_codepoints); + variables.insert("cover_sample".to_string(), json!(sample_text)); + } + } + + gen_html( + &cli.font1, + &cli.font2.unwrap_or_else(|| cli.font1.clone()), + Path::new(&cli.output), + tera, + "diff3proof.html", + &variables.into(), + "diff3proof.html", + cli.point_size, + ); +} + +fn longest_sampletext(st: &SampleTextProto) -> &str { + if let Some(text) = &st.specimen_16 { + return text; + } + if let Some(text) = &st.specimen_21 { + return text; + } + if let Some(text) = &st.specimen_32 { + return text; + } + if let Some(text) = &st.specimen_36 { + return text; + } + if let Some(text) = &st.specimen_48 { + return text; + } + if let Some(text) = &st.tester { + return text; + } + "" +} + +fn language_sample_texts(codepoints: &HashSet) -> HashMap> { + let mut texts = HashMap::new(); + let re = fancy_regex::Regex::new(r"^(.{20,})(\1)").unwrap(); + let mut seen_cps = HashSet::new(); + // Sort languages by number of speakers + let mut languages: Vec<_> = LANGUAGES.values().collect(); + languages.sort_by_key(|lang| -lang.population.unwrap_or(0)); + + for lang in languages.iter() { + if let Some(sample) = lang.sample_text.as_ref().map(longest_sampletext) { + let mut sample = sample.replace('\n', " "); + let sample_chars = sample.chars().map(|c| c as u32).collect::>(); + + // Can we render this text? + if !sample_chars.is_subset(codepoints) { + continue; + } + // Does this add anything new to the mix? + if sample_chars.is_subset(&seen_cps) { + continue; + } + seen_cps.extend(sample_chars); + let script = lang.script(); + let script_name = SCRIPTS.get(script).unwrap().name(); + // Remove repeated phrases + if let Ok(Some(captures)) = re.captures(&sample) { + sample = captures.get(1).unwrap().as_str().to_string(); + } + texts + .entry(script_name.to_string()) + .or_insert_with(Vec::new) + .push((lang.name().to_string(), sample.to_string())); + } + } + texts +} + +fn cover_sample_texts(codepoints: &HashSet) -> String { + // Create a bag of shapable words + let mut words = HashSet::new(); + let mut languages: Vec<_> = LANGUAGES.values().collect(); + languages.sort_by_key(|lang| -lang.population.unwrap_or(0)); + + for lang in languages.iter() { + if let Some(sample) = lang.sample_text.as_ref().map(longest_sampletext) { + let sample = sample.replace('\n', " "); + for a_word in sample.split_whitespace() { + let word_chars = a_word.chars().map(|c| c as u32).collect::>(); + // Can we render this text? + if !word_chars.is_subset(codepoints) { + continue; + } + words.insert(a_word.to_string()); + } + } + } + + // Now do the greedy cover + let mut uncovered_codepoints = codepoints.clone(); + let mut best_words = vec![]; + let mut prev_count = usize::MAX; + while !uncovered_codepoints.is_empty() { + if uncovered_codepoints.len() == prev_count { + break; + } + prev_count = uncovered_codepoints.len(); + let best_word = words + .iter() + .max_by_key(|word| { + let word_chars = word.chars().map(|c| c as u32).collect::>(); + word_chars.intersection(&uncovered_codepoints).count() + }) + .unwrap(); + for char in best_word.chars() { + uncovered_codepoints.remove(&(char as u32)); + } + best_words.push(best_word.to_string()); + } + best_words.sort(); + best_words.join(" ") +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ade9960 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +manifest-path = "diffenator3-cli/Cargo.toml" +bindings = "bin" +include = [ + { path = "README.md", format = "sdist" }, + { path = "LICENSE.md", format = "sdist" }, +] +exclude = [ + { path = "diffenator3-cli/**/*", format = "wheel" }, +] + +[project] +name = "diffenator3" +description = "A utility for comparing two font files" +dynamic = ["version"] +readme = "README.md"