Fix chunking correctness, update tooling, and relicense to OSL-3.0

.github/dependabot.yml

@@ -6,4 +6,11 @@ updates:
     interval: daily
     time: "10:00"
   open-pull-requests-limit: 10
-  rebase-strategy: disabled
+  rebase-strategy: auto
+  groups:
+    development-dependencies:
+      dependency-type: development
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/ci.yml

@@ -1,60 +1,53 @@
+name: ci
+
 on:
   push:
     branches-ignore:
       - '**.tmp'
+  pull_request:
+  workflow_dispatch:
 
-name: ci
+permissions:
+  contents: read
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        rust:
-          - stable
-          - beta
-          - nightly
-          - 1.47.0
-
+        rust: [stable, beta, nightly, '1.88.0']
     steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
         with:
-          profile: minimal
           toolchain: ${{ matrix.rust }}
-          override: true
+      - uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2
+      - run: cargo test --all-features
+      - run: cargo check --all-targets --all-features
 
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v1
+  features:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
+      - uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2
+      - run: |
+          cargo test --no-default-features
+          for feature in bup buzhash fastcdc gear gzip mii pigz ram zpaq zstd; do
+            cargo test --no-default-features --features "$feature"
+          done
 
-      - name: Run all tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all
   check:
-    runs-on: ubuntu-20.04
-
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
         with:
-          profile: minimal
-          toolchain: beta
-          override: true
           components: rustfmt, clippy
-
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v1
-
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: -- -D warnings
+      - uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2
+      - run: cargo fmt --all -- --check
+      - run: cargo clippy --all-targets --all-features -- -D warnings
+      - run: cargo bench --bench compare -- --test
+      - run: cargo doc --no-deps --all-features
+        env:
+          RUSTDOCFLAGS: -D warnings

CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+## Unreleased
+
+- License hash-roll's original code under OSL-3.0 and include upstream notices.
+- Return exclusive chunk end offsets consistently from both chunking APIs.
+  Correct Gear, FastCDC, RAM, and Zstd incremental offsets, including FastCDC's
+  forced maximum boundary and Zstd's retained window after a split.
+- Reset Bup and MII state consistently across APIs and support non-power-of-two
+  Bup windows.
+- Correct BuzHash's outgoing-byte rotation for 32-bit hashes and enforce its
+  maximum chunk size in both APIs. Correct Zpaq's slice API size accounting.
+- Stop strict slice iterators after processing their incomplete remainder once.
+- Validate Gear, FastCDC, Bup, BuzHash, Gzip, and Pigz constructor parameters.
+- Test fragmented input, empty input, multiple boundaries, independent BuzHash
+  window recomputation, and equivalence between both APIs.
+- Update development dependencies, repair benchmarks for all ten algorithms,
+  and check individual features in CI. The minimum supported Rust version is 1.88.
+- Correct documentation links, examples, and the spelling of incremental.
+
+Chunk boundaries can change because of these correctness fixes. Applications
+that persist chunk boundaries should account for this when upgrading. Published
+versions retain the licenses under which they were distributed.

COMPARE.md

@@ -1,3 +1,8 @@
+This comparison records the ecosystem as surveyed in 2020. Version numbers and
+activity descriptions below are historical. Run `cargo bench --bench compare`
+for current measurements of hash-roll's enabled algorithms and a Bup reference
+from the `rollsum` crate.
+
 
 # Algorithms
 
@@ -10,7 +15,7 @@
  - LMC (chunking)
  - RAM Chunking (Rapid  Asymmetric  Maximum)
    - doi:10.1016/j.future.2017.02.013 
- - MII (minimal incrimental interval)
+ - MII (minimal incremental interval)
    - doi:10.1109/access.2019.2926195 
  - [TTTD](https://scholarworks.sjsu.edu/cgi/viewcontent.cgi?referer=&httpsredir=1&article=1041&context=etd_projects)
  - [FBC](doi:10.1109/mascots.2010.37)
@@ -32,7 +37,7 @@
 
 # Impl Features
 
- - Incrimental input: rather than require a single `&[u8]` up front, allow
+ - Incremental input: rather than require a single `&[u8]` up front, allow
    providing a number of `&[u8]`s over the life of the splitter/hasher.
 
  - Slice input vs byte-at-a-time: By allowing algorithms to take in larger
@@ -45,8 +50,8 @@
    - latest release: 2017-09-09
      - inactive development (as of 2020-06-21)
    - algorithm(s): "Rabin64" (polynomial based, 64-bit)
-   - incrimental input: no
-     - no documentation indicates incrimental input is possible
+   - incremental input: no
+     - no documentation indicates incremental input is possible
      - while one could use a special impl of `Iterator<Item=u8>` that can be
        extended, this would only work if the `SeperatorIter` or `ChunkIter` had
        not emitted a final incomplete chunk/seperator.
@@ -67,7 +72,7 @@
    - latest release: 2020-03-19, v1.0.3
      - active development (as of 2020-06-21)
    - algorithm(s): FastCDC
-   - incrimental input: no
+   - incremental input: no
    - api:
      - input: one `&[u8]`
      - output: `Iterator<Item=Chunk> where Chunk: (offset: usize, size:
@@ -82,7 +87,7 @@
    - latest release: 2018-12-17 v1.0.0 (no other releases)
      - inactive development (as of 2020-06-21)
    - algorithm(s): AE (with modifications/extensions)
-   - incrimental input: no
+   - incremental input: no
    - api:
      - input: one `&[u8]`
      - output: `Iterator<Item=&[u8]>`
@@ -96,7 +101,7 @@
    - latest release: 2020-04-12 v0.1.3
      - active development (as of 2020-06-21)
    - algorithm(s): gear
-   - incrimental input: yes
+   - incremental input: yes
    - provides simd & scalar impls
    - includes a static table for gearhash
    - api: call `next_match()` repeatedly with new slices. Returns a
@@ -130,7 +135,7 @@
    - algorithm(s):
      - rollsum (based on bupsplit, based on rsync chunking)
      - gear
-   - incrimental input: yes
+   - incremental input: yes
    - includes a static table for gearhash
    - low level trait has byte-by-byte and slice based interfaces
    - exposes conditionality of chunk edge (ie: like a rolling-sum) in trait,

Cargo.toml

@@ -4,11 +4,13 @@ version = "0.3.0"
 authors = ["Cody P Schafer <dev@codyps.com>"]
 description = "Rolling hashes & Content Defined Chunking (cdc)"
 keywords = [ "hash", "rolling", "incremental", "split" , "cdc"]
-license = "AGPL-3.0-or-later"
-repository = "https://github.com/jmesmon/hash-roll.git"
+license = "OSL-3.0"
+repository = "https://github.com/codyps/hash-roll.git"
 documentation = "https://docs.rs/hash-roll"
-include = ["Cargo.toml", "**/*.rs", "README.md", "COMPARE.md"]
+include = ["Cargo.toml", "**/*.rs", "README.md", "COMPARE.md", "CHANGELOG.md", "LICENSE-OSL-3.0", "THIRD-PARTY-NOTICES"]
 edition = "2018"
+rust-version = "1.88"
+resolver = "3"
 
 [features]
 default = [
@@ -37,16 +39,14 @@ zstd = []
 
 [dependencies]
 fmt-extra = "0.2"
-#circbuf = "0.1.4"
 
 [dev-dependencies]
-rand = "0.7.3"
-histogram = "0.6"
-quickcheck = "0.9"
+rand = "0.10.3"
+quickcheck = "1.1"
 rollsum = "0.3"
-criterion = "0.3"
-rand_pcg = "0.2.1"
-proptest = "0.10.0"
+criterion = "0.8.2"
+rand_pcg = "0.10.2"
+proptest = "1.12"
 
 [[bench]]
 name = "compare"