Add mechamism to lock edition definition and how to evolve objects in editions

.github/workflows/ci.yml

@@ -729,6 +729,9 @@ jobs:
         with:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+        with:
+          # check-editions compares against the merge base, so it needs real history.
+          fetch-depth: 0
       - uses: ./.github/actions/setup-prebuild
         with:
           enable-sccache: "true"
@@ -738,6 +741,16 @@ jobs:
         run: |
           cargo run --profile ci -p xtask -- generate-fbs
           cargo run --profile ci -p xtask -- generate-proto
+      - name: "regenerate the edition records"
+        run: |
+          cargo run --profile ci -p xtask -- generate-editions
+      - name: "check frozen edition records never change"
+        # Independent of the regeneration above: a stale record must not mask a frozen one
+        # being edited, nor the other way round.
+        if: "!cancelled()"
+        run: |
+          BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}"
+          cargo run --profile ci -p xtask -- check-editions --base "$BASE"
       - name: "regenerate FFI header file"
         run: |
           cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi

Cargo.toml

@@ -171,6 +171,7 @@ geo-types = "0.7.19"
 geoarrow = "0.8.0"
 geoarrow-cast = "0.8.0"
 get_dir = "0.5.0"
+git2 = { version = "0.21", default-features = false }
 glob = "0.3.2"
 goldenfile = "1"
 half = { version = "2.7.1", features = ["std", "num-traits"] }
@@ -276,6 +277,7 @@ thiserror = "2.0.3"
 tokio = { version = "1.52" }
 tokio-stream = "0.1.17"
 tokio-util = "0.7.17"
+toml = "0.9"
 tpchgen = "3.0.0"
 tpchgen-arrow = "3.0.0"
 tracing = { version = "0.1.41", default-features = false }

encodings/alp/src/alp/plugin.rs

@@ -7,17 +7,18 @@
 //! This enables zero-cost backward compatibility with previously written datasets.
 
 use vortex_array::Array;
+use vortex_array::ArrayContext;
+use vortex_array::ArrayDeserialization;
 use vortex_array::ArrayId;
 use vortex_array::ArrayPlugin;
 use vortex_array::ArrayRef;
+use vortex_array::ArraySerialization;
 use vortex_array::ArrayVTable;
 use vortex_array::IntoArray;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::Patched;
-use vortex_array::buffer::BufferHandle;
-use vortex_array::dtype::DType;
-use vortex_array::serde::ArrayChildren;
 use vortex_error::VortexResult;
+use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_session::VortexSession;
 
@@ -41,23 +42,31 @@ impl ArrayPlugin for ALPPatchedPlugin {
     fn serialize(
         &self,
         array: &ArrayRef,
+        ctx: &ArrayContext,
         session: &VortexSession,
-    ) -> VortexResult<Option<Vec<u8>>> {
+    ) -> VortexResult<Option<ArraySerialization>> {
         // Delegate to ALP's metadata serde
-        ALP.serialize(array, session)
+        ArrayPlugin::serialize(&ALP, array, ctx, session)
     }
 
     fn deserialize(
         &self,
-        dtype: &DType,
-        len: usize,
-        metadata: &[u8],
-        buffers: &[BufferHandle],
-        children: &dyn ArrayChildren,
+        parts: ArrayDeserialization<'_>,
         session: &VortexSession,
     ) -> VortexResult<ArrayRef> {
+        vortex_ensure!(
+            parts.serialized_id == self.id(),
+            "ALP plugin does not recognize serialized ID {}",
+            parts.serialized_id,
+        );
         let alp_array = Array::<ALP>::try_from_parts(ArrayVTable::deserialize(
-            &ALP, dtype, len, metadata, buffers, children, session,
+            &ALP,
+            parts.dtype,
+            parts.len,
+            parts.metadata,
+            parts.buffers,
+            parts.children,
+            session,
         )?)
         .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?;
 
@@ -91,6 +100,8 @@ mod tests {
     use std::f64::consts::PI;
     use std::sync::LazyLock;
 
+    use vortex_array::ArrayContext;
+    use vortex_array::ArrayDeserialization;
     use vortex_array::ArrayPlugin;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
@@ -133,7 +144,9 @@ mod tests {
 
         let array = alp_encoded.as_array();
 
-        let metadata = SESSION.array_serialize(array)?.unwrap();
+        let serialization = SESSION
+            .array_serialize(array, &ArrayContext::empty())?
+            .unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -142,11 +155,14 @@ mod tests {
             .collect::<Vec<_>>();
 
         let deserialized = ALPPatchedPlugin.deserialize(
-            array.dtype(),
-            array.len(),
-            &metadata,
-            &buffers,
-            &children,
+            ArrayDeserialization::new(
+                ALPPatchedPlugin.id(),
+                array.dtype(),
+                array.len(),
+                &serialization.metadata,
+                &buffers,
+                &children,
+            ),
             &SESSION,
         )?;
 
@@ -182,7 +198,9 @@ mod tests {
 
         let array = alp_encoded.as_array();
 
-        let metadata = SESSION.array_serialize(array)?.unwrap();
+        let serialization = SESSION
+            .array_serialize(array, &ArrayContext::empty())?
+            .unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -191,11 +209,14 @@ mod tests {
             .collect::<Vec<_>>();
 
         let deserialized = ALPPatchedPlugin.deserialize(
-            array.dtype(),
-            array.len(),
-            &metadata,
-            &buffers,
-            &children,
+            ArrayDeserialization::new(
+                ALPPatchedPlugin.id(),
+                array.dtype(),
+                array.len(),
+                &serialization.metadata,
+                &buffers,
+                &children,
+            ),
             &SESSION,
         )?;
 
@@ -213,7 +234,10 @@ mod tests {
     fn primitive_array_returns_error() {
         let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array();
 
-        let metadata = SESSION.array_serialize(&array).unwrap().unwrap();
+        let serialization = SESSION
+            .array_serialize(&array, &ArrayContext::empty())
+            .unwrap()
+            .unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -223,11 +247,14 @@ mod tests {
 
         // This panics because PrimitiveArray has no children and ALP requires encoded child.
         let _result = ALPPatchedPlugin.deserialize(
-            array.dtype(),
-            array.len(),
-            &metadata,
-            &buffers,
-            &children,
+            ArrayDeserialization::new(
+                ALPPatchedPlugin.id(),
+                array.dtype(),
+                array.len(),
+                &serialization.metadata,
+                &buffers,
+                &children,
+            ),
             &SESSION,
         );
     }