chore: Migrate docs from EDoc to Markdown

.github/workflows/docs.yml

@@ -1,3 +1,3 @@
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -51,8 +51,8 @@
       - name: Install Erlang/OTP
         uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
         with:
-          otp-version: 25.1.0
-          rebar3-version: '3.18.0'
+          otp-version: '29.0'
+          rebar3-version: '3.27.0'
 
       - name: Cache Hex packages
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0

.github/workflows/erlang.yml

@@ -1,3 +1,3 @@
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -43,8 +43,8 @@
       - name: Install Erlang/OTP
         uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
         with:
-          otp-version: 25.1.0
-          rebar3-version: '3.18.0'
+          otp-version: '29.0'
+          rebar3-version: '3.27.0'
 
       - name: Cache Hex packages
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -81,8 +81,8 @@
       - name: Install Erlang/OTP
         uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
         with:
-          otp-version: 25.1.0
-          rebar3-version: '3.18.0'
+          otp-version: '29.0'
+          rebar3-version: '3.27.0'
 
       - name: Cache Hex packages
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -122,8 +122,8 @@
       - name: Install Erlang/OTP
         uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
         with:
-          otp-version: 25.1.0
-          rebar3-version: '3.18.0'
+          otp-version: '29.0'
+          rebar3-version: '3.27.0'
 
       - name: Cache Hex packages
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0

rebar.config

@@ -29,7 +29,7 @@
 ]}.
 
 {plugins, [rebar3_cargo]}.
-{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.18"}]}.
+{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.30"}]}.
 
 % Let erlfmt write files on format as opposed to not doing anything
 {erlfmt, [write]}.

src/arrow_array.erl

@@ -15,87 +15,54 @@
 % specific language governing permissions and limitations
 % under the License.
 
-%% @doc Provides a record, a behaviour, and functions to work with Apache Arrow
-%% Arrays.
-%%
-%% This module as mentioned, provides conveniences for working with Apache Arrow
-%% Arrays[1] of different Layouts[2]. Firstly, it provides a record to represent
-%% all the Layout. Secondly, it provides a behaviour for different layouts to
-%% adhere to common functionality. Lastly, it provides functions to work with
-%% Arrays in a common manner.
-%%
-%% == The Structure of an Array ==
-%%
-%% `arrow''s implementation of an Array has the following fields in its
-%% record definition:
-%%
-%% <ol>
-%%  <li>
-%%      `layout', of type {@link atom()}, which represents the Layout of the
-%%      Array.
-%%  </li>
-%%  <li>
-%%       `type', of type {@link arrow_type:arrow_type()}, which represents
-%%       the Logical Type[3] of the Array.
-%% </li>
-%%  <li>`len', of type {@link pos_integer()}, which represents the Array's Length[4].</li>
-%%  <li>
-%%      `element_len', of type {@link pos_integer()} or `undefined', which
-%%      represents the Length of each element in an Array. Currently it only has
-%%      an integer value in the Fixed-Size List Layout[5]
-%%  </li>
-%%  <li>
-%%      `null_count', of type  {@link non_neg_integer()}, which represents the
-%%      Array's Null Count[4], or the number of undefined values in the Array.
-%%  </li>
-%%  <li>
-%%      `validity_bitmap', which is a buffer (`arrow_buffer') or the atom
-%%       `undefined', which represents the Array's Validity Bitmap[6].
-%%  </li>
-%%  <li>
-%%      `offsets', which is a buffer (`arrow_buffer') which represents the
-%%      Offsets[7], or the start position of each slot in the data buffer of an
-%%      Array.
-%%  </li>
-%%  <li>
-%%      `data', which is a buffer (`arrow_buffer'), which represents the
-%%      Array's Value Buffer, whose layout differs based on the Array Layout.
-%%  </li>
-%% </ol>
-%%
-%% Certain fields are not required for certain layouts. For example, for the
-%% Fixed-Sized Primitive Layout, the offsets field is not required, in which
-%% case it is assigned as `undefined'. Similarly, the `validity_bitmap' is not
-%% required if there are no null values, in which case it is also assigned as
-%% `undefined'.
-%%
-%% == The Behaviour of an Array ==
-%%
-%% As of right now, a layout needs to implement the `c:from_erlang/2' callback. This is
-%% then used in the `from_erlang/3' function to create new arrays.
-%%
-%% == Functions for working with Arrays ==
-%%
-%% As of right now, only functions to access the various fields, to create
-%% new arrays, and to serialize arrays to arrow exist.
-%%
-%% == References ==
-%%
-%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-array]
-%%
-%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout]
-%%
-%% [3]: [https://arrow.apache.org/docs/format/Glossary.html#term-type]
-%%
-%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#null-count]
-%%
-%% [5]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout]
-%%
-%% [6]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps]
-%%
-%% [7]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout]
-%% @end
 -module(arrow_array).
+
+%% TODO Check formatting with erlfmt
+%% erlfmt:ignore
+-moduledoc """
+Provides a record, a behaviour, and functions to work with Apache Arrow Arrays.
+
+This module as mentioned, provides conveniences for working with Apache Arrow
+[Arrays](https://arrow.apache.org/docs/format/Glossary.html#term-array) of
+different
+[Layouts](https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout).
+Firstly, it provides a record to represent all the Layout. Secondly, it provides
+a behaviour for different layouts to adhere to common functionality. Lastly, it
+provides functions to work with Arrays in a common manner.
+
+# The Structure of an Array
+
+`arrow`'s implementation of an Array has the following fields in its record
+definition, each corresponding to a field in the Arrow Columnar Format
+specification:
+
+| Field             | Type                        | Representing                                                                              |
+|-------------------|-----------------------------|-------------------------------------------------------------------------------------------|
+| `layout`          | `t:atom/0`                  | Layout                                                                                    |
+| `type`            | `t:arrow_type:arrow_type/0` | [Logical Type](https://arrow.apache.org/docs/format/Glossary.html#term-type)              |
+| `len`             | `t:pos_integer/0`           | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths)                |
+| `element_len`     | `t:pos_integer/0`           | Length of each element                                                                    |
+| `null_count`      | `t:non_neg_integer/0`       | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count)               |
+| `validity_bitmap` | `t:arrow_buffer:buffer/0`   | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps)    |
+| `offsets`         | `t:arrow_buffer:buffer/0`   | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) |
+| `data`            | `t:arrow_buffer:buffer/0`   | Value Buffer                                                                              |
+
+
+Certain fields are not required for certain layouts. For example, for the
+Fixed-Sized Primitive Layout, the offsets field is not required, in which case
+it is assigned as `undefined`. Similarly, the `validity_bitmap` is not required
+if there are no null values, in which case it is also assigned as `undefined`.
+
+# The Behaviour of an Array
+
+As of right now, a layout needs to implement the `c:from_erlang/2` callback.
+This is then used in the `from_erlang/3` function to create new arrays.
+
+# Functions for working with Arrays
+
+As of right now, only functions to access the various fields, to create
+new arrays, and to serialize arrays to arrow exist.
+""".
 -export([
     from_erlang/3,
     layout/1,
@@ -113,29 +80,33 @@
 
 -export_type([array/0, layout/0]).
 
+-doc "Represents an Array.".
 -type array() :: #array{}.
-%% Represents an Array.
 
+-doc "Represents the Layout of an Array.".
 -type layout() :: fixed_primitive | variable_binary | fixed_list | variable_list.
-%% Represents the Layout of an Array.
 
 %%%%%%%%%%%%%%%%%%%%
 %% Array Creation %%
 %%%%%%%%%%%%%%%%%%%%
 
+-doc """
+Creates a new array of a certain layout, given its value and options from its
+erlang representation.
+""".
 -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) ->
-    Array :: #array{}.
-%% Creates a new array of a certain layout, given its value and options from its
-%% erlang representation.
+    Array :: array().
 
-%% @doc A common way to create a new array, given its layout, value, and options.
-%% from its erlang representation.
+-doc """
+A common way to create a new array, given its layout, value, and options.
+from its erlang representation.
+""".
 -spec from_erlang(
     Layout :: layout(),
     Value :: [arrow_type:native_type()],
     Opts :: map() | arrow_type:arrow_type()
 ) ->
-    Array :: #array{}.
+    Array :: array().
 from_erlang(Layout, Value, Opts) ->
     case Layout of
         fixed_primitive -> arrow_fixed_primitive_array:from_erlang(Value, Opts);
@@ -148,77 +119,79 @@ from_erlang(Layout, Value, Opts) ->
 %% Array Data and Metadata Access %%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-%% @doc Returns the layout of an array.
--spec layout(Array :: #array{}) -> Layout :: layout().
+-doc "Returns the layout of an array.".
+-spec layout(Array :: array()) -> Layout :: layout().
 layout(Array) ->
     Array#array.layout.
 
-%% @doc Returns the type of an array.
--spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type().
+-doc "Returns the type of an array.".
+-spec type(Array :: array()) -> Type :: arrow_type:arrow_type().
 type(Array) ->
     Array#array.type.
 
-%% @doc Returns the length of an array.
--spec len(Array :: #array{}) -> Length :: pos_integer().
+-doc "Returns the length of an array.".
+-spec len(Array :: array()) -> Length :: pos_integer().
 len(Array) ->
     Array#array.len.
 
-%% @doc Returns the length of an array.
--spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined.
+-doc "Returns the length of an element in an array.".
+-spec element_len(Array :: array()) -> Length :: pos_integer() | undefined.
 element_len(Array) ->
     Array#array.element_len.
 
-%% @doc Returns the null count of an array.
--spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer().
+-doc "Returns the null count of an array.".
+-spec null_count(Array :: array()) -> NullCount :: non_neg_integer().
 null_count(Array) ->
     Array#array.null_count.
 
-%% @doc Returns the validity bitmap of an array.
--spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined.
+-doc "Returns the validity bitmap of an array.".
+-spec validity_bitmap(Array :: array()) -> ValidityBitmap :: arrow_buffer:buffer() | undefined.
 validity_bitmap(Array) ->
     Array#array.validity_bitmap.
 
-%% @doc Returns the offsets of an array.
--spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined.
+-doc "Returns the offsets of an array.".
+-spec offsets(Array :: array()) -> Offsets :: arrow_buffer:buffer() | undefined.
 offsets(Array) ->
     Array#array.offsets.
 
-%% @doc Returns the data of an array.
--spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined.
+-doc "Returns the data of an array.".
+-spec data(Array :: array()) -> Data :: arrow_buffer:buffer() | array() | undefined.
 data(Array) ->
     Array#array.data.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%
 %% Array Serialization %%
 %%%%%%%%%%%%%%%%%%%%%%%%%
 
-%% @doc Serializes an array into the Arrow binary form.
-%%
-%% Serializes the buffers of an Array and concatenates them in the following
-%% order:
-%%
-%% <ol>
-%%  <li>`validity'</li>
-%%  <li>`offsets'</li>
-%%  <li>`data'</li>
-%% </ol>
-%%
-%% In case an array doesn't have any of the following buffers, it is ommitted.
-%% (e.g. validity in arrays with a null count of 0, offsets in fixed primitive
-%% arrays). In the case of a nested array, `data' will be serialized form of
-%% nested array.
-%%
-%% Do note that this is just binary form that includes the buffers in an Array,
-%% and not IPC.
--spec to_arrow(Array :: #array{}) -> Arrow :: binary().
+%% TODO Check formatting with erlfmt
+%% erlfmt:ignore
+-doc """ 
+Serializes an array into the Arrow binary form.
+
+Serializes the buffers of an Array and concatenates them in the following
+order:
+
+1. `validity`
+2. `offsets`
+3. `data`
+
+In case an array doesn't have any of the following buffers, it is omitted.
+(e.g. validity in arrays with a null count of 0, offsets in fixed primitive
+arrays). In the case of a nested array, `data` will be serialized form of nested
+array.
+
+Do note that this is just binary form that includes the buffers in an Array,
+and not IPC.
+""".
+-spec to_arrow(Array :: array()) -> Arrow :: binary().
 to_arrow(Array) ->
     Validity = some(validity_bitmap(Array)),
     Offsets = some(offsets(Array)),
     Data = some(data(Array)),
 
     <<Validity/binary, Offsets/binary, Data/binary>>.
 
--spec some(Value :: #array{} | #buffer{} | undefined) -> Binary :: binary().
+-spec some(Value :: array() | arrow_buffer:buffer() | undefined) -> Binary :: binary().
 some(undefined) ->
     <<>>;
 some(Buffer) when is_record(Buffer, buffer) ->

src/arrow_array.hrl

@@ -23,7 +23,7 @@
     len :: pos_integer(),
     element_len :: pos_integer() | undefined,
     null_count :: non_neg_integer(),
-    validity_bitmap :: #buffer{} | undefined,
-    offsets :: #buffer{} | undefined,
-    data :: #buffer{} | #array{}
+    validity_bitmap :: arrow_buffer:buffer() | undefined,
+    offsets :: arrow_buffer:buffer() | undefined,
+    data :: arrow_buffer:buffer() | #array{}
 }).