Skip to content

Loss of precision when round-tripping with float64 #142

Description

@FGasper

Given a high enough integer value, apd seems to lose precision after some round-trips:

package main

import (
	"math/rand/v2"
	"testing"

	"github.com/cockroachdb/apd/v2"
)

func TestRoundTrip(t *testing.T) {
	// This number is exactly representable as a float64:
	f64 := float64(rand.Int64())

	dec := apd.New(0, 0).SetInt64(int64(f64))

	f64RT, err := dec.Float64()
	if err != nil {
		t.Fatalf("round-trip f64->dec->f64: %v", err)
	}

	if f64 != f64RT {
		t.Fatalf("mismatched: expect %f; got %f", f64, f64RT)
	}

	dec2, err := apd.New(0, 0).SetFloat64(f64)
	if err != nil {
		t.Fatalf("round-trip f64->dec->f64->dec: %v", err)
	}

	if dec.Cmp(dec2) != 0 {
		t.Fatalf("mismatched: expect %v; got %v", dec, dec2)
	}
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions