@@ -694,12 +694,6 @@ def test_hardtanh(self):
694694 with self .assertRaisesRegex (AssertionError , "floating-point" ):
695695 self ._check (Lambda (F .relu6 ), (torch .arange (6 ).reshape (2 , 3 ) - 2 ,))
696696
697- def test_relu6_of_half_inputs (self ):
698- x = torch .randn (2 , 3 , 4 ) * 8.0
699- for dtype in (torch .float16 , torch .bfloat16 ):
700- with self .subTest (dtype = dtype ):
701- self ._check (Lambda (F .relu6 ), (x .to (dtype ),), atol = 0 , rtol = 0 )
702-
703697 def test_elu (self ):
704698 self ._check (Lambda (F .elu ), (torch .randn (2 , 3 , 4 ) * 3.0 ,))
705699 # Eager's expm1 is within rtol of these; exp(x) - 1 is 19% off at -1e-7.
@@ -1091,20 +1085,12 @@ def test_layer_norm_without_affine(self):
10911085 torch .nn .LayerNorm (16 , elementwise_affine = False ), (torch .randn (2 , 3 , 16 ),)
10921086 )
10931087
1094- def test_layer_norm_half (self ):
1095- norm = torch .nn .LayerNorm (16 )
1096- torch .nn .init .uniform_ (norm .weight , 0.5 , 1.5 )
1097- torch .nn .init .uniform_ (norm .bias , - 0.5 , 0.5 )
1098- x = torch .randn (2 , 3 , 16 )
1099- for dtype in (torch .float16 , torch .bfloat16 ):
1100- with self .subTest (dtype = dtype ):
1101- # One rounding of the output; eager normalizes in F32 too.
1102- self ._check (
1103- copy .deepcopy (norm ).to (dtype ),
1104- (x .to (dtype ),),
1105- atol = 1e-2 if dtype == torch .bfloat16 else 2e-3 ,
1106- rtol = 1e-2 if dtype == torch .bfloat16 else 2e-3 ,
1107- )
1088+ def test_layer_norm_rejects_half (self ):
1089+ with self .assertRaisesRegex (AssertionError , "F32 tensors only" ):
1090+ self ._check (
1091+ torch .nn .LayerNorm (16 ).half (),
1092+ (torch .randn (2 , 3 , 16 , dtype = torch .float16 ),),
1093+ )
11081094
11091095 def test_layer_norm_rejects_used_mean (self ):
11101096 def forward (x ):
@@ -1981,34 +1967,6 @@ def test_reductions_and_sqrt_of_half_linears(self):
19811967 rtol = 1e-2 ,
19821968 )
19831969
1984- def test_layer_norm_and_hardtanh_of_half_intermediates (self ):
1985- # A half linear or embedding result is rounded to its dtype, so
1986- # layer_norm, which computes in F32 only, and hardtanh, which ggml-cpu
1987- # clamps in F32 and F16 only, must widen it again.
1988- torch .manual_seed (0 )
1989- norm = torch .nn .LayerNorm (64 )
1990- torch .nn .init .uniform_ (norm .weight , 0.5 , 1.5 )
1991- torch .nn .init .uniform_ (norm .bias , - 0.5 , 0.5 )
1992- linear = torch .nn .Linear (16 , 64 )
1993- x = torch .randn (4 , 16 )
1994- cases = {
1995- "layer_norm of a linear" : (linear , norm , x ),
1996- "layer_norm of an embedding" : (
1997- torch .nn .Embedding (10 , 64 ),
1998- norm ,
1999- torch .tensor ([[1 , 7 , 3 ]]),
2000- ),
2001- "hardtanh of a linear" : (linear , torch .nn .Hardtanh (- 0.5 , 0.5 ), x ),
2002- "relu6 of a linear" : (linear , torch .nn .ReLU6 (), x * 8 ),
2003- }
2004- for name , (first , second , x ) in cases .items ():
2005- for dtype in (torch .float16 , torch .bfloat16 ):
2006- with self .subTest (case = name , dtype = dtype ):
2007- model = torch .nn .Sequential (first , second )
2008- half = copy .deepcopy (model ).to (dtype ).requires_grad_ (False )
2009- inputs = (x .to (dtype ) if x .is_floating_point () else x ,)
2010- self ._check (half , inputs , atol = 1e-2 , rtol = 1e-2 )
2011-
20121970 def test_rope_llama3 (self ):
20131971 config = LlamaConfig (
20141972 hidden_size = 256 ,
0 commit comments