You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Copy file name to clipboardExpand all lines: _bibliography/references.bib
+21
Original file line number
Diff line number
Diff line change
@@ -1,5 +1,26 @@
1
1
---
2
2
---
3
+
4
+
@inproceedings{
5
+
jamadandi2024spectral,
6
+
title={Spectral Graph Pruning Against Over-Squashing and Over-Smoothing},
7
+
author={Adarsh Jamadandi and Celia Rubio-Madrigal and Rebekka Burkholz},
8
+
booktitle={Thirty-eigth Conference on Neural Information Processing Systems},
9
+
year={2024},
10
+
url={https://openreview.net/forum?id=EMkrwJY2de},
11
+
pdf={https://openreview.net/pdf?id=EMkrwJY2de},
12
+
abstract={Message Passing Graph Neural Networks are known to suffer from two problems that are sometimes believed to be diametrically opposed: over-squashing and over-smoothing. The former results from topological bottlenecks that hamper the information flow from distant nodes and are mitigated by spectral gap maximization, primarily, by means of edge additions. However, such additions often promote over-smoothing that renders nodes of different classes less distinguishable. Inspired by the Braess phenomenon, we argue that deleting edges can address over-squashing and over-smoothing simultaneously. This insight explains how edge deletions can improve generalization, thus connecting spectral gap optimization to a seemingly disconnected objective of reducing computational resources by pruning graphs for lottery tickets. To this end, we propose a computationally effective spectral gap optimization framework to add or delete edges and demonstrate its effectiveness on the long range graph benchmark and on larger heterophilous datasets.},
13
+
}
14
+
15
+
@inproceedings{
16
+
mustafa2024training,
17
+
title={Training GNNs in Balance by Dynamic Rescaling},
18
+
author={Nimrah Mustafa and Rebekka Burkholz},
19
+
booktitle={Thirty-eigth Conference on Neural Information Processing Systems},
20
+
year={2024},
21
+
abstract={Graph neural networks exhibiting a rescale invariance, like GATs, obey a conservation law of its parameters, which has been exploited to derive a balanced state that induces good initial trainability. Yet, finite learning rates as used in practice topple the network out of balance during training. This effect is even more pronounced with larger learning rates that tend to induce improved generalization but make the training dynamics less robust. To support even larger learning rates, we propose to dynamically balance the network according to a different criterion, based on relative gradients, that promotes faster and better. In combination with large learning rates and gradient clipping, dynamic rebalancing significantly improves generalization on real-world data. We observe that rescaling provides us with the flexibility to control the order in which network layers are trained. This leads to novel insights into similar phenomena as grokking, which can further boost generalization performance.},
22
+
}
23
+
3
24
@inproceedings{
4
25
mustafa2024gate,
5
26
title={{GATE}: How to Keep Out Intrusive Neighbors},
<divclass="title"><ahref="https://openreview.net/forum?id=EMkrwJY2de"><b>Spectral Graph Pruning Against Over-Squashing and Over-Smoothing</b></a></div>
<p>Message Passing Graph Neural Networks are known to suffer from two problems that are sometimes believed to be diametrically opposed: over-squashing and over-smoothing. The former results from topological bottlenecks that hamper the information flow from distant nodes and are mitigated by spectral gap maximization, primarily, by means of edge additions. However, such additions often promote over-smoothing that renders nodes of different classes less distinguishable. Inspired by the Braess phenomenon, we argue that deleting edges can address over-squashing and over-smoothing simultaneously. This insight explains how edge deletions can improve generalization, thus connecting spectral gap optimization to a seemingly disconnected objective of reducing computational resources by pruning graphs for lottery tickets. To this end, we propose a computationally effective spectral gap optimization framework to add or delete edges and demonstrate its effectiveness on the long range graph benchmark and on larger heterophilous datasets.</p>
<spanclass="na">title</span><spanclass="p">=</span><spanclass="s">{Spectral Graph Pruning Against Over-Squashing and Over-Smoothing}</span><spanclass="p">,</span>
126
+
<spanclass="na">author</span><spanclass="p">=</span><spanclass="s">{Jamadandi, Adarsh and Rubio-Madrigal, Celia and Burkholz, Rebekka}</span><spanclass="p">,</span>
127
+
<spanclass="na">booktitle</span><spanclass="p">=</span><spanclass="s">{Thirty-eigth Conference on Neural Information Processing Systems}</span><spanclass="p">,</span>
<p>Graph neural networks exhibiting a rescale invariance, like GATs, obey a conservation law of its parameters, which has been exploited to derive a balanced state that induces good initial trainability. Yet, finite learning rates as used in practice topple the network out of balance during training. This effect is even more pronounced with larger learning rates that tend to induce improved generalization but make the training dynamics less robust. To support even larger learning rates, we propose to dynamically balance the network according to a different criterion, based on relative gradients, that promotes faster and better. In combination with large learning rates and gradient clipping, dynamic rebalancing significantly improves generalization on real-world data. We observe that rescaling provides us with the flexibility to control the order in which network layers are trained. This leads to novel insights into similar phenomena as grokking, which can further boost generalization performance.</p>
<spanclass="na">title</span><spanclass="p">=</span><spanclass="s">{Training GNNs in Balance by Dynamic Rescaling}</span><spanclass="p">,</span>
174
+
<spanclass="na">author</span><spanclass="p">=</span><spanclass="s">{Mustafa, Nimrah and Burkholz, Rebekka}</span><spanclass="p">,</span>
175
+
<spanclass="na">booktitle</span><spanclass="p">=</span><spanclass="s">{Thirty-eigth Conference on Neural Information Processing Systems}</span><spanclass="p">,</span>
0 commit comments