flagos-ai · xinhaoH · Mar 19, 2026
@@ -0,0 +1,30 @@
+<div align="center">
+<h1>SoLA</h1>
+<h3>
+[AAAI 2025] SoLA: Leveraging Soft Activation Sparsity and Low-Rank Decomposition for Large Language Model Compression
+<h3>
+</div>
+
+<p align="center">
+<img width="100%" alt="image" src="data/overview.jpg">    
+</p>
+
+## Usage
+
+### Construct Calibration Data
+    sola_neuron_idx/utils.py get_wikitext2() --> data/wiki_256_4096.json
+
+### Compute Neurons Norm
+    bash sola_neuron_idx/playground.sh --> data/hitter_dict_15_256_4096_wiki_13b.json
+
+### Low-Rank Decomposition and Evaluation
+    bash sola/playground_sola.sh
+
+    Llama-2-7B/13B low-rank decomposition and evaluation
+
+    Remember modify file path to your own 
+
+### Others
+    sola_neuron_idx/hook.py llama_self_attn() past_key_value.update_get() in transformers-4.37.1/src/transformers/cache_utils.py
+
+    You can install modified transformers package: cd transformers-4.37.1; pip install -e .
@@ -0,0 +1,355 @@
+about-time==4.2.1
+absl-py==2.1.0
+accelerate==1.6.0
+addict==2.4.0
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+alive-progress==3.2.0
+altair==5.5.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+appdirs==1.4.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.2.0
+auto_gptq==0.7.1
+autograd==1.7.0
+babel==2.17.0
+beautifulsoup4==4.13.3
+bitsandbytes==0.43.2.dev0
+bleach==6.2.0
+boto3==1.35.90
+botocore==1.35.90
+certifi==2024.2.2
+cffi==1.17.1
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==2.2.1
+cma==3.2.2
+colorama==0.4.6
+comm==0.2.2
+ConfigSpace==0.6.0
+contextlib2==21.6.0
+contourpy==1.2.1
+cpm-kernels==1.0.11
+cycler==0.12.1
+Cython==3.0.10
+dataclasses-json==0.6.7
+DataProperty==1.0.1
+datasets==3.2.0
+debugpy==1.8.12
+decorator==5.2.1
+deepeval==2.0
+defusedxml==0.7.1
+Deprecated==1.2.15
+dill==0.3.8
+distro==1.9.0
+dnspython==2.6.1
+docker==7.1.0
+docker-pycreds==0.4.0
+docstring_parser==0.16
+docx2txt==0.8
+einops==0.8.0
+email_validator==2.2.0
+emcee==3.1.6
+et_xmlfile==2.0.0
+eval_type_backport==0.2.2
+evaluate==0.4.6
+exceptiongroup==1.2.1
+execnet==2.1.1
+executing==2.2.0
+fastapi==0.111.1
+fastapi-cli==0.0.4
+fastjsonschema==2.21.1
+ffmpy==0.3.2
+filelock==3.14.0
+fire==0.7.1
+flash-attn==2.5.8
+fonttools==4.51.0
+fqdn==1.5.1
+frozenlist==1.4.1
+fschat==0.2.24
+fsspec==2024.3.1
+func-timeout==4.3.5
+fuzzywuzzy==0.18.0
+gekko==1.3.0
+gitdb==4.0.11
+GitPython==3.1.43
+google-pasta==0.2.0
+googleapis-common-protos==1.66.0
+gradio==3.50.2
+gradio_client==0.6.1
+grapheme==0.6.0
+greenlet==3.1.1
+grpcio==1.63.2
+h11==0.14.0
+h5py==3.13.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.2
+httpx-sse==0.4.0
+huggingface-hub==0.30.1
+-e git+https://github.com/openai/human-eval@6d43fb980f9fee3c892a914eda09951f772ad10d#egg=human_eval
+idna==3.7
+immutabledict==4.2.1
+importlib-metadata==6.11.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+inquirerpy==0.3.4
+ipykernel==6.29.5
+ipython==8.18.1
+isoduration==20.11.0
+jedi==0.19.2
+jieba==0.42.1
+Jinja2==3.1.3
+jiter==0.8.0
+jmespath==1.0.1
+joblib==1.4.2
+json5==0.10.0
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.3.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+kiwisolver==1.4.5
+langchain==0.3.9
+langchain-community==0.3.8
+langchain-core==0.3.21
+langchain-openai==0.2.10
+langchain-text-splitters==0.3.2
+langsmith==0.1.147
+latex2mathml==3.77.0
+Levenshtein==0.27.1
+lion-pytorch==0.1.4
+llm-blender @ git+https://github.com/yuchenlin/LLM-Blender.git@33204d2712944b6b17996f7c079e74cd963ccc7c
+-e git+https://github.com/EleutherAI/lm-evaluation-harness@9d4a04a0aac37fff88ca1a04d667012e1502c21d#egg=lm_eval
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown2==2.5.3
+MarkupSafe==2.1.5
+marshmallow==3.23.1
+matplotlib==3.8.4
+matplotlib-inline==0.1.7
+mbstrdecoder==1.1.3
+mdurl==0.1.2
+mistune==3.1.2
+ml-collections==0.1.1
+mmengine-lite==0.10.7
+mock==4.0.3
+more-itertools==10.3.0
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+narwhals==1.34.1
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.2.1
+nh3==0.2.21
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==7.3.2
+notebook_shim==0.2.4
+numexpr==2.10.1
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+omegaconf==2.2.3
+openai==1.55.3
+openbox==0.8.3
+OpenCC==1.1.9
+-e git+ssh://[email protected]/xinhaoH/KV_C.git@c899b84206943dbded60678a15425ce34b4bcb38#egg=opencompass&subdirectory=opencompass
+opencv-python-headless==4.11.0.86
+openpyxl==3.1.5
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
+optimum==2.0.0
+orjson==3.10.6
+overrides==7.7.0
+packaging==24.0
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.4
+pathos==0.3.3
+pathvalidate==3.2.0
+patsy==0.5.6
+# Editable install with no version control (peft==0.10.0)
+-e /home/xinhao/peft-0.10.0
+pexpect==4.9.0
+pfzy==0.3.4
+pillow==10.3.0
+platformdirs==4.2.2
+Platypus-Opt==1.0.4
+pluggy==1.5.0
+portalocker==2.10.1
+pox==0.3.5
+ppft==1.7.6.9
+prettytable==3.10.2
+prometheus_client==0.21.1
+prompt_toolkit==3.0.50
+protobuf==4.25.5
+psutil==5.9.8
+ptflops==0.7.3
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyaml==24.7.0
+pyarrow==16.0.0
+pyarrow-hotfix==0.6
+pybind11==2.13.1
+pycountry==24.6.1
+pycparser==2.22
+pydantic==1.10.21
+pydantic-settings==2.6.1
+pydantic_core==2.20.1
+pydub==0.25.1
+pyext==0.7
+Pygments==2.18.0
+pymoo==0.6.0
+pyparsing==3.1.2
+pysbd==0.3.4
+pytablewriter==1.2.0
+pytest==8.2.0
+pytest-repeat==0.9.3
+pytest-xdist==3.6.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-json-logger==3.2.1
+python-Levenshtein==0.27.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==26.2.1
+ragas==0.2.6
+rank-bm25==0.2.2
+RapidFuzz==3.12.2
+referencing==0.35.1
+regex==2024.4.28
+requests==2.32.3
+requests-toolbelt==1.0.0
+retrying==1.3.4
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rouge==1.0.1
+rouge-chinese==1.0.3
+rouge-score==0.1.2
+rpds-py==0.22.3
+ruff==0.5.5
+s3fs==0.4.2
+s3transfer==0.10.4
+sacrebleu==2.4.2
+safetensors==0.4.3
+sagemaker==2.237.1
+sagemaker-core==1.0.17
+schema==0.7.7
+scikit-learn==1.5.0
+scikit-optimize==0.10.2
+scipy==1.10.1
+seaborn==0.13.2
+semantic-version==2.10.0
+Send2Trash==1.8.3
+sentence-transformers==4.0.1
+sentencepiece==0.2.0
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+shellingham==1.5.4
+shortuuid==1.0.13
+shtab==1.7.1
+six==1.16.0
+smdebug-rulesconfig==1.0.1
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soupsieve==2.6
+SQLAlchemy==2.0.35
+sqlitedict==2.1.0
+stack-data==0.6.3
+starlette==0.37.2
+statsmodels==0.14.2
+svgwrite==1.4.3
+sympy==1.12
+syne_tune==0.13.0
+tabledata==1.3.3
+tabulate==0.9.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.4.2
+termcolor==3.0.1
+terminado==0.18.1
+threadpoolctl==3.5.0
+tiktoken==0.8.0
+timeout-decorator==0.5.0
+tinycss2==1.4.0
+tokenizers==0.15.2
+tomli==2.0.1
+tomlkit==0.12.0
+torch==2.3.0
+tornado==6.4.2
+tqdm==4.66.4
+tqdm-multiprocess==0.0.11
+traitlets==5.14.3
+-e git+ssh://[email protected]/xinhaoH/SOLA_xinhao.git@0a46232d27a29d73794599c3530fb07157c60b0c#egg=transformers&subdirectory=transformers-4.37.1
+tree-sitter==0.21.3
+tree-sitter-languages==1.10.2
+triton==2.3.0
+trl @ git+https://github.com/huggingface/trl.git@eab175d434b9bb9badee20335c7945991a26dfac
+typeguard==4.4.2
+typepy==1.3.2
+typer==0.12.3
+types-python-dateutil==2.9.0.20241206
+typing-inspect==0.9.0
+typing_extensions==4.13.2
+tyro==0.9.18
+tzdata==2024.1
+ujson==5.10.0
+uri-template==1.3.0
+urllib3==1.26.20
+uvicorn==0.30.3
+uvloop==0.19.0
+wandb==0.17.5
+watchfiles==0.22.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+websockets==11.0.3
+word2number==1.1
+wrapt==1.17.0
+xxhash==3.4.1
+yapf==0.43.0
+yarl==1.9.4
+zipp==3.18.1
+zstandard==0.23.0