Merge branch 'master' into master

.github/workflows/build-and-upload.yml

@@ -29,8 +29,8 @@ jobs:
     needs: pick-devN
     strategy:
       matrix:
-        python_version: ['3.8', '3.9', '3.10', '3.11']
-        plat: ['manylinux2014', 'manylinux_2_28', 'macos-latest', 'windows-latest']
+        python_version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        plat: ['manylinux2014', 'manylinux_2_28', 'macos-13', 'macos-latest', 'windows-latest']
         include:
           - plat: manylinux2014
             os: ubuntu-latest
@@ -40,6 +40,9 @@ jobs:
             os: ubuntu-latest
             container: quay.io/pypa/manylinux_2_28_x86_64  # https://github.com/pypa/manylinux
 
+          - plat: macos-13
+            os: macos-13
+
           - plat: macos-latest
             os: macos-latest
 
@@ -72,7 +75,12 @@ jobs:
         if: github.event_name != 'release' && matrix.os == 'windows-latest'
         run: ("DEV_BUILD=" + (get-content devN.txt)) >> $env:GITHUB_ENV  # for setup.py
 
+      # downgraded to @v3 if using container: https://github.com/actions/checkout/issues/1487
       - uses: actions/checkout@v3
+        if: matrix.container != ''
+
+      - uses: actions/checkout@v4
+        if: matrix.container == ''
 
       - name: Mark workspace safe for git
         # needed for container and self-hosted runners; see https://github.com/actions/checkout/issues/766
@@ -83,9 +91,18 @@ jobs:
           # leading to missing files in the archive.  Run it separately to force a failure in that case.
           (cd scalene; git archive --prefix scalene/ HEAD | tar -t > /dev/null)
   
+      - name: select Xcode version
+        # MacOS > 14.2 requires Xcode >= 15.3; otherwise loading native extension modules fails with e.g.:
+        # dlopen(/opt/homebrew/lib/python3.11/site-packages/slipcover/probe.abi3.so, 0x0002): bad bind opcode 0x00 
+        if: startsWith(matrix.os, 'macos-')
+        run: |
+          if [ -d /Applications/Xcode_15.3.app/Contents/Developer ]; then sudo xcode-select --switch /Applications/Xcode_15.3.app/Contents/Developer; fi
+          clang++ --version
+          g++ --version
+
       - name: Set up python (script version)
         if: matrix.container == ''
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python_version }}
 
@@ -109,7 +126,10 @@ jobs:
         run: make sdist
 
       - name: Build binary dist
-        run: make bdist
+        # Invoking with "bash -c" works around the "bash.exe: ... could not find /tmp" issue on Windows.
+        # The issue is somehow related to git providing a sh.exe (the "git shell")
+        # See eg https://help.appveyor.com/discussions/problems/1531-having-issues-with-configured-git-bash
+        run: bash -c "make bdist"
 
       - name: Check that all required platforms are included
         if: matrix.os == 'macos-latest'

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/test-smoketests.yml

@@ -10,17 +10,17 @@ on:
 jobs:
   smoketests:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 15  # on macos-latest, pypy-3.7 it builds numpy and others
+    timeout-minutes: 15
     strategy:
       matrix:
-        os: [ ubuntu-latest, macos-latest ] # disabling windows-latest for now
-        python: [ '3.9', '3.10', '3.11' ] # temporarily disabling pypy-3.7 (eventually add 3.12)
+        os: [ ubuntu-latest, macos-13 ] # disabling windows-latest for now
+        python: [ '3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/tests.yml

@@ -0,0 +1,64 @@
+name: tests
+
+on:
+  push:
+    branches: [ master ]
+
+  pull_request:
+    branches: [ master ]
+
+  workflow_dispatch:
+
+
+jobs:
+  run-tests:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-13 ]
+        python: [ '3.9', '3.10', '3.11', '3.12' ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: select Xcode version
+      # MacOS > 14.2 requires Xcode >= 15.3; otherwise loading native extension modules fails with e.g.:
+      # dlopen(/opt/homebrew/lib/python3.11/site-packages/slipcover/probe.abi3.so, 0x0002): bad bind opcode 0x00 
+      if: startsWith(matrix.os, 'macos-')
+      run: |
+        if [ -d /Applications/Xcode_15.3.app/Contents/Developer ]; then sudo xcode-select --switch /Applications/Xcode_15.3.app/Contents/Developer; fi
+        clang++ --version
+        g++ --version
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Work around arm64 support on MacOS
+      # https://github.com/actions/virtual-environments/issues/2557
+      if: matrix.os == 'macos-latest'
+      run: sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r requirements.txt
+        python -m pip install numpy
+
+    - name: Build scalene
+      run: pip -v install -e .
+
+    - name: install test dependencies
+      run: |
+        python3 -m pip install pytest pytest-asyncio hypothesis
+        python3 -m pip install .
+
+    - name: run tests
+      run: |
+        python3 -m pytest

.gitignore

@@ -149,3 +149,7 @@ scalene/libscalene.dylib
 scalene/libscalene.dylib.dSYM/Contents/Info.plist
 scalene/libscalene.dylib.dSYM/Contents/Resources/DWARF/libscalene.dylib
 
+# Vendor is regenerated by `make`
+vendor/
+!vendor/README.md
+

GNUmakefile

@@ -7,13 +7,14 @@ C_SOURCES = src/source/*.cpp src/include/*.h*
 .PHONY: black clang-format prettier format upload vendor-deps
 
 # CXXFLAGS = -std=c++14 -g -O0 # FIXME
-CXXFLAGS = -std=c++14 -Wall -g -O3 -DNDEBUG -D_REENTRANT=1 -DHL_USE_XXREALLOC=1 -pipe -fno-builtin-malloc -fvisibility=hidden
+CXXFLAGS = -std=c++14 -Wall -g -O3 -DNDEBUG -D_REENTRANT=1 -DHL_USE_XXREALLOC=1 -pipe -fno-builtin-malloc -fvisibility=hidden -Wno-unused-result
 # CXX = g++
 
 INCLUDES  = -Isrc -Isrc/include
 INCLUDES := $(INCLUDES) -Ivendor/Heap-Layers -Ivendor/Heap-Layers/wrappers -Ivendor/Heap-Layers/utility
 INCLUDES := $(INCLUDES) -Ivendor/printf
-INCLUDES := $(INCLUDES) $(shell python3-config --includes)
+# python3-config may not be available in venv and such
+INCLUDES := $(INCLUDES) -I$(shell python3 -c "import sysconfig; print(sysconfig.get_path('include'))")
 
 ifeq ($(shell uname -s),Darwin)
   LIBFILE := lib$(LIBNAME).dylib
@@ -54,26 +55,20 @@ clean:
 $(WRAPPER) : vendor/Heap-Layers
 
 vendor/Heap-Layers:
-	mkdir -p vendor && cd vendor && git clone https://github.com/emeryberger/Heap-Layers
+	cd vendor && git clone https://github.com/emeryberger/Heap-Layers
 
 TMP := $(shell mktemp -d || echo /tmp)
 
 vendor/printf/printf.cpp:
-	mkdir -p vendor && cd vendor && git clone https://github.com/mpaland/printf
+	cd vendor && git clone https://github.com/mpaland/printf
 	cd vendor/printf && ln -s printf.c printf.cpp
 	sed -e 's/^#define printf printf_/\/\/&/' vendor/printf/printf.h > $(TMP)/printf.h.$$ && mv $(TMP)/printf.h.$$ vendor/printf/printf.h
 	sed -e 's/^#define vsnprintf vsnprintf_/\/\/&/' vendor/printf/printf.h > $(TMP)/printf.h.$$ && mv $(TMP)/printf.h.$$ vendor/printf/printf.h
 
-clear-vendor-dirs:
-	rm -fr vendor/
-
-vendor/crdp:
-	mkdir -p vendor && cd vendor && git clone https://github.com/plasma-umass/crdp
-
-vendor-deps: clear-vendor-dirs vendor/Heap-Layers vendor/printf/printf.cpp vendor/crdp
+vendor-deps: vendor/Heap-Layers vendor/printf/printf.cpp
 
 mypy:
-	-mypy $(PYTHON_SOURCES)
+	-mypy --no-warn-unused-ignores $(PYTHON_SOURCES)
 
 format: black clang-format prettier
 

MANIFEST.in

@@ -2,6 +2,4 @@ graft vendor/Heap-Layers
 prune vendor/Heap-Layers/.git
 graft vendor/printf
 prune vendor/printf/.git
-graft vendor/crdp
-prune vendor/crdp/.git
 exclude scalene/old/*

README.md

@@ -14,6 +14,8 @@ by [Emery Berger](https://emeryberger.com), [Sam Stern](https://samstern.me/), a
 
 ![Semantic Scholar success story](https://github.com/plasma-umass/scalene/raw/master/docs/semantic-scholar-success.png)
 
+***Scalene web-based user interface:*** [http://plasma-umass.org/scalene-gui/](http://plasma-umass.org/scalene-gui/)
+
 ## About Scalene
 
 Scalene is a high-performance CPU, GPU *and* memory profiler for
@@ -104,6 +106,8 @@ from scalene import scalene_profiler
 # Turn profiling on
 scalene_profiler.start()
 
+# your code
+
 # Turn profiling off
 scalene_profiler.stop()
 ```
@@ -382,17 +386,6 @@ Scalene is distributed as a `conda` package and works on Mac OS X, Linux (includ
 > 
 </details>
 
-<details>
-<summary>Using <code>Homebrew</code> (Mac OS X)</summary>
-
-As an alternative to `pip`, you can use Homebrew to install the current version of Scalene from this repository:
-
-```console
-  % brew tap plasma-umass/scalene
-  % brew install --head plasma-umass/scalene/scalene
-```
-</details>
-
 <details>
 <summary>On ArchLinux</summary>
 
@@ -402,7 +395,7 @@ manually download the `PKGBUILD` and run `makepkg -cirs` to build. Note that thi
 `libscalene.so` in `/usr/lib`; modify the below usage instructions accordingly.
 </details>
 
-# Asked Questions
+# Frequently Asked Questions
 
 <details>
 <summary>
@@ -425,7 +418,7 @@ Is there any way to get shorter profiles or do more targeted profiling?
 1. Use `--reduced-profile` to include only lines and files with memory/CPU/GPU activity.
 2. Use `--profile-only` to include only filenames containing specific strings (as in, `--profile-only foo,bar,baz`).
 3. Decorate functions of interest with `@profile` to have Scalene report _only_ those functions.
-4. Turn profiling on and off programmatically by importing Scalene (`import scalene`) and then turning profiling on and off via `scalene_profiler.start()` and `scalene_profiler.stop()`. By default, Scalene runs with profiling on, so to delay profiling until desired, use the `--off` command-line option (`python3 -m scalene --off yourprogram.py`).
+4. Turn profiling on and off programmatically by importing Scalene profiler (`from scalene import scalene_profiler`) and then turning profiling on and off via `scalene_profiler.start()` and `scalene_profiler.stop()`. By default, Scalene runs with profiling on, so to delay profiling until desired, use the `--off` command-line option (`python3 -m scalene --off yourprogram.py`).
 </details>
 
 <details>
@@ -444,6 +437,22 @@ How do I use Scalene with Django?
 **A:** Pass in the `--noreload` option (see https://github.com/plasma-umass/scalene/issues/178).
 </details>
 
+
+<details>
+<summary>
+Does Scalene work with gevent/Greenlets?
+</summary>
+
+**A:** Yes! Put the following code in the beginning of your program, or modify the call to `monkey.patch_all` as below:
+
+```python
+from gevent import monkey
+monkey.patch_all(thread=False)
+```
+</details>
+
+
+
 <details>
 <summary>
 How do I use Scalene with PyTorch on the Mac?
@@ -456,10 +465,33 @@ How do I use Scalene with PyTorch on the Mac?
 
 For details about how Scalene works, please see the following paper, which won the Jay Lepreau Best Paper Award at [OSDI 2023](https://www.usenix.org/conference/osdi23/presentation/berger): [Triangulating Python Performance Issues with Scalene](https://arxiv.org/pdf/2212.07597). (Note that this paper does not include information about the AI-driven proposed optimizations.)
 
+<details>
+<summary>
+To cite Scalene in an academic paper, please use the following:
+</summary>
+
+```latex
+@inproceedings{288540,
+author = {Emery D. Berger and Sam Stern and Juan Altmayer Pizzorno},
+title = {Triangulating Python Performance Issues with {S}calene},
+booktitle = {{17th USENIX Symposium on Operating Systems Design and Implementation (OSDI 23)}},
+year = {2023},
+isbn = {978-1-939133-34-2},
+address = {Boston, MA},
+pages = {51--64},
+url = {https://www.usenix.org/conference/osdi23/presentation/berger},
+publisher = {USENIX Association},
+month = jul
+}
+```
+</details>
+
+
 # Success Stories
 
 If you use Scalene to successfully debug a performance problem, please [add a comment to this issue](https://github.com/plasma-umass/scalene/issues/58)!
 
+
 # Acknowledgements
 
 Logo created by [Sophia Berger](https://www.linkedin.com/in/sophia-berger/).