From f18b966a96800b1f2e9e1066a89e60dcfce06cad Mon Sep 17 00:00:00 2001 From: iamamyth Date: Wed, 8 Jan 2025 16:16:51 -0800 Subject: [PATCH] build: auto-set GitHub runner make job count Dynamically determine an appropriate number of make jobs for the GitHub runner container, based upon the number of available CPU cores and RAM, using the rule that each job requires a dedicated core and 2.25GiB of RAM. --- .github/actions/set-make-job-count/action.yml | 22 +++++++++++++++++++ .github/workflows/build.yml | 21 +++++++++++------- .github/workflows/depends.yml | 3 ++- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 .github/actions/set-make-job-count/action.yml diff --git a/.github/actions/set-make-job-count/action.yml b/.github/actions/set-make-job-count/action.yml new file mode 100644 index 000000000..68779ef3f --- /dev/null +++ b/.github/actions/set-make-job-count/action.yml @@ -0,0 +1,22 @@ +name: 'set-make-job-count' +description: 'Set the MAKE_JOB_COUNT environment variable to a value suitable for the host runner' +runs: + using: "composite" + steps: + # Each job runner requires 2.25 GiB (i.e. 1024 * 9/4 MiB) memory and + # a dedicated logical CPU core + - name: set-jobs-macOS + if: runner.os == 'macOS' + run: | + echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV + shell: bash + - name: set-jobs-windows + if: runner.os == 'Windows' + run: | + echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV + shell: msys2 {0} + - name: set-jobs-linux + if: runner.os == 'Linux' + run: | + echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV + shell: bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d9794b01..4584cac23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,8 +13,7 @@ on: # The below variables reduce repetitions across similar targets env: REMOVE_BUNDLED_PACKAGES : sudo rm -rf /usr/local - BUILD_DEFAULT_LINUX: | - cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --parallel 4 + BUILD_DEFAULT_LINUX: 'cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build' APT_INSTALL_LINUX: 'apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler ccache git' APT_SET_CONF: | tee -a /etc/apt/apt.conf.d/80-custom << EOF @@ -42,6 +41,7 @@ jobs: path: /Users/runner/Library/Caches/ccache key: ccache-${{ runner.os }}-build-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build- + - uses: ./.github/actions/set-make-job-count - name: install dependencies run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq miniupnpc expat libunwind-headers protobuf@21 ccache @@ -49,7 +49,7 @@ jobs: - name: build run: | ${{env.CCACHE_SETTINGS}} - make -j3 + make -j${{env.MAKE_JOB_COUNT}} build-windows: name: 'Windows (MSYS2)' @@ -73,10 +73,11 @@ jobs: with: update: true install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git pkg-config + - uses: ./.github/actions/set-make-job-count - name: build run: | ${{env.CCACHE_SETTINGS}} - make release-static-win64 -j4 + make release-static-win64 -j${{env.MAKE_JOB_COUNT}} build-debian: name: 'Debian 10' @@ -97,8 +98,9 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + - uses: ./.github/actions/set-make-job-count - name: build - run: ${{env.BUILD_DEFAULT_LINUX}} + run: ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}} # See the OS labels and monitor deprecations here: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources @@ -125,6 +127,7 @@ jobs: path: ~/.ccache key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }} + - uses: ./.github/actions/set-make-job-count - name: remove bundled packages run: ${{env.REMOVE_BUNDLED_PACKAGES}} - name: set apt conf @@ -136,7 +139,7 @@ jobs: - name: build run: | ${{env.CCACHE_SETTINGS}} - ${{env.BUILD_DEFAULT_LINUX}} + ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}} libwallet-ubuntu: name: "Ubuntu 20.04 (libwallet)" @@ -152,6 +155,7 @@ jobs: path: ~/.ccache key: ccache-${{ runner.os }}-libwallet-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-libwallet- + - uses: ./.github/actions/set-make-job-count - name: remove bundled packages run: ${{env.REMOVE_BUNDLED_PACKAGES}} - name: set apt conf @@ -164,7 +168,7 @@ jobs: run: | ${{env.CCACHE_SETTINGS}} cmake . - make wallet_api -j4 + make wallet_api -j${{env.MAKE_JOB_COUNT}} test-ubuntu: name: "Ubuntu 20.04 (tests)" @@ -182,6 +186,7 @@ jobs: path: ~/.ccache key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest + - uses: ./.github/actions/set-make-job-count - name: remove bundled packages run: ${{env.REMOVE_BUNDLED_PACKAGES}} - name: set apt conf @@ -198,7 +203,7 @@ jobs: DNS_PUBLIC: tcp://9.9.9.9 run: | ${{env.CCACHE_SETTINGS}} - ${{env.BUILD_DEFAULT_LINUX}} + ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}} cmake --build build --target test # ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere. diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 77ff606f7..6ad2fa6c0 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -79,6 +79,7 @@ jobs: restore-keys: | depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} depends-${{ matrix.toolchain.host }}- + - uses: ./.github/actions/set-make-job-count - name: set apt conf run: ${{env.APT_SET_CONF}} - name: install dependencies @@ -91,7 +92,7 @@ jobs: - name: build run: | ${{env.CCACHE_SETTINGS}} - make depends target=${{ matrix.toolchain.host }} -j4 + make depends target=${{ matrix.toolchain.host }} -j${{env.MAKE_JOB_COUNT}} - uses: actions/upload-artifact@v4 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin' || matrix.toolchain.host == 'aarch64-apple-darwin' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} with: