I develop Popcorn which compiles OTP/BEAM for browsers. The VM is short-lived for some uses and having it boot fast is important to me.
I recently benchmarked few things in that area and found out that checksumming process could improve, changing from 19.5 ms to 0.3 ms time spent calculating checksums. I used XXH3-128 but any modern non-cryptographic hash function with SIMD support should work.
Maybe it’s a good discussion to have? I did not find anything on past github issues and mailing list archives.
Measurement
The benchmark uses release builds of OTP 29.0.5 on Linux/aarch64. Each VM uses one scheduler, loads 168 stdlib modules, and exits. The results contain 10 runs after three warmups.
| Algorithm | Checksum time | Boot time | Checksum share |
|---|---|---|---|
| MD5 | 19.563 ms ±0.76 ms | 175.092 ms ±25.84 ms | 11.17% |
| XXH3-128 | 0.327 ms ±0.13 ms | 161.853 ms ±37.05 ms | 0.20% |
| Delta | -19.24 ms | -13.24 ms |
“Boot time” covers spawning VM process and waiting for it to exit — measurements are quite noisy. “Checksum time” includes timings around checksum calculation and are more precise.
I observed 13 ms speedup on my machine (macOS M3, benchmarks ran in docker) but would love to see it replicated by other people.
The benchmark patch replaces the loader checksum globally to measure upper bound of time savings.
Benchmarking setup (LLM generated, looks ~reasonable to me)
```bashmkdir -p 'report'
# report/Dockerfile
cat > 'report/Dockerfile' <<'EOF'
FROM debian:bookworm-slim AS build-base
RUN apt-get update && apt-get install -y --no-install-recommends \
autoconf \
build-essential \
ca-certificates \
git \
libncurses-dev \
libxxhash-dev \
m4 \
patch \
perl \
&& rm -rf /var/lib/apt/lists/*
ARG OTP_TAG=OTP-29.0.5
RUN git clone --branch "${OTP_TAG}" --depth 1 https://github.com/erlang/otp.git /src
COPY module-hash.patch /tmp/module-hash.patch
WORKDIR /src
RUN patch -p1 -i /tmp/module-hash.patch
FROM build-base AS md5-build
RUN CFLAGS="-O3 -DBEAM_MODULE_HASH_PROFILE" \
./configure \
--prefix=/opt/otp-md5 \
--disable-debug \
--without-debugger \
--without-et \
--without-javac \
--without-odbc \
--without-observer \
--without-ssl \
--without-wx
RUN make -j"$(nproc)" && make install
FROM build-base AS xxh3-build
RUN CFLAGS="-O3 -DBEAM_MODULE_HASH_PROFILE -DBEAM_MODULE_HASH_XXH3" \
LIBS="-lxxhash" \
./configure \
--prefix=/opt/otp-xxh3 \
--disable-debug \
--without-debugger \
--without-et \
--without-javac \
--without-odbc \
--without-observer \
--without-ssl \
--without-wx
RUN make -j"$(nproc)" && make install
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
libncurses6 \
libstdc++6 \
libxxhash0 \
python3 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=md5-build /opt/otp-md5 /opt/otp-md5
COPY --from=xxh3-build /opt/otp-xxh3 /opt/otp-xxh3
COPY benchmark.py /usr/local/bin/module-hash-benchmark
ENTRYPOINT ["python3", "/usr/local/bin/module-hash-benchmark"]
EOF
# report/benchmark.py
cat > 'report/benchmark.py' <<'EOF'
#!/usr/bin/env python3
import argparse
import json
import re
import statistics
import subprocess
import time
parser = argparse.ArgumentParser()
parser.add_argument("--runs", type=int, default=10)
parser.add_argument("--warmups", type=int, default=3)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
assert args.runs > 1
variants = {
"md5": "/opt/otp-md5/bin/erl",
"xxh3-128": "/opt/otp-xxh3/bin/erl",
}
eval_code = '''Dir=code:lib_dir(stdlib,ebin), Files=filelib:wildcard(filename:join(Dir,"*.beam")), lists:foreach(fun(File) -> Root=filename:rootname(File), Mod=list_to_atom(filename:basename(Root)), case code:is_loaded(Mod) of false -> {module,Mod}=code:load_abs(Root); _ -> ok end end, Files), halt().'''
def command(erl):
return [erl, "+S", "1:1", "-noinput", "-noshell", "-eval", eval_code]
def run(algorithm):
started = time.perf_counter_ns()
result = subprocess.run(
command(variants[algorithm]),
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
check=True,
)
wall_ns = time.perf_counter_ns() - started
profile = re.search(
r"module_hash algorithm=\S+ count=(\d+) bytes=(\d+) ns=(\d+)",
result.stderr,
)
assert profile
return {
"wall_ms": wall_ns / 1_000_000,
"hash_ms": int(profile.group(3)) / 1_000_000,
"modules": int(profile.group(1)),
"hashed_bytes": int(profile.group(2)),
}
for _ in range(args.warmups):
for algorithm in variants:
run(algorithm)
samples = {algorithm: [] for algorithm in variants}
for index in range(args.runs):
order = list(variants) if index % 2 == 0 else list(reversed(variants))
for algorithm in order:
samples[algorithm].append(run(algorithm))
def summarize(values):
return {
"median_ms": statistics.median(values),
"mean_ms": statistics.mean(values),
"sample_variance_ms2": statistics.variance(values),
"sample_stddev_ms": statistics.stdev(values),
"min_ms": min(values),
"max_ms": max(values),
}
report = {
"runs": args.runs,
"warmups": args.warmups,
"variants": {},
}
for algorithm, runs in samples.items():
report["variants"][algorithm] = {
"modules": int(statistics.median(run["modules"] for run in runs)),
"hashed_bytes": int(statistics.median(run["hashed_bytes"] for run in runs)),
"wall": summarize([run["wall_ms"] for run in runs]),
"hash": summarize([run["hash_ms"] for run in runs]),
"samples": runs,
}
md5 = report["variants"]["md5"]
xxh3 = report["variants"]["xxh3-128"]
report["comparison"] = {
"hash_speedup": md5["hash"]["median_ms"] / xxh3["hash"]["median_ms"],
"wall_saved_ms": md5["wall"]["median_ms"] - xxh3["wall"]["median_ms"],
"wall_saved_percent": 100
* (md5["wall"]["median_ms"] - xxh3["wall"]["median_ms"])
/ md5["wall"]["median_ms"],
}
if args.json:
print(json.dumps(report, indent=2))
else:
print("algorithm\tmodules\tbytes\twall median ms\twall variance ms^2\twall stddev ms\thash median ms\thash variance ms^2\thash stddev ms")
for algorithm, result in report["variants"].items():
print(
f'{algorithm}\t{result["modules"]}\t{result["hashed_bytes"]}'
f'\t{result["wall"]["median_ms"]:.3f}'
f'\t{result["wall"]["sample_variance_ms2"]:.3f}'
f'\t{result["wall"]["sample_stddev_ms"]:.3f}'
f'\t{result["hash"]["median_ms"]:.3f}'
f'\t{result["hash"]["sample_variance_ms2"]:.6f}'
f'\t{result["hash"]["sample_stddev_ms"]:.3f}'
)
comparison = report["comparison"]
print(
f'XXH3-128 hash speedup: {comparison["hash_speedup"]:.1f}x; '
f'median wall time saved: {comparison["wall_saved_ms"]:.3f} ms '
f'({comparison["wall_saved_percent"]:.2f}%)'
)
EOF
# report/module-hash.patch
cat > 'report/module-hash.patch' <<'EOF'
diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c
index 1938c5e6ab..232e7c8e82 100644
--- a/erts/emulator/beam/beam_file.c
+++ b/erts/emulator/beam/beam_file.c
@@ -25,6 +25,16 @@
#endif
#include <stddef.h>
+#ifdef BEAM_MODULE_HASH_PROFILE
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#endif
+#ifdef BEAM_MODULE_HASH_XXH3
+#define XXH_STATIC_LINKING_ONLY
+#include <xxhash.h>
+#endif
#include "beam_file.h"
#include "beam_load.h"
#include "erl_zlib.h"
@@ -34,6 +44,101 @@
#include "erl_global_literals.h"
#include "erl_record.h"
+#ifdef BEAM_MODULE_HASH_XXH3
+typedef XXH3_state_t ModuleHashContext;
+
+static void module_hash_init(ModuleHashContext *context) {
+ XXH_errorcode result;
+
+ XXH3_INITSTATE(context);
+ result = XXH3_128bits_reset(context);
+ (void)result;
+ ASSERT(result == XXH_OK);
+}
+
+static void module_hash_update(ModuleHashContext *context,
+ const byte *data,
+ size_t size) {
+ XXH_errorcode result = XXH3_128bits_update(context, data, size);
+
+ (void)result;
+ ASSERT(result == XXH_OK);
+}
+
+static void module_hash_finish(byte checksum[CHECKSUM_SIZE],
+ ModuleHashContext *context) {
+ XXH128_canonical_t canonical;
+
+ XXH128_canonicalFromHash(&canonical, XXH3_128bits_digest(context));
+ sys_memcpy(checksum, canonical.digest, sizeof(canonical.digest));
+}
+
+#define MODULE_HASH_NAME "xxh3-128"
+#else
+typedef erts_md5_state ModuleHashContext;
+
+static void module_hash_init(ModuleHashContext *context) {
+ erts_md5_init(context);
+}
+
+static void module_hash_update(ModuleHashContext *context,
+ const byte *data,
+ size_t size) {
+ erts_md5_update(context, data, size);
+}
+
+static void module_hash_finish(byte checksum[CHECKSUM_SIZE],
+ ModuleHashContext *context) {
+ erts_md5_finish(checksum, context);
+}
+
+#define MODULE_HASH_NAME "md5"
+#endif
+
+#ifdef BEAM_MODULE_HASH_PROFILE
+static uint64_t module_hash_count;
+static uint64_t module_hash_bytes;
+static uint64_t module_hash_nanoseconds;
+
+static uint64_t module_hash_profile_now(void) {
+ struct timespec now;
+ int result = clock_gettime(CLOCK_MONOTONIC, &now);
+
+ (void)result;
+ ASSERT(result == 0);
+ return ((uint64_t)now.tv_sec * 1000000000) + now.tv_nsec;
+}
+
+static void module_hash_profile_report(void) {
+ fprintf(stderr,
+ "module_hash algorithm=%s count=%" PRIu64 " bytes=%" PRIu64
+ " ns=%" PRIu64 "\n",
+ MODULE_HASH_NAME,
+ module_hash_count,
+ module_hash_bytes,
+ module_hash_nanoseconds);
+}
+
+static void module_hash_profile_start(uint64_t *started) {
+ static int registered;
+
+ if (!registered) {
+ int result = atexit(module_hash_profile_report);
+
+ (void)result;
+ ASSERT(result == 0);
+ registered = 1;
+ }
+ *started = module_hash_profile_now();
+}
+
+static void module_hash_profile_finish(uint64_t started, uint64_t bytes) {
+ module_hash_count++;
+ module_hash_bytes += bytes;
+ module_hash_nanoseconds += module_hash_profile_now() - started;
+}
+#endif
+
#define LoadError(Expr) \
do { \
(void)(Expr); \
@@ -1750,23 +1855,36 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
/* Compute module checksum. Please keep all parsing above this section */
{
- erts_md5_state md5;
+ ModuleHashContext hash;
+#ifdef BEAM_MODULE_HASH_PROFILE
+ uint64_t hash_started;
+ uint64_t hash_bytes = 0;
+#define MODULE_HASH_UPDATE(Data, Size) \
+ do { \
+ size_t hash_size = (Size); \
+ module_hash_update(&hash, Data, hash_size);\
+ hash_bytes += hash_size; \
+ } while (0)
+ module_hash_profile_start(&hash_started);
+#else
+#define MODULE_HASH_UPDATE(Data, Size) module_hash_update(&hash, Data, Size)
+#endif
- erts_md5_init(&md5);
+ module_hash_init(&hash);
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[UTF8_ATOM_CHUNK].data,
chunks[UTF8_ATOM_CHUNK].size);
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[CODE_CHUNK].data,
chunks[CODE_CHUNK].size);
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[STR_CHUNK].data,
chunks[STR_CHUNK].size);
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[IMP_CHUNK].data,
chunks[IMP_CHUNK].size);
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[EXP_CHUNK].data,
chunks[EXP_CHUNK].size);
@@ -1778,7 +1896,7 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
* checksum hash, as it's derived using a (broken and superseded)
* endian-dependent hash function. */
if (left >= 4) {
- erts_md5_update(&md5, (byte*)start, 4);
+ MODULE_HASH_UPDATE((byte*)start, 4);
start += 4;
left -= 4;
@@ -1787,9 +1905,9 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
static byte zero[4] = {0, 0, 0, 0};
/* Include: Function Arity Index NumFree */
- erts_md5_update(&md5, (byte*)start, 20);
+ MODULE_HASH_UPDATE((byte*)start, 20);
/* Set to zero: OldUniq */
- erts_md5_update(&md5, (byte*)zero, 4);
+ MODULE_HASH_UPDATE((byte*)zero, 4);
start += 24;
left -= 24;
@@ -1803,30 +1921,34 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
}
if (chunks[LITERAL_CHUNK].size > 0) {
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[LITERAL_CHUNK].data,
chunks[LITERAL_CHUNK].size);
}
if (chunks[META_CHUNK].size > 0) {
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[META_CHUNK].data,
chunks[META_CHUNK].size);
}
if (chunks[RECORD_CHUNK].size > 0) {
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[RECORD_CHUNK].data,
chunks[RECORD_CHUNK].size);
}
if (chunks[DEBUG_CHUNK].size > 0) {
- erts_md5_update(&md5,
+ MODULE_HASH_UPDATE(
(byte*)chunks[DEBUG_CHUNK].data,
chunks[DEBUG_CHUNK].size);
}
- erts_md5_finish(beam->checksum, &md5);
+ module_hash_finish(beam->checksum, &hash);
+#ifdef BEAM_MODULE_HASH_PROFILE
+ module_hash_profile_finish(hash_started, hash_bytes);
+#endif
+#undef MODULE_HASH_UPDATE
}
return BEAMFILE_READ_SUCCESS;
EOF
```
Proposed changes (or rather, rough approximation of them)
- Deprecate
md5field inmodule_info/1. - Add
beam_lib:checksum/1andchecksumfield inmodule_info/1which return tuple of(algorithm, digest) - After deprecation period, return
undefinedformd5field frommodule_info(see “ETF” below). We can also consider doing this forbeam_lib:md5/1as well.
-type module_checksum() ::
{xxh3_128, <<_:128>>}.
-spec checksum(Beam) ->
{ok, {module(), module_checksum()}} |
{error, beam_lib, chnk_rsn()}
when
Beam :: beam().
-spec module_info(checksum) -> module_checksum().
ETF
ETF also uses MD5 in form of Uniq field in NEW_FUN_EXT (local funs, closures). I’m light on details there and would love some discussion on how to approach migration for ETF (especially considering ETF is used in the distribution and stored in databases/files).
With the new checksum algorithm, nodes in the distribution would most likely need to negotiate between old vs new algorithm for transition period.
.beam
AFAIK, this wouldn’t affect the .beam files layout at all.
The possible extension would be to calculate checksum when compiling and storing it in the separate chunk. I personally don’t think it’s worth it:
- newer hash functions are plenty fast, so we’d only save some IO costs (also: in contexts where bytecode bytes should be already in caches).
- having checksum complicates working with bytecode — patching or stripping
.beamwould change the checksum and require to recalculate it.