From 76734a5672377a1d09c584fe607650ee48da4076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Thu, 3 Sep 2026 00:45:04 -0400 Subject: [PATCH 1/5] build: bump opendht minimum version to 3.0 version 3.0 introduced the persistance API, which we need. --- CMakeLists.txt | 2 +- README.md | 2 +- configure.ac | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c5892e..357f9ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") ############################ # Find required packages # ############################ -find_package(opendht 1.2.0 REQUIRED) +find_package(opendht 3.0.0 REQUIRED) find_package(CURLpp REQUIRED) find_package(glibmm REQUIRED) find_package(B64 REQUIRED) diff --git a/README.md b/README.md index 16ad1f0..0482680 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Milis Linux: mps kur dpaste (https://github.com/milisarge/malfs-milis/blob/maste ## Dependencies -- [OpenDHT](https://github.com/savoirfairelinux/opendht/) (minimal version: 1.2.0) +- [OpenDHT](https://github.com/savoirfairelinux/opendht/) (minimal version: 3.0.0) - [msgpack-c](https://github.com/msgpack/msgpack-c) - [gpgmepp](https://github.com/KDE/gpgmepp) - [json.hpp](https://github.com/nlohmann/json) (required version for CMake: 2.1.1) diff --git a/configure.ac b/configure.ac index 3ff8a0d..00d5bcd 100644 --- a/configure.ac +++ b/configure.ac @@ -18,7 +18,7 @@ AS_IF([test "x$enable_debug" = "xyes"], AC_PROG_CXX AC_PROG_RANLIB -PKG_CHECK_MODULES([OpenDHT], [opendht >= 1.2]) +PKG_CHECK_MODULES([OpenDHT], [opendht >= 3.0.0]) PKG_CHECK_MODULES([CURLPP], [curlpp]) PKG_CHECK_MODULES([GLIBMM], [glibmm-2.4]) From d17666f7c72326c9701b9763159b2c57b6c625d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Thu, 3 Sep 2026 00:47:23 -0400 Subject: [PATCH 2/5] perf: add a DHT cache machanism --- README.md | 8 ++++++ src/node.cpp | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/node.h | 33 +++++++++++++++++------ 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0482680..088a08a 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,14 @@ Milis Linux: mps kur dpaste (https://github.com/milisarge/malfs-milis/blob/maste - [catch](https://github.com/catchorg/Catch2) for unit tests - getopt (util-linux) +## Caching + +To avoid the multi-second DHT cold start on every invocation, `dpaste` caches +the DHT identity and the node state (routing table) on disk. The cache lives +in `${XDG_CACHE_HOME}/dpaste` (usually `~/.cache/dpaste`) and can be relocated +with the `DPASTE_CACHE_DIR` environment variable. The state is refreshed at +the end of every run; the first run remains slow, subsequent ones start warm. + ## Pastebin over DHT A DHT is efficient and requires no infrastructure. In practice, you can always diff --git a/src/node.cpp b/src/node.cpp index dcbeca2..25752a4 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -21,8 +21,14 @@ #include #include #include +#include +#include +#include +#include +#include #include +#include #include "node.h" @@ -30,6 +36,75 @@ namespace dpaste { const constexpr char* Node::DPASTE_USER_TYPE; +namespace { + +/** + * Directory holding the on-disk caches (identity + node state). + * Can be overridden with the DPASTE_CACHE_DIR environment variable + * (e.g. for tests). Defaults to ${XDG_CACHE_HOME}/dpaste. + */ +std::string cacheDir() { + const char* env = std::getenv("DPASTE_CACHE_DIR"); + std::string dir = env and *env ? env : Glib::get_user_cache_dir() + "/dpaste"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + return dir; +} + +} /* anonymous namespace */ + +dht::crypto::Identity Node::loadIdentity() { + /* Only try the cache if the files exist: a missing cache is the normal + * first-run case and shouldn't print a scary error. */ + std::ifstream key_file(identity_path_ + ".pem"); + if (key_file.good()) { + try { + auto id = dht::crypto::loadIdentity(identity_path_); + if (id.first and id.second) + return id; + } catch (const std::exception& e) { + std::cerr << "dpaste: cached identity is corrupt (" << e.what() << "), generating a new one." << std::endl; + } + } + + auto id = dht::crypto::generateIdentity(); + try { + /* Write to temporary files first, then rename, so concurrent dpaste + * processes never leave a half-written identity in the cache. */ + auto tmp_path = identity_path_ + ".tmp"; + dht::crypto::saveIdentity(id, tmp_path); + std::rename((tmp_path + ".pem").c_str(), (identity_path_ + ".pem").c_str()); + std::rename((tmp_path + ".crt").c_str(), (identity_path_ + ".crt").c_str()); + } catch (const std::exception& e) { + std::cerr << "dpaste: failed to cache identity: " << e.what() << std::endl; + } + return id; +} + +void Node::run(uint16_t port, std::string bootstrap_hostname, std::string bootstrap_port) { + if (running_) + return; + + auto dir = cacheDir(); + identity_path_ = dir + "/identity"; + nodes_path_ = dir + "/nodes"; + + /* Load (or generate and cache) the identity so we don't pay for RSA key + * generation on every run. */ + auto identity = loadIdentity(); + + /* Ask OpenDHT to load its state (routing table) on start and save it on + * shutdown; this turns the multi-second DHT cold start into a warm one. */ + dht::DhtRunner::Config config; + config.dht_config.id = identity; + config.dht_config.node_config.persist_path = nodes_path_; + config.threaded = true; + node_.run(port, config); + + node_.bootstrap(bootstrap_hostname, bootstrap_port); + running_ = true; +} + bool Node::paste(const std::string& code, dht::Blob&& blob, dht::DoneCallbackSimple&& cb) { auto v = std::make_shared(std::forward(blob)); v->user_type = DPASTE_USER_TYPE; diff --git a/src/node.h b/src/node.h index 48f1f75..71af14e 100644 --- a/src/node.h +++ b/src/node.h @@ -54,15 +54,22 @@ class Node { static const constexpr char* DPASTE_USER_TYPE = "dpaste"; Node() {} - virtual ~Node () {} - - void run(uint16_t port = 0, std::string bootstrap_hostname = DEFAULT_BOOTSTRAP_NODE, std::string bootstrap_port = DEFAULT_BOOTSTRAP_PORT) { + virtual ~Node() { + /* Persist the node state (routing table) on disk so the next run + * starts warm instead of cold-bootstrapping the DHT. */ if (running_) - return; - node_.run(port, dht::crypto::generateIdentity(), true); - node_.bootstrap(bootstrap_hostname, bootstrap_port); - running_ = true; - }; + stop(); + } + + /** + * Start the DHT node. The identity and the node state are cached on disk + * (see DPASTE_CACHE_DIR below) so subsequent runs connect much faster. + * + * @param port Local port to bind (0 for random). + * @param bootstrap_hostname Hostname of the bootstrap node. + * @param bootstrap_port Port of the bootstrap node. + */ + void run(uint16_t port = 0, std::string bootstrap_hostname = DEFAULT_BOOTSTRAP_NODE, std::string bootstrap_port = DEFAULT_BOOTSTRAP_PORT); void stop() { std::condition_variable cv; @@ -116,9 +123,19 @@ class Node { private: + /** + * Load the DHT identity from the on-disk cache, generating and caching a + * new one if the cache is missing or corrupt. + */ + dht::crypto::Identity loadIdentity(); + dht::DhtRunner node_; bool running_ {false}; + /* on-disk cache locations (identity + DHT node state) */ + std::string identity_path_ {}; + std::string nodes_path_ {}; + std::uniform_int_distribution codeDist_; std::mt19937_64 rand_; }; From b0c88af6f7b775e7b30dfbd24400f57c6dc965ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 7 Sep 2026 18:27:07 -0400 Subject: [PATCH 3/5] remove identity stuff --- CMakeLists.txt | 2 +- README.md | 9 +++++---- configure.ac | 3 +-- src/node.cpp | 44 +++----------------------------------------- src/node.h | 18 ++++-------------- 5 files changed, 14 insertions(+), 62 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 357f9ab..4fa8d39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") ############################ # Find required packages # ############################ -find_package(opendht 3.0.0 REQUIRED) +find_package(opendht 1.8.2 REQUIRED) find_package(CURLpp REQUIRED) find_package(glibmm REQUIRED) find_package(B64 REQUIRED) diff --git a/README.md b/README.md index 088a08a..ac05441 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Milis Linux: mps kur dpaste (https://github.com/milisarge/malfs-milis/blob/maste ## Dependencies -- [OpenDHT](https://github.com/savoirfairelinux/opendht/) (minimal version: 3.0.0) +- [OpenDHT](https://github.com/savoirfairelinux/opendht/) (minimal version: 1.8.2) - [msgpack-c](https://github.com/msgpack/msgpack-c) - [gpgmepp](https://github.com/KDE/gpgmepp) - [json.hpp](https://github.com/nlohmann/json) (required version for CMake: 2.1.1) @@ -92,10 +92,12 @@ Milis Linux: mps kur dpaste (https://github.com/milisarge/malfs-milis/blob/maste ## Caching To avoid the multi-second DHT cold start on every invocation, `dpaste` caches -the DHT identity and the node state (routing table) on disk. The cache lives +the OpenDHT node state (including its routing table) on disk. The cache lives in `${XDG_CACHE_HOME}/dpaste` (usually `~/.cache/dpaste`) and can be relocated with the `DPASTE_CACHE_DIR` environment variable. The state is refreshed at -the end of every run; the first run remains slow, subsequent ones start warm. +the end of every run, allowing known peers to be reused and improving +bootstrap resilience. The first run remains slow, and random-key lookups on +the public DHT still cannot be eliminated. ## Pastebin over DHT @@ -123,4 +125,3 @@ not likely to be "down". - Simon Désaulniers - Adrien Béraud - diff --git a/configure.ac b/configure.ac index 00d5bcd..304ccb4 100644 --- a/configure.ac +++ b/configure.ac @@ -18,7 +18,7 @@ AS_IF([test "x$enable_debug" = "xyes"], AC_PROG_CXX AC_PROG_RANLIB -PKG_CHECK_MODULES([OpenDHT], [opendht >= 3.0.0]) +PKG_CHECK_MODULES([OpenDHT], [opendht >= 1.8.2]) PKG_CHECK_MODULES([CURLPP], [curlpp]) PKG_CHECK_MODULES([GLIBMM], [glibmm-2.4]) @@ -45,4 +45,3 @@ AM_COND_IF([DPASTE_TEST], AC_OUTPUT # vim: set ts=2 sw=2 tw=120 et : - diff --git a/src/node.cpp b/src/node.cpp index 25752a4..121a363 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -21,10 +21,8 @@ #include #include #include -#include #include #include -#include #include #include @@ -39,7 +37,7 @@ const constexpr char* Node::DPASTE_USER_TYPE; namespace { /** - * Directory holding the on-disk caches (identity + node state). + * Directory holding the on-disk OpenDHT node state. * Can be overridden with the DPASTE_CACHE_DIR environment variable * (e.g. for tests). Defaults to ${XDG_CACHE_HOME}/dpaste. */ @@ -53,51 +51,15 @@ std::string cacheDir() { } /* anonymous namespace */ -dht::crypto::Identity Node::loadIdentity() { - /* Only try the cache if the files exist: a missing cache is the normal - * first-run case and shouldn't print a scary error. */ - std::ifstream key_file(identity_path_ + ".pem"); - if (key_file.good()) { - try { - auto id = dht::crypto::loadIdentity(identity_path_); - if (id.first and id.second) - return id; - } catch (const std::exception& e) { - std::cerr << "dpaste: cached identity is corrupt (" << e.what() << "), generating a new one." << std::endl; - } - } - - auto id = dht::crypto::generateIdentity(); - try { - /* Write to temporary files first, then rename, so concurrent dpaste - * processes never leave a half-written identity in the cache. */ - auto tmp_path = identity_path_ + ".tmp"; - dht::crypto::saveIdentity(id, tmp_path); - std::rename((tmp_path + ".pem").c_str(), (identity_path_ + ".pem").c_str()); - std::rename((tmp_path + ".crt").c_str(), (identity_path_ + ".crt").c_str()); - } catch (const std::exception& e) { - std::cerr << "dpaste: failed to cache identity: " << e.what() << std::endl; - } - return id; -} - void Node::run(uint16_t port, std::string bootstrap_hostname, std::string bootstrap_port) { if (running_) return; auto dir = cacheDir(); - identity_path_ = dir + "/identity"; - nodes_path_ = dir + "/nodes"; - - /* Load (or generate and cache) the identity so we don't pay for RSA key - * generation on every run. */ - auto identity = loadIdentity(); - /* Ask OpenDHT to load its state (routing table) on start and save it on - * shutdown; this turns the multi-second DHT cold start into a warm one. */ + * shutdown; this reuses known peers and improves bootstrap resilience. */ dht::DhtRunner::Config config; - config.dht_config.id = identity; - config.dht_config.node_config.persist_path = nodes_path_; + config.dht_config.node_config.persist_path = dir + "/nodes"; config.threaded = true; node_.run(port, config); diff --git a/src/node.h b/src/node.h index 71af14e..db61602 100644 --- a/src/node.h +++ b/src/node.h @@ -55,15 +55,15 @@ class Node { Node() {} virtual ~Node() { - /* Persist the node state (routing table) on disk so the next run - * starts warm instead of cold-bootstrapping the DHT. */ + /* Persist the node state (routing table) on disk so the next run can + * reuse known peers instead of cold-bootstrapping the DHT. */ if (running_) stop(); } /** - * Start the DHT node. The identity and the node state are cached on disk - * (see DPASTE_CACHE_DIR below) so subsequent runs connect much faster. + * Start the DHT node. Its routing state is cached on disk (see + * DPASTE_CACHE_DIR below) so subsequent runs can connect more quickly. * * @param port Local port to bind (0 for random). * @param bootstrap_hostname Hostname of the bootstrap node. @@ -123,19 +123,9 @@ class Node { private: - /** - * Load the DHT identity from the on-disk cache, generating and caching a - * new one if the cache is missing or corrupt. - */ - dht::crypto::Identity loadIdentity(); - dht::DhtRunner node_; bool running_ {false}; - /* on-disk cache locations (identity + DHT node state) */ - std::string identity_path_ {}; - std::string nodes_path_ {}; - std::uniform_int_distribution codeDist_; std::mt19937_64 rand_; }; From 2e059b5ebec9ceb704e32f6497bdf297762b9295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 7 Sep 2026 19:44:52 -0400 Subject: [PATCH 4/5] add a regression test --- tests/node.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/node.cpp b/tests/node.cpp index 4624061..106f383 100644 --- a/tests/node.cpp +++ b/tests/node.cpp @@ -20,6 +20,10 @@ #include +#include +#include +#include + #include "tests.h" #include "node.h" @@ -35,13 +39,58 @@ class PirateNodeTester { bool is_running(const dpaste::Node& n) const { return n.running_; } }; +class CacheFileGuard { + int fd_ {-1}; + std::string path_; + bool had_previous_value_ {false}; + std::string previous_value_; + +public: + CacheFileGuard() { + const char* previous = std::getenv("DPASTE_CACHE_DIR"); + had_previous_value_ = previous != nullptr; + if (had_previous_value_) + previous_value_ = previous; + + char path[] = "/tmp/dpaste-cache-XXXXXX"; + fd_ = mkstemp(path); + if (fd_ == -1) + throw std::runtime_error("could not create temporary cache file"); + path_ = path; + + if (setenv("DPASTE_CACHE_DIR", path_.c_str(), 1) != 0) { + close(fd_); + unlink(path_.c_str()); + throw std::runtime_error("could not set DPASTE_CACHE_DIR"); + } + } + + ~CacheFileGuard() { + if (had_previous_value_) + setenv("DPASTE_CACHE_DIR", previous_value_.c_str(), 1); + else + unsetenv("DPASTE_CACHE_DIR"); + close(fd_); + unlink(path_.c_str()); + } +}; + +TEST_CASE("Node refuses a cache path that is a regular file", "[Node][cache]") { + PirateNodeTester pt; + CacheFileGuard cache; + + dpaste::Node node {}; + REQUIRE_FALSE(node.run()); + REQUIRE_FALSE(pt.is_running(node)); +} + TEST_CASE("Node get/paste on DHT", "[Node][get][paste]") { PirateNodeTester pt; const std::string PIN = random_pin(); std::vector data = {0, 1, 2, 3, 4}; dpaste::Node node {}; - node.run(); + REQUIRE(node.run()); SECTION ( "pasting data {0,1,2,3,4}" ) { REQUIRE ( node.paste(PIN, std::vector {data}) ); @@ -60,4 +109,3 @@ TEST_CASE("Node get/paste on DHT", "[Node][get][paste]") { } /* dpaste */ /* vim: set ts=4 sw=4 tw=120 et :*/ - From cc8a1d0d970b72259a620ddd31560ed4c9a78229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 7 Sep 2026 19:45:00 -0400 Subject: [PATCH 5/5] fix post-reviews comments --- src/bin.cpp | 11 ++++++----- src/bin.h | 2 +- src/node.cpp | 25 +++++++++++++++++-------- src/node.h | 3 +-- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/bin.cpp b/src/bin.cpp index 42b04b6..81718f8 100644 --- a/src/bin.cpp +++ b/src/bin.cpp @@ -36,7 +36,7 @@ namespace dpaste { const constexpr uint8_t Bin::PROTO_VERSION; -Bin::Bin() { +Bin::Bin() : node_ready_(node.run()) { /* load dpaste config */ auto config_file = conf::ConfigurationFile(); config_file.load(); @@ -48,7 +48,6 @@ Bin::Bin() { conv >> port; } - node.run(); http_client_ = std::make_unique(conf_.at("host"), port); } @@ -69,13 +68,16 @@ std::pair Bin::get(std::string&& code, bool no_decrypt) { std::vector data {data_str.begin(), data_str.end()}; /* if fail, then perform request from local node */ - if (data.empty()) { + if (data.empty() and node_ready_) { /* get a pasted blob */ auto values = node.get(lcode); if (not values.empty()) data = values.front(); } + if (data.empty() and not node_ready_) + return {false, ""}; + if (not data.empty()) { Packet p; try { @@ -182,7 +184,7 @@ std::string Bin::paste(std::vector&& data, std::unique_ptrput(code, {bin_packet.begin(), bin_packet.end()}); - if (not success) + if (not success and node_ready_) success = node.paste(code, std::move(bin_packet)); return success ? DPASTE_URI_PREFIX+code+pwd : ""; @@ -225,4 +227,3 @@ void Bin::Packet::deserialize(const std::vector& pbuffer) { } /* dpaste */ /* vim:set et sw=4 ts=4 tw=120: */ - diff --git a/src/bin.h b/src/bin.h index 50acf0d..b0abea5 100644 --- a/src/bin.h +++ b/src/bin.h @@ -124,9 +124,9 @@ class Bin { /* transport */ std::unique_ptr http_client_ {}; Node node {}; + bool node_ready_ {false}; }; } /* dpaste */ /* vim:set et sw=4 ts=4 tw=120: */ - diff --git a/src/node.cpp b/src/node.cpp index 121a363..aa9c526 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -24,11 +24,13 @@ #include #include #include +#include #include #include #include "node.h" +#include "log.h" namespace dpaste { @@ -41,30 +43,37 @@ namespace { * Can be overridden with the DPASTE_CACHE_DIR environment variable * (e.g. for tests). Defaults to ${XDG_CACHE_HOME}/dpaste. */ -std::string cacheDir() { +std::optional create_cache_dir() { const char* env = std::getenv("DPASTE_CACHE_DIR"); - std::string dir = env and *env ? env : Glib::get_user_cache_dir() + "/dpaste"; + const std::filesystem::path cache_dir = env and *env ? env : Glib::get_user_cache_dir() + "/dpaste"; std::error_code ec; - std::filesystem::create_directories(dir, ec); - return dir; + std::filesystem::create_directories(cache_dir, ec); + if (ec) { + DPASTE_MSG("Failed to create cache directory '%s': %s", cache_dir.string().c_str(), ec.message().c_str()); + return std::nullopt; + } + return cache_dir; } } /* anonymous namespace */ -void Node::run(uint16_t port, std::string bootstrap_hostname, std::string bootstrap_port) { +bool Node::run(uint16_t port, std::string bootstrap_hostname, std::string bootstrap_port) { if (running_) - return; + return true; - auto dir = cacheDir(); + const auto cache_dir = create_cache_dir(); + if (not cache_dir) + return false; /* Ask OpenDHT to load its state (routing table) on start and save it on * shutdown; this reuses known peers and improves bootstrap resilience. */ dht::DhtRunner::Config config; - config.dht_config.node_config.persist_path = dir + "/nodes"; + config.dht_config.node_config.persist_path = (*cache_dir / "nodes").string(); config.threaded = true; node_.run(port, config); node_.bootstrap(bootstrap_hostname, bootstrap_port); running_ = true; + return true; } bool Node::paste(const std::string& code, dht::Blob&& blob, dht::DoneCallbackSimple&& cb) { diff --git a/src/node.h b/src/node.h index db61602..64204ca 100644 --- a/src/node.h +++ b/src/node.h @@ -69,7 +69,7 @@ class Node { * @param bootstrap_hostname Hostname of the bootstrap node. * @param bootstrap_port Port of the bootstrap node. */ - void run(uint16_t port = 0, std::string bootstrap_hostname = DEFAULT_BOOTSTRAP_NODE, std::string bootstrap_port = DEFAULT_BOOTSTRAP_PORT); + bool run(uint16_t port = 0, std::string bootstrap_hostname = DEFAULT_BOOTSTRAP_NODE, std::string bootstrap_port = DEFAULT_BOOTSTRAP_PORT); void stop() { std::condition_variable cv; @@ -131,4 +131,3 @@ class Node { }; } /* dpaste */ -