Skip to content

Commit c597534

Browse files
committed
admin : added a new admin endpoint - /logs to fetch log files + related bugfix
- Implement `GET /logs?type=<mlog|qlog>&connection_id=<hex>` route. - Offload blocking file I/O (`std::ifstream::read`) to `folly::getGlobalCPUExecutor()` to avoid stalling the admin event loop. - Bugfix: create log dir if not exists
1 parent 4915b55 commit c597534

8 files changed

Lines changed: 361 additions & 1 deletion

File tree

CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ add_library(moqx_core STATIC
158158
src/admin/BuiltinRoutes.cpp
159159
src/admin/CachePurgeHandler.cpp
160160
src/admin/ConfigHandler.cpp
161+
src/admin/ConnectionLogsHandler.cpp
161162
src/admin/MetricsHandler.cpp
162163
src/admin/TrackMetricsHandler.cpp
163164
src/admin/StateHandler.cpp
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/*
2+
* Copyright (c) OpenMOQ contributors.
3+
* This source code is licensed under the Apache 2.0 license found in the
4+
* LICENSE file in the root directory of this source tree.
5+
*/
6+
7+
#include "admin/ConnectionLogsHandler.h"
8+
9+
#include <cctype>
10+
#include <string>
11+
#include <string_view>
12+
13+
#include <fcntl.h>
14+
#include <sys/stat.h>
15+
16+
#include <folly/CancellationToken.h>
17+
#include <folly/File.h>
18+
#include <folly/FileUtil.h>
19+
#include <folly/coro/Invoke.h>
20+
#include <folly/coro/Task.h>
21+
#include <folly/coro/WithCancellation.h>
22+
#include <folly/executors/GlobalExecutor.h>
23+
#include <folly/io/IOBuf.h>
24+
#include <folly/io/async/EventBaseManager.h>
25+
#include <folly/logging/xlog.h>
26+
#include <proxygen/httpserver/ResponseBuilder.h>
27+
#include <proxygen/lib/http/HTTPMessage.h>
28+
29+
#include "admin/AdminResponse.h"
30+
#include "admin/AdminServer.h"
31+
32+
namespace openmoq::moqx::admin {
33+
34+
namespace {
35+
36+
constexpr size_t kMaxDownloadBytes = 512ULL * 1024 * 1024; // 512 MB hard cap
37+
38+
// Normalize a raw connection ID string:
39+
// - strip 0x/0X prefix
40+
// - lowercase hex digits
41+
// - validate hex-only, 1–40 chars
42+
std::optional<std::string> normalizeConnectionId(std::string_view raw) {
43+
if (raw.size() >= 2 && raw[0] == '0' && (raw[1] == 'x' || raw[1] == 'X')) {
44+
raw.remove_prefix(2);
45+
}
46+
std::string result;
47+
result.reserve(raw.size());
48+
for (char c : raw) {
49+
if (!std::isxdigit(static_cast<unsigned char>(c)))
50+
return std::nullopt;
51+
result += static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
52+
}
53+
if (result.empty() || result.size() > 40)
54+
return std::nullopt;
55+
return result;
56+
}
57+
58+
// Read an entire file into an IOBuf. Returns nullptr if the file cannot be
59+
// opened, is empty, or exceeds maxBytes.
60+
std::unique_ptr<folly::IOBuf> readFileToIOBuf(const std::string& path, size_t maxBytes) {
61+
folly::File file;
62+
try {
63+
file = folly::File(path, O_RDONLY);
64+
} catch (const std::exception&) {
65+
return nullptr;
66+
}
67+
68+
struct stat st{};
69+
if (::fstat(file.fd(), &st) != 0 || !S_ISREG(st.st_mode))
70+
return nullptr;
71+
const auto size = static_cast<size_t>(st.st_size);
72+
if (size == 0 || size > maxBytes)
73+
return nullptr;
74+
75+
auto content = std::make_unique<std::string>();
76+
if (!folly::readFile(file.fd(), *content, size) || content->size() != size)
77+
return nullptr;
78+
79+
auto* data = content->data();
80+
const auto len = content->size();
81+
return folly::IOBuf::takeOwnership(
82+
data,
83+
len,
84+
[](void*, void* userData) { delete static_cast<std::string*>(userData); },
85+
content.release()
86+
);
87+
}
88+
89+
} // namespace
90+
91+
void registerConnectionLogsRoutes(
92+
AdminServer& adminServer,
93+
const std::optional<config::LoggingConfig>& logging
94+
) {
95+
std::string mlogDir, qlogDir;
96+
if (logging) {
97+
if (logging->mlog && !logging->mlog->dir.empty()) {
98+
mlogDir = logging->mlog->dir;
99+
}
100+
if (logging->qlog && !logging->qlog->dir.empty()) {
101+
qlogDir = logging->qlog->dir;
102+
}
103+
}
104+
105+
// ── GET /logs?connection_id=<hex>&type=mlog|qlog
106+
//
107+
// Path is constructed directly as {dir}/{normalized_cid}.{ext}
108+
adminServer.addRoute(
109+
"GET",
110+
"/logs",
111+
[mlogDir = std::move(mlogDir), qlogDir = std::move(qlogDir)](
112+
std::unique_ptr<proxygen::HTTPMessage> req,
113+
std::unique_ptr<folly::IOBuf> /*body*/,
114+
proxygen::ResponseHandler* downstream,
115+
folly::CancellationToken cancelToken
116+
) {
117+
// Resolve type → directory, file extension, Content-Type.
118+
const auto& typeStr = req->getQueryParam("type");
119+
const std::string* dir = nullptr;
120+
const char* ext = nullptr;
121+
if (typeStr == "mlog") {
122+
dir = &mlogDir;
123+
ext = ".mlog";
124+
} else if (typeStr == "qlog") {
125+
dir = &qlogDir;
126+
ext = ".qlog";
127+
} else {
128+
sendError(downstream, 400, "type must be 'mlog' or 'qlog'\n");
129+
return;
130+
}
131+
132+
if (dir->empty()) {
133+
sendError(downstream, 503, "that log type is not configured\n");
134+
return;
135+
}
136+
137+
const auto& rawCid = req->getQueryParam("connection_id");
138+
if (rawCid.empty()) {
139+
sendError(downstream, 400, "missing connection_id\n");
140+
return;
141+
}
142+
143+
auto normCid = normalizeConnectionId(rawCid);
144+
if (!normCid) {
145+
sendError(downstream, 400, "invalid connection_id\n");
146+
return;
147+
}
148+
149+
// {dir}/{normalizedCid}.{ext}
150+
auto filePath = *dir + "/" + *normCid + ext;
151+
auto fileName = *normCid + ext;
152+
153+
auto* evb = folly::EventBaseManager::get()->getEventBase();
154+
folly::coro::co_withCancellation(
155+
cancelToken,
156+
folly::coro::co_withExecutor(
157+
evb,
158+
[](auto path, auto name, auto* ds, auto token) -> folly::coro::Task<void> {
159+
if (token.isCancellationRequested())
160+
co_return;
161+
162+
// Read the file on the global CPU pool to avoid blocking
163+
// the admin event-loop thread.
164+
std::unique_ptr<folly::IOBuf> fileBuf;
165+
try {
166+
fileBuf = co_await folly::coro::co_withExecutor(
167+
folly::getGlobalCPUExecutor(),
168+
folly::coro::co_invoke(
169+
[path = std::move(path), maxBytes = kMaxDownloadBytes](
170+
) -> folly::coro::Task<std::unique_ptr<folly::IOBuf>> {
171+
co_return readFileToIOBuf(path, maxBytes);
172+
}
173+
)
174+
);
175+
} catch (const std::exception& e) {
176+
XLOG(ERR) << "ConnectionLogsHandler: file read threw: " << e.what();
177+
if (!token.isCancellationRequested()) {
178+
sendError(ds, 500, "internal error\n");
179+
}
180+
co_return;
181+
}
182+
183+
if (token.isCancellationRequested())
184+
co_return;
185+
186+
if (!fileBuf) {
187+
sendError(ds, 404, "log file not found or exceeds size limit\n");
188+
co_return;
189+
}
190+
191+
proxygen::ResponseBuilder(ds)
192+
.status(200, proxygen::HTTPMessage::getDefaultReason(200))
193+
.header("Content-Type", "application/json")
194+
.header("Content-Disposition", "attachment; filename=\"" + name + "\"")
195+
.body(std::move(fileBuf))
196+
.sendWithEOM();
197+
}(std::move(filePath), std::move(fileName), downstream, cancelToken)
198+
)
199+
)
200+
.start();
201+
}
202+
);
203+
}
204+
205+
} // namespace openmoq::moqx::admin

src/admin/ConnectionLogsHandler.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright (c) OpenMOQ contributors.
3+
* This source code is licensed under the Apache 2.0 license found in the
4+
* LICENSE file in the root directory of this source tree.
5+
*/
6+
7+
#pragma once
8+
9+
#include <optional>
10+
11+
#include "config/Config.h"
12+
13+
namespace openmoq::moqx::admin {
14+
15+
class AdminServer;
16+
17+
// Registers GET /logs on the admin server.
18+
//
19+
// GET /logs?connection_id=<hex>&type=mlog|qlog
20+
// Resolves the file path as {log_dir}/{normalized_cid}.{ext} and streams
21+
// the file directly. No index or disk scan is required — files written
22+
// after startup are immediately reachable.
23+
// Responds 400 for missing/invalid params, 503 if the requested type is
24+
// not configured, 404 if the file does not exist.
25+
void registerConnectionLogsRoutes(
26+
AdminServer& adminServer,
27+
const std::optional<config::LoggingConfig>& logging
28+
);
29+
30+
} // namespace openmoq::moqx::admin

src/config/ConfigResolver.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1240,7 +1240,11 @@ folly::Expected<ResolvedConfig, std::string> resolveConfig(const ParsedConfig& c
12401240
if (!mlogConfig.dir.empty()) {
12411241
std::error_code ec;
12421242
const auto st = std::filesystem::status(mlogConfig.dir, ec);
1243-
if (ec) {
1243+
// A missing directory is not an error here: LogSetup creates it
1244+
// (via create_directories) before logging starts. Only reject
1245+
// genuine access failures (e.g. permission denied on a parent
1246+
// directory).
1247+
if (ec && st.type() != std::filesystem::file_type::not_found) {
12441248
return folly::makeUnexpected(
12451249
"Failed to access mlog directory '" + mlogConfig.dir + "': " + ec.message()
12461250
);

src/main.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "admin/BuiltinRoutes.h"
1111
#include "admin/CachePurgeHandler.h"
1212
#include "admin/ConfigHandler.h"
13+
#include "admin/ConnectionLogsHandler.h"
1314
#include "admin/MetricsHandler.h"
1415
#include "admin/StateHandler.h"
1516
#include "admin/TrackMetricsHandler.h"
@@ -204,6 +205,7 @@ int main(int argc, char* argv[]) {
204205
}
205206
admin::registerTrackMetricsRoute(adminServer, context, trackLimits);
206207
admin::registerConfigRoute(adminServer, std::make_shared<const cfg::Config>(config));
208+
admin::registerConnectionLogsRoutes(adminServer, config.logging);
207209

208210
// === 8. Start serving ===
209211
for (auto& server : servers) {

test/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,10 @@ add_test(
348348
NAME admin_config_endpoint
349349
COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_config.sh $<TARGET_FILE:moqx>
350350
)
351+
add_test(
352+
NAME admin_connection_logs_endpoint
353+
COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_connection_logs.sh $<TARGET_FILE:moqx>
354+
)
351355
add_test(
352356
NAME admin_cache_purge_concurrency_test
353357
COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_cache_purge_race.sh $<TARGET_FILE:moqx>

test/test_admin_connection_logs.sh

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
BINARY="${1:-$(dirname "$0")/../build/moqx}"
5+
# shellcheck source=test_ports.sh
6+
source "$(dirname "$0")/test_ports.sh"
7+
LISTEN_PORT=$TEST_ADMIN_CONNECTION_LOGS_LISTEN
8+
ADMIN_PORT=$TEST_ADMIN_CONNECTION_LOGS_ADMIN
9+
LOGS_URL="http://localhost:${ADMIN_PORT}/logs"
10+
INFO_URL="http://localhost:${ADMIN_PORT}/info"
11+
12+
if [[ ! -x "$BINARY" ]]; then
13+
echo "ERROR: binary not found or not executable: $BINARY" >&2
14+
exit 1
15+
fi
16+
17+
TMPDIR=$(mktemp -d)
18+
MOQX_PID=""
19+
cleanup() {
20+
if [[ -n "${MOQX_PID:-}" ]]; then
21+
kill "$MOQX_PID" 2>/dev/null || true
22+
wait "$MOQX_PID" 2>/dev/null || true
23+
fi
24+
rm -rf "$TMPDIR"
25+
}
26+
trap cleanup EXIT
27+
28+
# Set up fake log directories
29+
mkdir -p "$TMPDIR/mlog"
30+
mkdir -p "$TMPDIR/qlog"
31+
32+
# Add logging config to the test config
33+
"$(dirname "$0")/make_test_config.sh" "$LISTEN_PORT" "$ADMIN_PORT" > "$TMPDIR/config.yaml"
34+
cat <<EOF >> "$TMPDIR/config.yaml"
35+
logging:
36+
mlog:
37+
dir: "$TMPDIR/mlog"
38+
qlog:
39+
dir: "$TMPDIR/qlog"
40+
EOF
41+
42+
# Create a fake mlog file
43+
echo '{"fake":"mlog"}' > "$TMPDIR/mlog/abcdef123456.mlog"
44+
45+
# Start moqx with the generated config in the background.
46+
"$BINARY" --config="$TMPDIR/config.yaml" &
47+
MOQX_PID=$!
48+
49+
# Wait for readiness
50+
for i in $(seq 1 100); do
51+
HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "$INFO_URL" 2>/dev/null || echo "000")
52+
if [[ "$HTTP_CODE" == "200" ]]; then
53+
break
54+
fi
55+
sleep 0.1
56+
if [[ $i -eq 100 ]]; then
57+
echo "ERROR: admin /info endpoint did not become ready in time" >&2
58+
exit 1
59+
fi
60+
done
61+
62+
echo "Running tests..."
63+
64+
# Test 1: Missing params returns 400
65+
HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}" 2>/dev/null || true)
66+
if [[ "$HTTP_CODE" != "400" ]]; then
67+
echo "FAIL: expected HTTP 400 for missing params, got $HTTP_CODE" >&2
68+
exit 1
69+
fi
70+
71+
# Test 2: Invalid type returns 400
72+
HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}?type=invalid&connection_id=123" 2>/dev/null || true)
73+
if [[ "$HTTP_CODE" != "400" ]]; then
74+
echo "FAIL: expected HTTP 400 for invalid type, got $HTTP_CODE" >&2
75+
exit 1
76+
fi
77+
78+
# Test 3: Valid type but missing file returns 404
79+
HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}?type=mlog&connection_id=abcd" 2>/dev/null || true)
80+
if [[ "$HTTP_CODE" != "404" ]]; then
81+
echo "FAIL: expected HTTP 404 for missing file, got $HTTP_CODE" >&2
82+
exit 1
83+
fi
84+
85+
# Test 4: Valid file returns 200 and correct content
86+
HEADERS_FILE=$(mktemp)
87+
trap 'rm -f "$HEADERS_FILE"' RETURN
88+
HTTP_CODE=$(curl -sw "%{http_code}" -D "$HEADERS_FILE" -o /tmp/logs_response.txt "${LOGS_URL}?type=mlog&connection_id=abcdef123456" 2>/dev/null || true)
89+
90+
if [[ "$HTTP_CODE" != "200" ]]; then
91+
echo "FAIL: expected HTTP 200 for existing mlog, got $HTTP_CODE" >&2
92+
exit 1
93+
fi
94+
95+
HEADERS=$(cat "$HEADERS_FILE")
96+
RESPONSE=$(cat /tmp/logs_response.txt)
97+
rm -f /tmp/logs_response.txt
98+
99+
if ! grep -qi 'content-type:.*application/json' <<<"$HEADERS"; then
100+
echo "FAIL: expected application/json content type" >&2
101+
echo "Got headers: $HEADERS" >&2
102+
exit 1
103+
fi
104+
105+
if ! grep -q '{"fake":"mlog"}' <<<"$RESPONSE"; then
106+
echo "FAIL: response body did not match expected mlog content" >&2
107+
exit 1
108+
fi
109+
110+
echo "PASS"

0 commit comments

Comments
 (0)