refine mincatcher

Signed-off-by: Christine Dodrill <me@christine.website>
This commit is contained in:
Cadey Ratio 2021-09-09 19:48:56 -04:00
parent 8b24578910
commit b7b3cefe2d
7 changed files with 40 additions and 26 deletions

2
Cargo.lock generated
View File

@ -724,6 +724,7 @@ dependencies = [
"structopt", "structopt",
"tokio", "tokio",
"tower", "tower",
"tower-http",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"zstd", "zstd",
@ -1612,6 +1613,7 @@ dependencies = [
"pin-project", "pin-project",
"tower-layer", "tower-layer",
"tower-service", "tower-service",
"tracing",
] ]
[[package]] [[package]]

View File

@ -1,2 +1,6 @@
[workspace] [workspace]
members = [ "cmd/*", "crates/*", "examples/*" ] members = [ "cmd/*", "crates/*", "examples/*" ]
[profile.release]
opt-level = 3
lto = true

View File

@ -43,3 +43,7 @@ features = [ "derive" ]
[dependencies.tokio] [dependencies.tokio]
version = "1" version = "1"
features = [ "full" ] features = [ "full" ]
[dependencies.tower-http]
version = "0.1"
features = [ "trace" ]

View File

@ -6,10 +6,7 @@ extern crate prometheus;
extern crate tracing; extern crate tracing;
use anyhow::Result; use anyhow::Result;
use std::{ use std::net::{IpAddr, SocketAddr};
net::{IpAddr, SocketAddr},
path::PathBuf,
};
use structopt::StructOpt; use structopt::StructOpt;
mod server; mod server;
@ -18,10 +15,6 @@ use server::*;
#[derive(Debug, StructOpt)] #[derive(Debug, StructOpt)]
#[structopt(name = "logcatcher", about = "The log catcher service")] #[structopt(name = "logcatcher", about = "The log catcher service")]
pub(crate) struct Config { pub(crate) struct Config {
/// The maximum number of bytes that can be in-ram at once.
#[structopt(short, long, default_value = "1048576000")]
pub(crate) max_bytes: usize,
/// The TCP host to listen on for HTTP traffic. /// The TCP host to listen on for HTTP traffic.
#[structopt(short, long, default_value = "127.0.0.1:3848")] #[structopt(short, long, default_value = "127.0.0.1:3848")]
pub(crate) http_host: SocketAddr, pub(crate) http_host: SocketAddr,
@ -33,10 +26,9 @@ pub(crate) struct Config {
/// The IP address/es to expect proxied requests from. /// The IP address/es to expect proxied requests from.
#[structopt(short, long)] #[structopt(short, long)]
pub(crate) proxy_ips: Option<Vec<IpAddr>>, pub(crate) proxy_ips: Option<Vec<IpAddr>>,
// /// The place to store logs to disk.
/// The place to store logs to disk. // #[structopt(short, long, default_value = "./var/log")]
#[structopt(short, long, default_value = "./var/log")] // pub(crate) log_dir: PathBuf,
pub(crate) log_dir: PathBuf,
} }
#[tokio::main] #[tokio::main]

View File

@ -8,11 +8,19 @@ use chrono::prelude::*;
use headers::{Header, HeaderName, HeaderValue}; use headers::{Header, HeaderName, HeaderValue};
use hyper::http::StatusCode; use hyper::http::StatusCode;
use logtail::PublicID; use logtail::PublicID;
use prometheus::{Encoder, IntGauge}; use prometheus::{Encoder, IntCounterVec, IntGauge};
use std::{convert::TryInto, net::SocketAddr, path::PathBuf}; use std::{convert::TryInto, net::SocketAddr};
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use tower_http::trace::TraceLayer;
const BYTE_MAX: usize = 1048576000;
lazy_static! { lazy_static! {
static ref LINES_PER_COLLECTION: IntCounterVec = register_int_counter_vec!(
opts!("lines_per_collection", "log lines per log collection"),
&["collection"]
)
.unwrap();
static ref BYTES_IN_USE: IntGauge = register_int_gauge!( static ref BYTES_IN_USE: IntGauge = register_int_gauge!(
"buffer_bytes_in_use", "buffer_bytes_in_use",
"Number of bytes currently in use by zstd buffers" "Number of bytes currently in use by zstd buffers"
@ -23,24 +31,22 @@ lazy_static! {
"Maximum number of bytes that can be in use by zstd buffers" "Maximum number of bytes that can be in use by zstd buffers"
) )
.unwrap(); .unwrap();
static ref BUF_SEM: Semaphore = Semaphore::new(1048576000); static ref BUF_SEM: Semaphore = Semaphore::new(BYTE_MAX);
static ref ORIG_CONTENT_LENGTH: HeaderName = static ref ORIG_CONTENT_LENGTH: HeaderName =
axum::http::header::HeaderName::from_static("orig-content-length"); axum::http::header::HeaderName::from_static("orig-content-length");
} }
pub struct Server { pub struct Server {
// config values // config values
log_dir: PathBuf,
http_host: SocketAddr, http_host: SocketAddr,
debug_host: SocketAddr, debug_host: SocketAddr,
} }
impl Server { impl Server {
pub(crate) fn new(cfg: super::Config) -> Result<Self> { pub(crate) fn new(cfg: super::Config) -> Result<Self> {
MAX_BUFFER_SIZE.set(cfg.max_bytes.try_into()?); MAX_BUFFER_SIZE.set(BYTE_MAX as i64);
Ok(Self { Ok(Self {
log_dir: cfg.log_dir,
http_host: cfg.http_host, http_host: cfg.http_host,
debug_host: cfg.debug_host, debug_host: cfg.debug_host,
}) })
@ -49,7 +55,9 @@ impl Server {
pub(crate) async fn run(&self) -> Result<()> { pub(crate) async fn run(&self) -> Result<()> {
tokio::spawn(metrics(self.debug_host.clone())); tokio::spawn(metrics(self.debug_host.clone()));
let router = Router::new().route("/c/:collection/:private_id", post(put_logs)); let router = Router::new()
.route("/c/:collection/:private_id", post(put_logs))
.layer(TraceLayer::new_for_http());
hyper::Server::bind(&self.http_host) hyper::Server::bind(&self.http_host)
.serve(router.into_make_service()) .serve(router.into_make_service())
@ -102,7 +110,7 @@ async fn put_logs(
error!("error acquiring buffer: {}", why); error!("error acquiring buffer: {}", why);
return StatusCode::INSUFFICIENT_STORAGE; return StatusCode::INSUFFICIENT_STORAGE;
} }
BYTES_IN_USE.set(BUF_SEM.available_permits().try_into().unwrap()); BYTES_IN_USE.set(BUF_SEM.available_permits() as i64 - BYTE_MAX as i64);
let mut decompressed_body: Vec<u8> = Vec::new(); let mut decompressed_body: Vec<u8> = Vec::new();
decompressed_body.resize(orig_size.0 as usize, 0); decompressed_body.resize(orig_size.0 as usize, 0);
@ -131,6 +139,10 @@ async fn put_logs(
}, },
) )
.unwrap(); .unwrap();
println!("");
LINES_PER_COLLECTION
.with_label_values(&[collection.clone().as_str()])
.inc();
} }
StatusCode::NO_CONTENT StatusCode::NO_CONTENT

View File

@ -189,13 +189,13 @@ pub struct Egress {
} }
impl Egress { impl Egress {
fn pull(&mut self) -> Vec<serde_json::Value> { fn pull(&self) -> Vec<serde_json::Value> {
let mut values: Vec<serde_json::Value> = vec![]; let mut values: Vec<serde_json::Value> = vec![];
loop { loop {
match self.rx.try_recv() { match self.rx.try_recv() {
Ok(val) => values.push(val), Ok(val) => values.push(val),
Err(why) => { Err(_) => {
break; break;
} }
}; };
@ -206,7 +206,7 @@ impl Egress {
/// Pushes log messages to logtail. This will push everything buffered into the /// Pushes log messages to logtail. This will push everything buffered into the
/// log server. This should be called periodically. /// log server. This should be called periodically.
pub async fn post(&mut self) -> Result<(), Error> { pub async fn post(&self) -> Result<(), Error> {
let values = self.pull(); let values = self.pull();
self.push(values).await?; self.push(values).await?;
Ok(()) Ok(())
@ -247,7 +247,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn end_to_end() { async fn end_to_end() {
let (mut ing, mut eg) = Builder::default() let (ing, mut eg) = Builder::default()
.collection("rebterlai.logtail-poster.test".to_string()) .collection("rebterlai.logtail-poster.test".to_string())
.user_agent("rebterlai/test".to_string()) .user_agent("rebterlai/test".to_string())
.base_url("http://127.0.0.1:3848".to_string()) .base_url("http://127.0.0.1:3848".to_string())

View File

@ -9,11 +9,11 @@ struct Data {
async fn main() { async fn main() {
let collection = "rebterlai.example.logtail-poster".to_string(); let collection = "rebterlai.example.logtail-poster".to_string();
let cfg = logtail_poster::Config::load(collection.clone()).unwrap(); let cfg = logtail_poster::Config::load(collection.clone()).unwrap();
let (mut ing, mut eg) = Builder::default() let (ing, eg) = Builder::default()
.collection(collection) .collection(collection)
.private_id(cfg.private_id()) .private_id(cfg.private_id())
.user_agent("rebterlai/test".to_string()) .user_agent("rebterlai/test".to_string())
.base_url("http://127.0.0.1:48283".to_string()) .base_url("http://127.0.0.1:3848".to_string())
.build() .build()
.unwrap(); .unwrap();