commit ae86651cd3c0bfb15f5475109bb0b6155cd6a12d
parent 745bc554fe070d08001219e7c24b052672fe7acd
Author: Erik Loualiche <eloualic@umn.edu>
Date: Sun, 15 Feb 2026 22:54:24 -0600
fix custom logger bugs, clean up imports, loosen compat bounds
CustomLogger fixes:
- Fix module_specific_message_filter using wrong filter function
(was identical to absolute filter, making filtered_modules_specific a no-op)
- Fix isdir() called on Vector{String} instead of individual dirs
- Fix get_log_filenames hardcoding repeat length to 4
- Fix reduce(&, []) crash when no module filters provided
- Merge duplicate create_absolute_filter/create_specific_filter into
single create_module_filter
- Rename msg_to_singline -> msg_to_singleline (typo)
- Remove commented-out old demux_logger block
- Rewrite docstring with accurate params and example
Other:
- Remove unused imports (AbstractLogger, FileLogger, TransformerLogger,
ConsoleLogger)
- Loosen compat bounds to semver ranges, drop stdlib compat entries
- Remove duplicate deploydocs call and hardcoded version in docs/make.jl
- Update README, index.md, logger_guide.md, internals.md
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Diffstat:
8 files changed, 98 insertions(+), 128 deletions(-)
diff --git a/Project.toml b/Project.toml
@@ -12,13 +12,11 @@ LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
[compat]
-CodecZlib = "0.7.8"
-Dates = "1.11.0"
-JSON3 = "1.14.3"
-Logging = "1.11.0"
-LoggingExtras = "1.1.0"
-Tables = "1.12.1"
-julia = "1.6.7"
+CodecZlib = "0.7"
+JSON3 = "1.14"
+LoggingExtras = "1"
+Tables = "1.12"
+julia = "1.10"
[extras]
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
diff --git a/README.md b/README.md
@@ -11,10 +11,10 @@
It is a more mature version of [`Prototypes.jl`](https://github.com/louloulibs/Prototypes.jl) where I try a bunch of things out (there is overlap).
-So far the package provides a two sets of functions:
+The package provides:
- - [`custom_logger`](#custom-logging) is a custom logging output that builds on the standard julia logger
- - [`read_jsonl`](#json-lines) provides utilities to read and write json-lines files
+ - [`custom_logger`](#custom-logging): configurable logging with per-level file output, module filtering, and multiple format options (`pretty`, `log4j`, `syslog`)
+ - ~~`read_jsonl` / `stream_jsonl` / `write_jsonl`~~: **deprecated** — use [`JSON.jl`](https://github.com/JuliaIO/JSON.jl) v1 with `jsonlines=true` instead
## Installation
@@ -63,9 +63,15 @@ custom_logger(
```
-### JSON Lines
+### JSON Lines (deprecated)
-A easy way to read json lines files into julia leaning on `JSON3` reader.
+The JSONL functions (`read_jsonl`, `stream_jsonl`, `write_jsonl`) are deprecated.
+Use [`JSON.jl`](https://github.com/JuliaIO/JSON.jl) v1 instead:
+```julia
+using JSON
+data = JSON.parse("data.jsonl"; jsonlines=true) # read
+JSON.json("out.jsonl", data; jsonlines=true) # write
+```
## Other stuff
diff --git a/docs/make.jl b/docs/make.jl
@@ -41,21 +41,9 @@ makedocs(
)
-deploydocs(
- repo="github.com/LouLouLibs/BazerUtils.jl",
- target = "build",
-)
-
deploydocs(;
repo="github.com/LouLouLibs/BazerUtils.jl",
target = "build",
branch = "gh-pages",
- devbranch = "main", # or "master"
- versions = [
- "stable" => "0.8.2",
- "dev" => "dev",
- ],
+ devbranch = "main",
)
-
-
-# --------------------------------------------------------------------------------------------------
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -1,4 +1,22 @@
# BazerUtils.jl
-Utility functions for everyday julia.
+Utility functions for everyday Julia.
+
+## Features
+
+- **[Custom Logger](@ref Logging)**: Configurable logging with per-level file output, module filtering, and multiple format options (`pretty`, `log4j`, `syslog`).
+- **[JSON Lines](@ref "Working with JSON Lines Files")** *(deprecated)*: Read/write JSONL files. Use [`JSON.jl`](https://github.com/JuliaIO/JSON.jl) v1 with `jsonlines=true` instead.
+
+## Installation
+
+```julia
+using Pkg
+pkg"registry add https://github.com/LouLouLibs/loulouJL.git"
+Pkg.add("BazerUtils")
+```
+
+Or directly from GitHub:
+```julia
+Pkg.add(url="https://github.com/LouLouLibs/BazerUtils.jl")
+```
diff --git a/docs/src/lib/internals.md b/docs/src/lib/internals.md
@@ -5,4 +5,5 @@
```@autodocs
Modules = [BazerUtils]
Public = false
+Order = [:function, :type]
```
diff --git a/docs/src/man/logger_guide.md b/docs/src/man/logger_guide.md
@@ -106,16 +106,10 @@ You can use the same logger option with the `overwrite=false` option:
## Other
-For `log4j` I do modify the message string to fit on one line.
-You will find that the "\n" is now replaced by " | "; I guess I could have an option for which character delimitates lines, but this seems too fussy.
-
-I am trying to have a path shortener that would allow to reduce the path of the function to a fixed size.
-The cost is that path will no longer be "clickable" but we would keep things tidy as messages will all start at the same column.
-(see the `shorten_path_str` function).
-
-
-
+For `log4j` the message string is modified to fit on one line: `\n` is replaced by ` | `.
+There is also a path shortener (`shorten_path_str`) that reduces file paths to a fixed size.
+The cost is that paths will no longer be clickable, but log messages will start at the same column.
diff --git a/src/BazerUtils.jl b/src/BazerUtils.jl
@@ -3,9 +3,8 @@ module BazerUtils
# --------------------------------------------------------------------------------------------------
import Dates: format, now, Dates, ISODateTimeFormat
-import Logging: global_logger, Logging, Logging.Debug, Logging.Info, Logging.Warn, AbstractLogger
-import LoggingExtras: ConsoleLogger, EarlyFilteredLogger, FileLogger, FormatLogger,
- MinLevelLogger, TeeLogger, TransformerLogger
+import Logging: global_logger, Logging, Logging.Debug, Logging.Info, Logging.Warn
+import LoggingExtras: EarlyFilteredLogger, FormatLogger, MinLevelLogger, TeeLogger
import JSON3: JSON3
import Tables: Tables
import CodecZlib: CodecZlib
diff --git a/src/CustomLogger.jl b/src/CustomLogger.jl
@@ -25,7 +25,7 @@ function get_log_filenames(filename::AbstractString;
# files = ["$(filename)_error.log", "$(filename)_warn.log",
# "$(filename)_info.log", "$(filename)_debug.log"]
else
- files = repeat([filename], 4)
+ files = repeat([filename], length(file_loggers))
end
return files
end
@@ -72,29 +72,35 @@ end
"""
custom_logger(filename; kw...)
+Set up a custom global logger with per-level file output, module filtering, and configurable formatting.
+
+When `create_log_files=true`, creates one log file per level (e.g. `filename_error.log`, `filename_warn.log`, etc.).
+Otherwise all levels write to the same file.
+
# Arguments
- `filename::AbstractString`: base name for the log files
-- `output_dir::AbstractString=./log/`: name of directory where log files are written
-- `filtered_modules_specific::Vector{Symbol}=nothing`: which modules do you want to filter out of logging (only for info and stdout)
- Some packages just write too much log ... filter them out but still be able to check them out in other logs
-- `filtered_modules_all::Vector{Symbol}=nothing`: which modules do you want to filter out of logging (across all logs)
- Examples could be TranscodingStreams (noticed that it writes so much to logs that it sometimes slows down I/O)
-- `file_loggers::Union{Symbol, Vector{Symbol}}=[:error, :warn, :info, :debug]`: which file logger to register
-- `log_date_format::AbstractString="yyyy-mm-dd"`: time stamp format at beginning of each logged lines for dates
-- `log_time_format::AbstractString="HH:MM:SS"`: time stamp format at beginning of each logged lines for times
-- `displaysize::Tuple{Int,Int}=(50,100)`: how much to show on log (same for all logs for now!)
-- `log_format::Symbol=:log4j`: how to format the log files; I have added an option for pretty (all or nothing for now)
-- `log_format_stdout::Symbol=:pretty`: how to format the stdout; default is pretty
-- `overwrite::Bool=false`: do we overwrite previously created log files
-
-The custom_logger function creates four files in `output_dir` for four different levels of logging:
- from least to most verbose: `filename.info.log.jl`, `filename.warn.log.jl`, `filename.debug.log.jl`, `filename.full.log.jl`
-The debug logging offers the option to filter messages from specific packages (some packages are particularly verbose) using the `filter` optional argument
-The full logging gets all of the debug without any of the filters.
-Info and warn log the standard info and warning level logging messages.
-
-Note that the default **overwrites** old log files (specify overwrite=false to avoid this).
-
+- `filtered_modules_specific::Union{Nothing, Vector{Symbol}}=nothing`: modules to filter out of stdout and info-level file logs only (e.g. `[:TranscodingStreams]`)
+- `filtered_modules_all::Union{Nothing, Vector{Symbol}}=nothing`: modules to filter out of all logs (e.g. `[:HTTP]`)
+- `file_loggers::Union{Symbol, Vector{Symbol}}=[:error, :warn, :info, :debug]`: which file loggers to register
+- `log_date_format::AbstractString="yyyy-mm-dd"`: date format in log timestamps
+- `log_time_format::AbstractString="HH:MM:SS"`: time format in log timestamps
+- `displaysize::Tuple{Int,Int}=(50,100)`: display size for non-string log messages
+- `log_format::Symbol=:log4j`: format for file logs (`:log4j`, `:pretty`, or `:syslog`)
+- `log_format_stdout::Symbol=:pretty`: format for stdout
+- `shorten_path::Symbol=:relative_path`: path shortening strategy for log4j format
+- `create_log_files::Bool=false`: create separate files per log level
+- `overwrite::Bool=false`: overwrite existing log files
+- `create_dir::Bool=false`: create the log directory if it doesn't exist
+- `verbose::Bool=false`: warn about filtering non-imported modules
+
+# Example
+```julia
+custom_logger("/tmp/myapp";
+ filtered_modules_all=[:HTTP, :TranscodingStreams],
+ create_log_files=true,
+ overwrite=true,
+ log_format=:log4j)
+```
"""
function custom_logger(
sink::LogSink;
@@ -112,52 +118,34 @@ function custom_logger(
# warning if some non imported get filtered ...
imported_modules = filter((x) -> typeof(getfield(Main, x)) <: Module && x ≠ :Main,
names(Main, imported=true))
- all_filters = filter(x->!isnothing(x), unique([filtered_modules_specific; filtered_modules_all]))
- catch_nonimported = map(x -> x ∈ imported_modules, all_filters)
- if !(reduce(&, catch_nonimported)) && verbose
- @warn "Some non (directly) imported modules are being filtered ... $(join(string.(all_filters[.!catch_nonimported]), ", "))"
- end
-
- # Filter functions
- function create_absolute_filter(modules)
- return function(log)
- if isnothing(modules)
- return true
- else
- module_name = string(log._module)
- # Check if the module name starts with any of the filtered module names
- # some modules did not get filtered because of submodules...
- # Note: we might catch too many modules here so keep it in mind if something does not show up in log
- for m in modules
- if startswith(module_name, string(m))
- return false # Filter out if matches
- end
- end
- return true # Keep if no matches
- end
+ all_filters = Symbol[x for x in unique(vcat(
+ something(filtered_modules_specific, Symbol[]),
+ something(filtered_modules_all, Symbol[]))) if !isnothing(x)]
+ if !isempty(all_filters) && verbose
+ catch_nonimported = map(x -> x ∈ imported_modules, all_filters)
+ if !all(catch_nonimported)
+ @warn "Some non (directly) imported modules are being filtered ... $(join(string.(all_filters[.!catch_nonimported]), ", "))"
end
end
- module_absolute_message_filter = create_absolute_filter(filtered_modules_all)
- function create_specific_filter(modules)
+ # Create a log filter that drops messages from the given modules.
+ # Uses startswith to also catch submodules (e.g. :HTTP catches HTTP.ConnectionPool).
+ function create_module_filter(modules)
return function(log)
if isnothing(modules)
return true
- else
- module_name = string(log._module)
- # Check if the module name starts with any of the filtered module names
- # some modules did not get filtered because of submodules...
- # Note: we might catch too many modules here so keep it in mind if something does not show up in log
- for m in modules
- if startswith(module_name, string(m))
- return false # Filter out if matches
- end
+ end
+ module_name = string(log._module)
+ for m in modules
+ if startswith(module_name, string(m))
+ return false
end
- return true # Keep if no matches
end
+ return true
end
end
- module_specific_message_filter = create_absolute_filter(all_filters)
+ module_absolute_message_filter = create_module_filter(filtered_modules_all)
+ module_specific_message_filter = create_module_filter(filtered_modules_specific)
format_log_stdout = (io,log_record)->custom_format(io, log_record;
@@ -173,29 +161,6 @@ function custom_logger(
log_format=log_format,
shorten_path=shorten_path)
- # Create demux_logger using sink's IO streams
- # demux_logger = TeeLogger(
- # MinLevelLogger(
- # EarlyFilteredLogger(module_absolute_message_filter, # error
- # FormatLogger(format_log_file, sink.ios[1])),
- # Logging.Error),
- # MinLevelLogger(
- # EarlyFilteredLogger(module_absolute_message_filter, # warn
- # FormatLogger(format_log_file, sink.ios[2])),
- # Logging.Warn),
- # MinLevelLogger(
- # EarlyFilteredLogger(module_specific_message_filter, # info
- # FormatLogger(format_log_file, sink.ios[3])),
- # Logging.Info),
- # MinLevelLogger(
- # EarlyFilteredLogger(module_absolute_message_filter, # debug
- # FormatLogger(format_log_file, sink.ios[4])),
- # Logging.Debug),
- # MinLevelLogger(
- # EarlyFilteredLogger(module_specific_message_filter, # stdout
- # FormatLogger(format_log_stdout, stdout)),
- # Logging.Info)
- # )
demux_logger = create_demux_logger(sink, file_loggers,
module_absolute_message_filter, module_specific_message_filter, format_log_file, format_log_stdout)
@@ -226,12 +191,13 @@ function custom_logger(
# create directory if needed and bool true
# returns an error if directory does not exist and bool false
- log_dir = unique(dirname.(files))
- if create_dir && !isdir(log_dir)
- @warn "Creating directory for logs ... $(join(log_dir, ", "))"
- mkpath.(log_dir)
- elseif !isdir(log_dir)
- @error "Directory for logs does not exist ... $(join(log_dir, ", "))"
+ log_dirs = unique(dirname.(files))
+ missing_dirs = filter(d -> !isempty(d) && !isdir(d), log_dirs)
+ if create_dir && !isempty(missing_dirs)
+ @warn "Creating directory for logs ... $(join(missing_dirs, ", "))"
+ mkpath.(missing_dirs)
+ elseif !isempty(missing_dirs)
+ @error "Directory for logs does not exist ... $(join(missing_dirs, ", "))"
end
# Handle cleanup if needed
overwrite && foreach(f -> rm(f, force=true), files)
@@ -347,10 +313,10 @@ function custom_format(io, log_record::NamedTuple;
elseif log_format == :log4j
log_entry = log_record |>
- str -> format_log4j(str, shorten_path=shorten_path) |> msg_to_singline
+ str -> format_log4j(str, shorten_path=shorten_path) |> msg_to_singleline
println(io, log_entry)
elseif log_format == :syslog
- log_entry = log_record |> format_syslog |> msg_to_singline
+ log_entry = log_record |> format_syslog |> msg_to_singleline
println(io, log_entry)
end
@@ -384,7 +350,7 @@ function reformat_msg(log_record;
end
-function msg_to_singline(message::AbstractString)::AbstractString
+function msg_to_singleline(message::AbstractString)::AbstractString
message |>
str -> replace(str, r"\"\"\"[\r\n\s]*(.+?)[\r\n\s]*\"\"\""s => s"\1") |>
str -> replace(str, r"\n\s*" => " | ") |>
