commit bd0ee380695dbf0cb76c2ec4426fe946fc2890af
Author: Erik Loualiche <eloualic@umn.edu>
Date: Fri, 6 Feb 2026 09:12:41 -0600
Implement FFI for Nickel evaluation in Rust
Add native FFI bindings to evaluate Nickel code directly without spawning
a subprocess. The Rust wrapper uses nickel-lang-core to parse and evaluate
Nickel code, returning JSON output.
Components:
- rust/nickel-jl: C-compatible dynamic library with nickel_eval_string,
nickel_get_error, and nickel_free_string functions
- src/ffi.jl: Julia FFI bindings using ccall
- src/subprocess.jl: Fallback subprocess-based evaluation
- deps/build.jl: Build script for compiling the Rust library
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Diffstat:
11 files changed, 717 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,22 @@
+# Julia
+Manifest.toml
+
+# Rust build artifacts
+rust/*/target/
+rust/**/Cargo.lock
+
+# Compiled FFI libraries (these should be built locally)
+deps/*.dylib
+deps/*.so
+deps/*.dll
+
+# Editor
+*.swp
+*.swo
+*~
+.idea/
+.vscode/
+
+# OS
+.DS_Store
+Thumbs.db
diff --git a/Project.toml b/Project.toml
@@ -0,0 +1,17 @@
+name = "NickelEval"
+uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"
+authors = ["NickelJL Contributors"]
+version = "0.1.0"
+
+[deps]
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+
+[compat]
+JSON3 = "1"
+julia = "1.6"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]
diff --git a/deps/build.jl b/deps/build.jl
@@ -0,0 +1,73 @@
+# Build script for the Rust FFI library
+#
+# This script is run by Pkg.build() to compile the Rust wrapper library.
+# Currently a stub for Phase 2 implementation.
+
+using Libdl
+
+const RUST_PROJECT = joinpath(@__DIR__, "..", "rust", "nickel-jl")
+
+# Determine the correct library extension for the platform
+function library_extension()
+ if Sys.iswindows()
+ return ".dll"
+ elseif Sys.isapple()
+ return ".dylib"
+ else
+ return ".so"
+ end
+end
+
+# Determine library name with platform-specific prefix
+function library_name()
+ if Sys.iswindows()
+ return "nickel_jl$(library_extension())"
+ else
+ return "libnickel_jl$(library_extension())"
+ end
+end
+
+function build_rust_library()
+ if !isdir(RUST_PROJECT)
+ @warn "Rust project not found at $RUST_PROJECT, skipping FFI build"
+ return false
+ end
+
+ # Check if cargo is available
+ cargo = Sys.which("cargo")
+ if cargo === nothing
+ @warn "Cargo not found in PATH, skipping FFI build. Install Rust: https://rustup.rs/"
+ return false
+ end
+
+ @info "Building Rust FFI library..."
+
+ try
+ cd(RUST_PROJECT) do
+ run(`cargo build --release`)
+ end
+
+ # Copy the built library to deps/
+ src_lib = joinpath(RUST_PROJECT, "target", "release", library_name())
+ dst_lib = joinpath(@__DIR__, library_name())
+
+ if isfile(src_lib)
+ cp(src_lib, dst_lib; force=true)
+ @info "FFI library built successfully: $dst_lib"
+ return true
+ else
+ @warn "Built library not found at $src_lib"
+ return false
+ end
+ catch e
+ @warn "Failed to build Rust library: $e"
+ return false
+ end
+end
+
+# Only build if explicitly requested or in a CI environment
+if get(ENV, "NICKELEVAL_BUILD_FFI", "false") == "true"
+ build_rust_library()
+else
+ @info "Skipping FFI build (set NICKELEVAL_BUILD_FFI=true to enable)"
+end
diff --git a/rust/nickel-jl/Cargo.toml b/rust/nickel-jl/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "nickel-jl"
+version = "0.1.0"
+edition = "2021"
+description = "C-compatible wrapper around Nickel for Julia FFI"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+nickel-lang-core = "0.9"
+serde_json = "1.0"
+
+[profile.release]
+opt-level = 3
+lto = true
diff --git a/rust/nickel-jl/src/lib.rs b/rust/nickel-jl/src/lib.rs
@@ -0,0 +1,233 @@
+//! C-compatible wrapper around Nickel for Julia FFI
+//!
+//! This library provides C-compatible functions that can be called from Julia
+//! via ccall/FFI to evaluate Nickel code without spawning a subprocess.
+//!
+//! # Functions
+//!
+//! - `nickel_eval_string`: Evaluate Nickel code and return JSON string
+//! - `nickel_get_error`: Get the last error message
+//! - `nickel_free_string`: Free allocated string memory
+
+use std::ffi::{CStr, CString};
+use std::io::Cursor;
+use std::os::raw::c_char;
+use std::ptr;
+
+use nickel_lang_core::eval::cache::lazy::CBNCache;
+use nickel_lang_core::program::Program;
+use nickel_lang_core::serialize::{self, ExportFormat};
+
+// Thread-local storage for the last error message
+thread_local! {
+ static LAST_ERROR: std::cell::RefCell<Option<CString>> = const { std::cell::RefCell::new(None) };
+}
+
+/// Evaluate a Nickel code string and return the result as a JSON string.
+///
+/// # Safety
+/// - `code` must be a valid null-terminated C string
+/// - The returned pointer must be freed with `nickel_free_string`
+/// - Returns NULL on error; use `nickel_get_error` to retrieve error message
+#[no_mangle]
+pub unsafe extern "C" fn nickel_eval_string(code: *const c_char) -> *const c_char {
+ if code.is_null() {
+ set_error("Null pointer passed to nickel_eval_string");
+ return ptr::null();
+ }
+
+ let code_str = match CStr::from_ptr(code).to_str() {
+ Ok(s) => s,
+ Err(e) => {
+ set_error(&format!("Invalid UTF-8 in input: {}", e));
+ return ptr::null();
+ }
+ };
+
+ match eval_nickel(code_str) {
+ Ok(json) => {
+ match CString::new(json) {
+ Ok(cstr) => cstr.into_raw(),
+ Err(e) => {
+ set_error(&format!("Result contains null byte: {}", e));
+ ptr::null()
+ }
+ }
+ }
+ Err(e) => {
+ set_error(&e);
+ ptr::null()
+ }
+ }
+}
+
+/// Internal function to evaluate Nickel code and return JSON.
+fn eval_nickel(code: &str) -> Result<String, String> {
+ // Create a source from the code string
+ let source = Cursor::new(code.as_bytes());
+
+ // Create a program with a null trace (discard trace output)
+ let mut program: Program<CBNCache> = Program::new_from_source(source, "<ffi>", std::io::sink())
+ .map_err(|e| format!("Parse error: {}", e))?;
+
+ // Evaluate the program fully for export
+ let result = program
+ .eval_full_for_export()
+ .map_err(|e| program.report_as_str(e))?;
+
+ // Serialize to JSON
+ serialize::to_string(ExportFormat::Json, &result)
+ .map_err(|e| format!("Serialization error: {:?}", e))
+}
+
+/// Get the last error message.
+///
+/// # Safety
+/// - The returned pointer is valid until the next call to any nickel_* function
+/// - Do not free this pointer; it is managed internally
+#[no_mangle]
+pub unsafe extern "C" fn nickel_get_error() -> *const c_char {
+ LAST_ERROR.with(|e| {
+ e.borrow()
+ .as_ref()
+ .map(|s| s.as_ptr())
+ .unwrap_or(ptr::null())
+ })
+}
+
+/// Free a string allocated by this library.
+///
+/// # Safety
+/// - `ptr` must have been returned by `nickel_eval_string`
+/// - `ptr` must not be used after this call
+/// - Passing NULL is safe (no-op)
+#[no_mangle]
+pub unsafe extern "C" fn nickel_free_string(ptr: *const c_char) {
+ if !ptr.is_null() {
+ drop(CString::from_raw(ptr as *mut c_char));
+ }
+}
+
+fn set_error(msg: &str) {
+ LAST_ERROR.with(|e| {
+ *e.borrow_mut() = CString::new(msg).ok();
+ });
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::ffi::CString;
+
+ #[test]
+ fn test_null_input() {
+ unsafe {
+ let result = nickel_eval_string(ptr::null());
+ assert!(result.is_null());
+ let error = nickel_get_error();
+ assert!(!error.is_null());
+ }
+ }
+
+ #[test]
+ fn test_free_null() {
+ unsafe {
+ // Should not crash
+ nickel_free_string(ptr::null());
+ }
+ }
+
+ #[test]
+ fn test_eval_simple_number() {
+ unsafe {
+ let code = CString::new("1 + 2").unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(!result.is_null(), "Expected result, got error: {:?}",
+ CStr::from_ptr(nickel_get_error()).to_str());
+ let result_str = CStr::from_ptr(result).to_str().unwrap();
+ assert_eq!(result_str, "3");
+ nickel_free_string(result);
+ }
+ }
+
+ #[test]
+ fn test_eval_string() {
+ unsafe {
+ let code = CString::new(r#""hello""#).unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(!result.is_null(), "Expected result, got error: {:?}",
+ CStr::from_ptr(nickel_get_error()).to_str());
+ let result_str = CStr::from_ptr(result).to_str().unwrap();
+ assert_eq!(result_str, "\"hello\"");
+ nickel_free_string(result);
+ }
+ }
+
+ #[test]
+ fn test_eval_record() {
+ unsafe {
+ let code = CString::new("{ x = 1, y = 2 }").unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(!result.is_null(), "Expected result, got error: {:?}",
+ CStr::from_ptr(nickel_get_error()).to_str());
+ let result_str = CStr::from_ptr(result).to_str().unwrap();
+ // JSON output should have the fields
+ assert!(result_str.contains("\"x\""));
+ assert!(result_str.contains("\"y\""));
+ nickel_free_string(result);
+ }
+ }
+
+ #[test]
+ fn test_eval_array() {
+ unsafe {
+ let code = CString::new("[1, 2, 3]").unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(!result.is_null(), "Expected result, got error: {:?}",
+ CStr::from_ptr(nickel_get_error()).to_str());
+ let result_str = CStr::from_ptr(result).to_str().unwrap();
+ // JSON output is pretty-printed, so check for presence of elements
+ assert!(result_str.contains("1"));
+ assert!(result_str.contains("2"));
+ assert!(result_str.contains("3"));
+ nickel_free_string(result);
+ }
+ }
+
+ #[test]
+ fn test_eval_function_application() {
+ unsafe {
+ let code = CString::new("let add = fun x y => x + y in add 3 4").unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(!result.is_null(), "Expected result, got error: {:?}",
+ CStr::from_ptr(nickel_get_error()).to_str());
+ let result_str = CStr::from_ptr(result).to_str().unwrap();
+ assert_eq!(result_str, "7");
+ nickel_free_string(result);
+ }
+ }
+
+ #[test]
+ fn test_eval_syntax_error() {
+ unsafe {
+ let code = CString::new("{ x = }").unwrap();
+ let result = nickel_eval_string(code.as_ptr());
+ assert!(result.is_null());
+ let error = nickel_get_error();
+ assert!(!error.is_null());
+ let error_str = CStr::from_ptr(error).to_str().unwrap();
+ assert!(!error_str.is_empty());
+ }
+ }
+
+ #[test]
+ fn test_eval_internal() {
+ // Test the internal eval_nickel function directly
+ let result = eval_nickel("42").unwrap();
+ assert_eq!(result, "42");
+
+ let result = eval_nickel("{ a = 1 }").unwrap();
+ assert!(result.contains("\"a\""));
+ assert!(result.contains("1"));
+ }
+}
diff --git a/src/NickelEval.jl b/src/NickelEval.jl
@@ -0,0 +1,17 @@
+module NickelEval
+
+using JSON3
+
+export nickel_eval, nickel_eval_file, nickel_export, @ncl_str, NickelError
+
+# Custom exception for Nickel errors
+struct NickelError <: Exception
+ message::String
+end
+
+Base.showerror(io::IO, e::NickelError) = print(io, "NickelError: ", e.message)
+
+include("subprocess.jl")
+include("ffi.jl")
+
+end # module
diff --git a/src/ffi.jl b/src/ffi.jl
@@ -0,0 +1,79 @@
+# FFI bindings for Nickel
+#
+# Native FFI bindings to a Rust wrapper around Nickel for high-performance evaluation
+# without subprocess overhead.
+#
+# API:
+# - nickel_eval_ffi(code::String) -> Any
+# - Direct ccall to libnickel_jl
+# - Memory management via nickel_free_string
+#
+# Benefits over subprocess:
+# - No process spawn overhead
+# - Direct memory sharing
+# - Better performance for repeated evaluations
+
+using JSON3
+
+# Determine platform-specific library name
+const LIB_NAME = if Sys.iswindows()
+ "nickel_jl.dll"
+elseif Sys.isapple()
+ "libnickel_jl.dylib"
+else
+ "libnickel_jl.so"
+end
+
+# Path to the compiled library
+const LIB_PATH = joinpath(@__DIR__, "..", "deps", LIB_NAME)
+
+# Check if FFI library is available
+const FFI_AVAILABLE = isfile(LIB_PATH)
+
+"""
+ check_ffi_available() -> Bool
+
+Check if FFI bindings are available.
+Returns true if the native library is compiled and available.
+"""
+function check_ffi_available()
+ return FFI_AVAILABLE
+end
+
+"""
+ nickel_eval_ffi(code::String) -> Any
+
+Evaluate Nickel code using native FFI bindings.
+Returns the parsed JSON result.
+
+Throws an error if FFI is not available or if evaluation fails.
+"""
+function nickel_eval_ffi(code::String)
+ if !FFI_AVAILABLE
+ error("FFI not available. Build the Rust library with: NICKELEVAL_BUILD_FFI=true julia --project=. -e 'using Pkg; Pkg.build()'")
+ end
+
+ # Call the Rust function
+ result_ptr = ccall((:nickel_eval_string, LIB_PATH),
+ Ptr{Cchar}, (Cstring,), code)
+
+ if result_ptr == C_NULL
+ # Get error message
+ error_ptr = ccall((:nickel_get_error, LIB_PATH), Ptr{Cchar}, ())
+ if error_ptr != C_NULL
+ error_msg = unsafe_string(error_ptr)
+ error("Nickel evaluation error: $error_msg")
+ else
+ error("Nickel evaluation failed with unknown error")
+ end
+ end
+
+ # Convert result to Julia string
+ result_json = unsafe_string(result_ptr)
+
+ # Free the allocated memory
+ ccall((:nickel_free_string, LIB_PATH), Cvoid, (Ptr{Cchar},), result_ptr)
+
+ # Parse JSON and return
+ return JSON3.read(result_json)
+end
diff --git a/src/subprocess.jl b/src/subprocess.jl
@@ -0,0 +1,169 @@
+# Subprocess-based Nickel evaluation using CLI
+
+"""
+ find_nickel_executable() -> String
+
+Find the Nickel executable in PATH.
+"""
+function find_nickel_executable()
+ nickel_cmd = Sys.iswindows() ? "nickel.exe" : "nickel"
+ nickel_path = Sys.which(nickel_cmd)
+ if nickel_path === nothing
+ throw(NickelError("Nickel executable not found in PATH. Please install Nickel: https://nickel-lang.org/"))
+ end
+ return nickel_path
+end
+
+"""
+ nickel_export(code::String; format::Symbol=:json) -> String
+
+Export Nickel code to the specified format string.
+
+# Arguments
+- `code::String`: Nickel code to evaluate
+- `format::Symbol`: Output format, one of `:json`, `:yaml`, `:toml` (default: `:json`)
+
+# Returns
+- `String`: The exported content in the specified format
+
+# Throws
+- `NickelError`: If evaluation fails or format is unsupported
+"""
+function nickel_export(code::String; format::Symbol=:json)
+ valid_formats = (:json, :yaml, :toml, :raw)
+ if format ∉ valid_formats
+ throw(NickelError("Unsupported format: $format. Valid formats: $(join(valid_formats, ", "))"))
+ end
+
+ nickel_path = find_nickel_executable()
+
+ # Create a temporary file for the Nickel code
+ result = mktempdir() do tmpdir
+ ncl_file = joinpath(tmpdir, "input.ncl")
+ write(ncl_file, code)
+
+ # Build the command
+ cmd = `$nickel_path export --format=$(string(format)) $ncl_file`
+
+ # Run the command and capture output
+ stdout_buf = IOBuffer()
+ stderr_buf = IOBuffer()
+
+ try
+ proc = run(pipeline(cmd, stdout=stdout_buf, stderr=stderr_buf), wait=true)
+ return String(take!(stdout_buf))
+ catch e
+ stderr_content = String(take!(stderr_buf))
+ stdout_content = String(take!(stdout_buf))
+ error_msg = isempty(stderr_content) ? stdout_content : stderr_content
+ if isempty(error_msg)
+ error_msg = "Nickel evaluation failed with unknown error"
+ end
+ throw(NickelError(strip(error_msg)))
+ end
+ end
+
+ return result
+end
+
+"""
+ nickel_eval(code::String) -> Any
+
+Evaluate Nickel code and return a Julia value.
+
+The Nickel code is exported to JSON and parsed into Julia types:
+- Objects become `Dict{String, Any}`
+- Arrays become `Vector{Any}`
+- Numbers, strings, booleans map directly
+
+# Arguments
+- `code::String`: Nickel code to evaluate
+
+# Returns
+- `Any`: The evaluated result as a Julia value
+
+# Examples
+```julia
+julia> nickel_eval("1 + 2")
+3
+
+julia> nickel_eval("{ a = 1, b = 2 }")
+Dict{String, Any}("a" => 1, "b" => 2)
+
+julia> nickel_eval("let x = 5 in x * 2")
+10
+```
+"""
+function nickel_eval(code::String)
+ json_str = nickel_export(code; format=:json)
+ return JSON3.read(json_str)
+end
+
+"""
+ nickel_eval_file(path::String) -> Any
+
+Evaluate a Nickel file and return a Julia value.
+
+# Arguments
+- `path::String`: Path to the Nickel file
+
+# Returns
+- `Any`: The evaluated result as a Julia value
+
+# Throws
+- `NickelError`: If file doesn't exist or evaluation fails
+"""
+function nickel_eval_file(path::String)
+ if !isfile(path)
+ throw(NickelError("File not found: $path"))
+ end
+
+ nickel_path = find_nickel_executable()
+
+ cmd = `$nickel_path export --format=json $path`
+
+ stdout_buf = IOBuffer()
+ stderr_buf = IOBuffer()
+
+ try
+ run(pipeline(cmd, stdout=stdout_buf, stderr=stderr_buf), wait=true)
+ json_str = String(take!(stdout_buf))
+ return JSON3.read(json_str)
+ catch e
+ stderr_content = String(take!(stderr_buf))
+ stdout_content = String(take!(stdout_buf))
+ error_msg = isempty(stderr_content) ? stdout_content : stderr_content
+ if isempty(error_msg)
+ error_msg = "Nickel evaluation failed with unknown error"
+ end
+ throw(NickelError(strip(error_msg)))
+ end
+end
+
+"""
+ @ncl_str -> Any
+
+String macro for inline Nickel evaluation.
+
+# Examples
+```julia
+julia> ncl"1 + 2"
+3
+
+julia> ncl"{ name = \"test\", value = 42 }"
+Dict{String, Any}("name" => "test", "value" => 42)
+
+julia> ncl\"\"\"
+ let
+ x = 1,
+ y = 2
+ in x + y
+ \"\"\"
+3
+```
+"""
+macro ncl_str(code)
+ quote
+ nickel_eval($code)
+ end
+end
diff --git a/test/fixtures/simple.ncl b/test/fixtures/simple.ncl
@@ -0,0 +1,6 @@
+# Simple test fixture for NickelJL
+{
+ name = "test",
+ value = 42,
+ computed = value * 2,
+}
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -0,0 +1,20 @@
+using NickelEval
+using Test
+
+# Check if Nickel is available
+function nickel_available()
+ try
+ Sys.which("nickel") !== nothing
+ catch
+ false
+ end
+end
+
+@testset "NickelEval.jl" begin
+ if nickel_available()
+ include("test_subprocess.jl")
+ else
+ @warn "Nickel executable not found in PATH, skipping tests. Install from: https://nickel-lang.org/"
+ @test_skip "Nickel not available"
+ end
+end
diff --git a/test/test_subprocess.jl b/test/test_subprocess.jl
@@ -0,0 +1,65 @@
+@testset "Subprocess Evaluation" begin
+ @testset "Basic expressions" begin
+ @test nickel_eval("1 + 2") == 3
+ @test nickel_eval("10 - 3") == 7
+ @test nickel_eval("4 * 5") == 20
+ @test nickel_eval("true") == true
+ @test nickel_eval("false") == false
+ @test nickel_eval("\"hello\"") == "hello"
+ end
+
+ @testset "Let expressions" begin
+ @test nickel_eval("let x = 1 in x + 2") == 3
+ @test nickel_eval("let x = 5 in x * 2") == 10
+ @test nickel_eval("let x = 1 in let y = 2 in x + y") == 3
+ end
+
+ @testset "Records (Objects)" begin
+ result = nickel_eval("{ a = 1, b = 2 }")
+ @test result["a"] == 1
+ @test result["b"] == 2
+
+ result = nickel_eval("{ name = \"test\", value = 42 }")
+ @test result["name"] == "test"
+ @test result["value"] == 42
+ end
+
+ @testset "Arrays" begin
+ @test nickel_eval("[1, 2, 3]") == [1, 2, 3]
+ @test nickel_eval("[\"a\", \"b\"]") == ["a", "b"]
+ @test nickel_eval("[]") == []
+ end
+
+ @testset "Nested structures" begin
+ result = nickel_eval("{ outer = { inner = 42 } }")
+ @test result["outer"]["inner"] == 42
+
+ result = nickel_eval("{ items = [1, 2, 3] }")
+ @test result["items"] == [1, 2, 3]
+ end
+
+ @testset "String macro" begin
+ @test ncl"1 + 1" == 2
+ @test ncl"{ x = 10 }"["x"] == 10
+ end
+
+ @testset "File evaluation" begin
+ fixture_path = joinpath(@__DIR__, "fixtures", "simple.ncl")
+ result = nickel_eval_file(fixture_path)
+ @test result["name"] == "test"
+ @test result["value"] == 42
+ @test result["computed"] == 84
+ end
+
+ @testset "Export formats" begin
+ json_output = nickel_export("{ a = 1 }"; format=:json)
+ @test occursin("\"a\"", json_output)
+ @test occursin("1", json_output)
+ end
+
+ @testset "Error handling" begin
+ @test_throws NickelError nickel_eval("undefined_variable")
+ @test_throws NickelError nickel_eval_file("/nonexistent/path.ncl")
+ @test_throws NickelError nickel_export("1"; format=:invalid)
+ end
+end
