From ba61dfd69504ec6263a9dee9931d93adeb6f3142 Mon Sep 17 00:00:00 2001 From: Rutger Broekhoff Date: Mon, 7 Jul 2025 21:52:08 +0200 Subject: Initialize repository --- test/test_mininix.ml | 319 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 test/test_mininix.ml (limited to 'test/test_mininix.ml') diff --git a/test/test_mininix.ml b/test/test_mininix.ml new file mode 100644 index 0000000..f628a8a --- /dev/null +++ b/test/test_mininix.ml @@ -0,0 +1,319 @@ +open Core + +let with_dir path ~f = + let fd = Core_unix.opendir path in + f fd; + Core_unix.closedir fd + +let walk_dir path ~f = + with_dir path ~f:(fun fd -> + let rec go () = + match Core_unix.readdir_opt fd with + | Some entry -> + f (Filename.concat path entry); + go () + | None -> () + in + go ()) + +type testcase = { + name : string; + dir : string; + input : string; + expected_output : [ `Okay of string | `Fail ]; +} + +let testdata_dir = "./testdata" +and testcases = ref [] +and testcases_ignored = ref 0 + +let add_testcase c = testcases := c :: !testcases + +let print_testcase_stats () = + let okay, fail = + List.fold !testcases ~init:(0, 0) + ~f:(fun (okay, fail) { expected_output; _ } -> + match expected_output with + | `Okay _ -> (okay + 1, fail) + | `Fail -> (okay, fail + 1)) + in + printf + "Loaded %d test cases (ignored %d), expected results: okay %d, fail %d\n%!" + (okay + fail) !testcases_ignored okay fail + +let imports () = + Mininix.Import.materialize + [ { filename = "./testdata/lib.nix"; deps = [] } ] + ~relative_to:(Core_unix.getcwd ()) + +type eval_err = [ `Timeout | `ParseError | `ProgramError | `ElaborateError ] +[@@deriving sexp] + +type eval_result = (string, eval_err) Result.t [@@deriving sexp] + +let eval input ~name ~dir ~imports = + let dir = Filename.to_absolute_exn dir ~relative_to:(Core_unix.getcwd ()) in + try + input + |> Nix.parse ~filename:(name ^ ".nix") + |> Nix.elaborate ~dir:(Some dir) + |> Mininix.Nix2mininix.from_nix |> Mininix.apply_prelude + |> Mininix.interp_tl ~fuel:`Limited ~mode:`Deep ~imports + |> function + | Res (Some v) -> + Ok (v |> Mininix.Mininix2nix.from_val |> Nix.Printer.to_string) + | Res None -> Error `ProgramError + | NoFuel -> Error `Timeout + with + | Nix.ParseError _ -> Error `ParseError + | Nix.ElaborateError _ -> Error `ElaborateError + | Mininix.Nix2mininix.FromNixError _ -> Error `ElaborateError + +let eval_subproc input ~name ~dir ~imports = + let rxfd, txfd = Core_unix.pipe () in + match Core_unix.fork () with + | `In_the_child -> + let txc = Core_unix.out_channel_of_descr txfd in + eval input ~name ~dir ~imports + |> [%sexp_of: eval_result] |> Sexp.output txc; + exit 0 + | `In_the_parent child_pid -> + let select_res = + Core_unix.select ~restart:true ~read:[ rxfd ] ~write:[] ~except:[] + ~timeout:(`After (Time_ns.Span.of_min 1.)) + () + in + if List.is_empty select_res.read then ( + ignore (Signal_unix.send Signal.kill (`Pid child_pid)); + ignore (Core_unix.waitpid child_pid); + Error `Timeout) + else + let rxc = Core_unix.in_channel_of_descr rxfd in + let res = Sexp.input_sexp rxc |> [%of_sexp: eval_result] in + ignore (Core_unix.waitpid child_pid); + Core_unix.close ~restart:true rxfd; + Core_unix.close ~restart:true txfd; + res + +type test_result = + [ `Timeout + | `ParseError + | `ProgramError + | `ElaborateError + | `WrongOutput + | `UnexpectedSuccess + | `Okay ] + +let run_testcase ~imports = function + | { name; dir; input; expected_output = `Okay expected_output } -> ( + match eval_subproc input ~name ~dir ~imports with + | Ok got_output -> + if String.(strip got_output = strip expected_output) then `Okay + else `WrongOutput + | Error err -> (err :> test_result)) + | { name; dir; input; expected_output = `Fail } -> ( + match eval_subproc input ~name ~dir ~imports with + | Ok _ -> `UnexpectedSuccess + | Error _ -> `Okay) + +type test_stats = { + okay : int; + unexpected_success : int; + wrong_output : int; + parse_error : int; + elaborate_error : int; + program_error : int; + timeout : int; +} + +let test_stats_empty = + { + okay = 0; + unexpected_success = 0; + wrong_output = 0; + parse_error = 0; + elaborate_error = 0; + program_error = 0; + timeout = 0; + } + +let run_testcases () = + Nix.Printer.set_width 1000000; + let mat_imports = imports () in + let stats = + List.foldi !testcases ~init:test_stats_empty ~f:(fun i stats c -> + printf "[%d/%d] %s %!" (i + 1) (List.length !testcases) c.name; + match run_testcase c ~imports:mat_imports with + | `Okay -> + printf "okay\n%!"; + { stats with okay = stats.okay + 1 } + | `UnexpectedSuccess -> + printf "unexpectedly succeeded\n%!"; + { stats with unexpected_success = stats.unexpected_success + 1 } + | `WrongOutput -> + printf "gave wrong output\n%!"; + { stats with wrong_output = stats.wrong_output + 1 } + | `ParseError -> + printf "could not be parsed\n%!"; + { stats with parse_error = stats.parse_error + 1 } + | `ElaborateError -> + printf "could not be elaborated\n%!"; + { stats with elaborate_error = stats.elaborate_error + 1 } + | `ProgramError -> + printf "failed to execute\n%!"; + { stats with program_error = stats.program_error + 1 } + | `Timeout -> + printf "timed out\n%!"; + { stats with timeout = stats.timeout + 1 }) + in + printf + "Results:\n\ + \ %d gave the expected output\n\ + \ %d unexpectedly succeeded\n\ + \ %d gave wrong output\n\ + \ %d could not be parsed\n\ + \ %d could not be elaborated\n\ + \ %d failed to execute\n\ + \ %d timed out\n\ + %!" + stats.okay stats.unexpected_success stats.wrong_output stats.parse_error + stats.elaborate_error stats.program_error stats.timeout + +let try_add_testcase without_ext = + try + let dir = Filename.dirname without_ext in + let input = In_channel.read_all (without_ext ^ ".nix") in + let name = Filename.basename without_ext in + if String.is_prefix ~prefix:"eval-fail" name then + add_testcase { name; dir; input; expected_output = `Fail } + else if String.is_prefix ~prefix:"eval-okay" name then + let expected_output = In_channel.read_all (without_ext ^ ".exp") in + add_testcase { name; dir; input; expected_output = `Okay expected_output } + with + (* There are certain test cases where the '.nix' file is available, but + there is no '.exp' file. (Instead, for example, there may be a + '.exp-disabled' file, which we don't check for.) So [add_testcase] fails + when trying to read the '.exp' file, which does not exist. We catch the + exception that is then raised in [add_testcase] here. *) + | Sys_error _ -> + () + +let ignore_tests = + [ + (* We do not implement '«repeated»' *) + "eval-okay-repeated-empty-attrs"; + "eval-okay-repeated-empty-list"; + (* # Very specific / hard-to-implement builtins: *) + (* We do not implement conversion from/to JSON/XML *) + "eval-okay-toxml"; + "eval-okay-toxml2"; + "eval-okay-tojson"; + "eval-okay-fromTOML"; + "eval-okay-fromTOML-timestamps"; + "eval-okay-fromjson"; + "eval-okay-fromjson-escapes"; + "eval-fail-fromJSON-overflowing"; + "eval-fail-fromTOML-timestamps"; + "eval-fail-toJSON"; + (* We do not implement hasing *) + "eval-okay-convertHash"; + "eval-okay-hashstring"; + "eval-okay-hashfile"; + "eval-okay-groupBy"; + "eval-okay-zipAttrsWith"; + "eval-fail-hashfile-missing"; + (* We do not support filesystem operations *) + "eval-okay-readDir"; + "eval-okay-readfile"; + "eval-okay-readFileType"; + "eval-okay-symlink-resolution"; + (* We do not support version operations *) + "eval-okay-splitversion"; + "eval-okay-versions"; + (* We do not support flake references *) + "eval-okay-parse-flake-ref"; + "eval-okay-flake-ref-to-string"; + "eval-fail-flake-ref-to-string-negative-integer"; + (* We do not support regexes *) + "eval-okay-regex-match"; + "eval-okay-regex-split"; + (* # Features that the core interpreter lacks *) + (* We do not implement derivations and contexts *) + "eval-okay-derivation-legacy"; + "eval-okay-eq-derivations"; + "eval-fail-addDrvOutputDependencies-empty-context"; + "eval-fail-addDrvOutputDependencies-multi-elem-context"; + "eval-fail-addDrvOutputDependencies-wrong-element-kind"; + "eval-fail-assert-equal-derivations"; + "eval-fail-assert-equal-derivations-extra"; + "eval-fail-derivation-name"; + "eval-okay-context"; + "eval-okay-context-introspection"; + "eval-okay-substring-context"; + "eval-fail-addErrorContext-example"; + (* We do not support scopedImport *) + "eval-okay-import"; + (* We do not support tryEval *) + "eval-okay-redefine-builtin"; + "eval-okay-tryeval"; + (* We do not support unsafeGetAttrPos nor __curPos *) + "eval-okay-curpos"; + "eval-okay-getattrpos"; + "eval-okay-getattrpos-functionargs"; + "eval-okay-getattrpos-undefined"; + "eval-okay-inherit-attr-pos"; + (* We do not support environment variable lookup *) + "eval-okay-getenv"; + (* We do not support '__override's. Rationale: this construct has expressly + been avoided in Nixpkgs since the 13.10 release, see + https://github.com/NixOS/nixpkgs/issues/2112 *) + "eval-okay-attrs6"; + "eval-okay-overrides"; + "eval-fail-set-override"; + (* We do not implement the 'trace' builtin *) + "eval-okay-print"; + "eval-okay-inherit-from"; + (* ^ also uses __overrides, for which we lack support *) + (* We do not implement flags to set arguments / retrieve attributes + for the evaluator *) + (* We do not support setting variables outside of the program *) + "eval-okay-autoargs"; + (* We do not support paths *) + "eval-okay-baseNameOf"; + "eval-okay-path"; + "eval-okay-path-string-interpolation"; + "eval-okay-pathexists"; + "eval-okay-search-path"; + "eval-okay-string"; + "eval-okay-types"; + "eval-fail-assert-equal-paths"; + "eval-fail-bad-string-interpolation-2"; + "eval-fail-nonexist-path"; + "eval-fail-path-slash"; + "eval-fail-to-path"; + (* We do not implement the 'currentSystem' and 'dirOf' builtins *) + "eval-okay-builtins"; + (* We do not support fetch operations *) + "eval-fail-fetchTree-negative"; + "eval-fail-fetchurl-baseName"; + "eval-fail-fetchurl-baseName-attrs"; + "eval-fail-fetchurl-baseName-attrs-name"; + (* We do not support the pipe operator *) + "eval-fail-pipe-operators"; + ] + +let () = + Printf.printf "Running in %s\n%!" (Core_unix.getcwd ()); + walk_dir testdata_dir ~f:(fun entry -> + match Filename.split_extension entry with + | without_ext, Some "nix" -> + if + List.exists ignore_tests ~f:(fun name -> + String.(name = Filename.basename without_ext)) + then testcases_ignored := !testcases_ignored + 1 + else try_add_testcase without_ext + | _ -> ()); + testcases := + List.sort !testcases ~compare:(fun c1 c2 -> String.compare c1.name c2.name); + print_testcase_stats (); + run_testcases () -- cgit v1.2.3