1 files changed, 319 insertions, 0 deletions

diff --git a/test/test_mininix.ml b/test/test_mininix.ml
new file mode 100644
index 0000000..f628a8a
--- /dev/null
+++ b/test/test_mininix.ml
@@ -0,0 +1,319 @@
+open Core
+
+let with_dir path ~f =
+  let fd = Core_unix.opendir path in
+  f fd;
+  Core_unix.closedir fd
+
+let walk_dir path ~f =
+  with_dir path ~f:(fun fd ->
+      let rec go () =
+        match Core_unix.readdir_opt fd with
+        | Some entry ->
+            f (Filename.concat path entry);
+            go ()
+        | None -> ()
+      in
+      go ())
+
+type testcase = {
+  name : string;
+  dir : string;
+  input : string;
+  expected_output : [ `Okay of string | `Fail ];
+}
+
+let testdata_dir = "./testdata"
+and testcases = ref []
+and testcases_ignored = ref 0
+
+let add_testcase c = testcases := c :: !testcases
+
+let print_testcase_stats () =
+  let okay, fail =
+    List.fold !testcases ~init:(0, 0)
+      ~f:(fun (okay, fail) { expected_output; _ } ->
+        match expected_output with
+        | `Okay _ -> (okay + 1, fail)
+        | `Fail -> (okay, fail + 1))
+  in
+  printf
+    "Loaded %d test cases (ignored %d), expected results: okay %d, fail %d\n%!"
+    (okay + fail) !testcases_ignored okay fail
+
+let imports () =
+  Mininix.Import.materialize
+    [ { filename = "./testdata/lib.nix"; deps = [] } ]
+    ~relative_to:(Core_unix.getcwd ())
+
+type eval_err = [ `Timeout | `ParseError | `ProgramError | `ElaborateError ]
+[@@deriving sexp]
+
+type eval_result = (string, eval_err) Result.t [@@deriving sexp]
+
+let eval input ~name ~dir ~imports =
+  let dir = Filename.to_absolute_exn dir ~relative_to:(Core_unix.getcwd ()) in
+  try
+    input
+    |> Nix.parse ~filename:(name ^ ".nix")
+    |> Nix.elaborate ~dir:(Some dir)
+    |> Mininix.Nix2mininix.from_nix |> Mininix.apply_prelude
+    |> Mininix.interp_tl ~fuel:`Limited ~mode:`Deep ~imports
+    |> function
+    | Res (Some v) ->
+        Ok (v |> Mininix.Mininix2nix.from_val |> Nix.Printer.to_string)
+    | Res None -> Error `ProgramError
+    | NoFuel -> Error `Timeout
+  with
+  | Nix.ParseError _ -> Error `ParseError
+  | Nix.ElaborateError _ -> Error `ElaborateError
+  | Mininix.Nix2mininix.FromNixError _ -> Error `ElaborateError
+
+let eval_subproc input ~name ~dir ~imports =
+  let rxfd, txfd = Core_unix.pipe () in
+  match Core_unix.fork () with
+  | `In_the_child ->
+      let txc = Core_unix.out_channel_of_descr txfd in
+      eval input ~name ~dir ~imports
+      |> [%sexp_of: eval_result] |> Sexp.output txc;
+      exit 0
+  | `In_the_parent child_pid ->
+      let select_res =
+        Core_unix.select ~restart:true ~read:[ rxfd ] ~write:[] ~except:[]
+          ~timeout:(`After (Time_ns.Span.of_min 1.))
+          ()
+      in
+      if List.is_empty select_res.read then (
+        ignore (Signal_unix.send Signal.kill (`Pid child_pid));
+        ignore (Core_unix.waitpid child_pid);
+        Error `Timeout)
+      else
+        let rxc = Core_unix.in_channel_of_descr rxfd in
+        let res = Sexp.input_sexp rxc |> [%of_sexp: eval_result] in
+        ignore (Core_unix.waitpid child_pid);
+        Core_unix.close ~restart:true rxfd;
+        Core_unix.close ~restart:true txfd;
+        res
+
+type test_result =
+  [ `Timeout
+  | `ParseError
+  | `ProgramError
+  | `ElaborateError
+  | `WrongOutput
+  | `UnexpectedSuccess
+  | `Okay ]
+
+let run_testcase ~imports = function
+  | { name; dir; input; expected_output = `Okay expected_output } -> (
+      match eval_subproc input ~name ~dir ~imports with
+      | Ok got_output ->
+          if String.(strip got_output = strip expected_output) then `Okay
+          else `WrongOutput
+      | Error err -> (err :> test_result))
+  | { name; dir; input; expected_output = `Fail } -> (
+      match eval_subproc input ~name ~dir ~imports with
+      | Ok _ -> `UnexpectedSuccess
+      | Error _ -> `Okay)
+
+type test_stats = {
+  okay : int;
+  unexpected_success : int;
+  wrong_output : int;
+  parse_error : int;
+  elaborate_error : int;
+  program_error : int;
+  timeout : int;
+}
+
+let test_stats_empty =
+  {
+    okay = 0;
+    unexpected_success = 0;
+    wrong_output = 0;
+    parse_error = 0;
+    elaborate_error = 0;
+    program_error = 0;
+    timeout = 0;
+  }
+
+let run_testcases () =
+  Nix.Printer.set_width 1000000;
+  let mat_imports = imports () in
+  let stats =
+    List.foldi !testcases ~init:test_stats_empty ~f:(fun i stats c ->
+        printf "[%d/%d] %s  %!" (i + 1) (List.length !testcases) c.name;
+        match run_testcase c ~imports:mat_imports with
+        | `Okay ->
+            printf "okay\n%!";
+            { stats with okay = stats.okay + 1 }
+        | `UnexpectedSuccess ->
+            printf "unexpectedly succeeded\n%!";
+            { stats with unexpected_success = stats.unexpected_success + 1 }
+        | `WrongOutput ->
+            printf "gave wrong output\n%!";
+            { stats with wrong_output = stats.wrong_output + 1 }
+        | `ParseError ->
+            printf "could not be parsed\n%!";
+            { stats with parse_error = stats.parse_error + 1 }
+        | `ElaborateError ->
+            printf "could not be elaborated\n%!";
+            { stats with elaborate_error = stats.elaborate_error + 1 }
+        | `ProgramError ->
+            printf "failed to execute\n%!";
+            { stats with program_error = stats.program_error + 1 }
+        | `Timeout ->
+            printf "timed out\n%!";
+            { stats with timeout = stats.timeout + 1 })
+  in
+  printf
+    "Results:\n\
+    \  %d gave the expected output\n\
+    \  %d unexpectedly succeeded\n\
+    \  %d gave wrong output\n\
+    \  %d could not be parsed\n\
+    \  %d could not be elaborated\n\
+    \  %d failed to execute\n\
+    \  %d timed out\n\
+     %!"
+    stats.okay stats.unexpected_success stats.wrong_output stats.parse_error
+    stats.elaborate_error stats.program_error stats.timeout
+
+let try_add_testcase without_ext =
+  try
+    let dir = Filename.dirname without_ext in
+    let input = In_channel.read_all (without_ext ^ ".nix") in
+    let name = Filename.basename without_ext in
+    if String.is_prefix ~prefix:"eval-fail" name then
+      add_testcase { name; dir; input; expected_output = `Fail }
+    else if String.is_prefix ~prefix:"eval-okay" name then
+      let expected_output = In_channel.read_all (without_ext ^ ".exp") in
+      add_testcase { name; dir; input; expected_output = `Okay expected_output }
+  with
+  (* There are certain test cases where the '.nix' file is available, but
+     there is no '.exp' file. (Instead, for example, there may be a
+     '.exp-disabled' file, which we don't check for.) So [add_testcase] fails
+     when trying to read the '.exp' file, which does not exist. We catch the
+     exception that is then raised in [add_testcase] here. *)
+  | Sys_error _ ->
+    ()
+
+let ignore_tests =
+  [
+    (* We do not implement '«repeated»' *)
+    "eval-okay-repeated-empty-attrs";
+    "eval-okay-repeated-empty-list";
+    (* # Very specific / hard-to-implement builtins: *)
+    (* We do not implement conversion from/to JSON/XML *)
+    "eval-okay-toxml";
+    "eval-okay-toxml2";
+    "eval-okay-tojson";
+    "eval-okay-fromTOML";
+    "eval-okay-fromTOML-timestamps";
+    "eval-okay-fromjson";
+    "eval-okay-fromjson-escapes";
+    "eval-fail-fromJSON-overflowing";
+    "eval-fail-fromTOML-timestamps";
+    "eval-fail-toJSON";
+    (* We do not implement hasing *)
+    "eval-okay-convertHash";
+    "eval-okay-hashstring";
+    "eval-okay-hashfile";
+    "eval-okay-groupBy";
+    "eval-okay-zipAttrsWith";
+    "eval-fail-hashfile-missing";
+    (* We do not support filesystem operations *)
+    "eval-okay-readDir";
+    "eval-okay-readfile";
+    "eval-okay-readFileType";
+    "eval-okay-symlink-resolution";
+    (* We do not support version operations *)
+    "eval-okay-splitversion";
+    "eval-okay-versions";
+    (* We do not support flake references *)
+    "eval-okay-parse-flake-ref";
+    "eval-okay-flake-ref-to-string";
+    "eval-fail-flake-ref-to-string-negative-integer";
+    (* We do not support regexes *)
+    "eval-okay-regex-match";
+    "eval-okay-regex-split";
+    (* # Features that the core interpreter lacks *)
+    (* We do not implement derivations and contexts *)
+    "eval-okay-derivation-legacy";
+    "eval-okay-eq-derivations";
+    "eval-fail-addDrvOutputDependencies-empty-context";
+    "eval-fail-addDrvOutputDependencies-multi-elem-context";
+    "eval-fail-addDrvOutputDependencies-wrong-element-kind";
+    "eval-fail-assert-equal-derivations";
+    "eval-fail-assert-equal-derivations-extra";
+    "eval-fail-derivation-name";
+    "eval-okay-context";
+    "eval-okay-context-introspection";
+    "eval-okay-substring-context";
+    "eval-fail-addErrorContext-example";
+    (* We do not support scopedImport *)
+    "eval-okay-import";
+    (* We do not support tryEval *)
+    "eval-okay-redefine-builtin";
+    "eval-okay-tryeval";
+    (* We do not support unsafeGetAttrPos nor __curPos *)
+    "eval-okay-curpos";
+    "eval-okay-getattrpos";
+    "eval-okay-getattrpos-functionargs";
+    "eval-okay-getattrpos-undefined";
+    "eval-okay-inherit-attr-pos";
+    (* We do not support environment variable lookup *)
+    "eval-okay-getenv";
+    (* We do not support '__override's. Rationale: this construct has expressly
+       been avoided in Nixpkgs since the 13.10 release, see
+       https://github.com/NixOS/nixpkgs/issues/2112 *)
+    "eval-okay-attrs6";
+    "eval-okay-overrides";
+    "eval-fail-set-override";
+    (* We do not implement the 'trace' builtin *)
+    "eval-okay-print";
+    "eval-okay-inherit-from";
+    (* ^ also uses __overrides, for which we lack support *)
+    (* We do not implement flags to set arguments / retrieve attributes
+       for the evaluator *)
+    (* We do not support setting variables outside of the program *)
+    "eval-okay-autoargs";
+    (* We do not support paths *)
+    "eval-okay-baseNameOf";
+    "eval-okay-path";
+    "eval-okay-path-string-interpolation";
+    "eval-okay-pathexists";
+    "eval-okay-search-path";
+    "eval-okay-string";
+    "eval-okay-types";
+    "eval-fail-assert-equal-paths";
+    "eval-fail-bad-string-interpolation-2";
+    "eval-fail-nonexist-path";
+    "eval-fail-path-slash";
+    "eval-fail-to-path";
+    (* We do not implement the 'currentSystem' and 'dirOf' builtins *)
+    "eval-okay-builtins";
+    (* We do not support fetch operations *)
+    "eval-fail-fetchTree-negative";
+    "eval-fail-fetchurl-baseName";
+    "eval-fail-fetchurl-baseName-attrs";
+    "eval-fail-fetchurl-baseName-attrs-name";
+    (* We do not support the pipe operator *)
+    "eval-fail-pipe-operators";
+  ]
+
+let () =
+  Printf.printf "Running in %s\n%!" (Core_unix.getcwd ());
+  walk_dir testdata_dir ~f:(fun entry ->
+      match Filename.split_extension entry with
+      | without_ext, Some "nix" ->
+          if
+            List.exists ignore_tests ~f:(fun name ->
+                String.(name = Filename.basename without_ext))
+          then testcases_ignored := !testcases_ignored + 1
+          else try_add_testcase without_ext
+      | _ -> ());
+  testcases :=
+    List.sort !testcases ~compare:(fun c1 c2 -> String.compare c1.name c2.name);
+  print_testcase_stats ();
+  run_testcases ()