Portable OCaml Projects
Challenges in developing portable OCaml projects
Hi, I'm Manas.
prometheansacrifice
Reason and OCaml devtools
Maintaining esy
Worked with Ligo team
I helped them build a package manager and ship multi-platform builds
Currently helping a Fintech startup with their infrastructure
Also run Reason OCaml India virtual meetups
What is esy?
Alternative package manager for OCaml
Focuses on reproducibility.
Considers C and JS dependencies a part of the project solution
For a unified workflow for common OCaml projects
The problem: availability field
The following project will pull in ocamlfind whose latest version
isn't available on Windows.
{
"dependencies": {
"@opam/cmdliner": "*"
}
}
available: os != "win32"
What is 'os != "win32"'?
Boolean expressions that tell if a package is available or not
There are many factors that could affect the value
Could be because of os, arch, compiler variant etc.
Not available 32-bit ARM and Intel Architectures
available: arch != "arm32" & arch != "x86_32"
Not available on arm32 and x86_32 (32-bit systems). Available on
systems like Windows.
Even the opam-cli version can affect
available: opam-version >= "5.0"
Not available on Apple Silicon
available: !(os = "macos" & arch = "arm64")
What happens if we just ignore the field?
Then packages like Eio stop working.
Opam has many such platform specific packages.
Eio's opam file
available:
"eio_linux"
{= version & os = "linux" &
(os-distribution != "centos" | os-version > "7")}
"eio_posix" {= version & os != "win32"}
"eio_windows" {= version & os = "win32"}
On a Linux machine, ignoring the available field would install all
three variants (eio_linux, eio_posix and eio_windows), which is wrong!
Solution: parameterise the solver
And repeat the process for all platforms and persist the solution
Esy Implementation: code walkthrough
Introducing 'if solution is portable' check
diff --git a/bin/esy.re b/bin/esy.re
index 23d61045e..48dbeca54 100644
--- a/bin/esy.re
+++ b/bin/esy.re
@@ -1,3 +1,4 @@
+[@ocaml.warning "-69"];
open EsyPrimitives;
open EsyBuild;
open EsyPackageConfig;
@@ -999,6 +1000,31 @@ let solve = (force, dumpCudfInput, dumpCudfOutput, proj: Project.t) => {
};
};
+let checkSolutionPortability = (proj: Project.t) => {
+ open RunAsync.Syntax;
+ let lockPath = SandboxSpec.solutionLockPath(proj.projcfg.spec);
+ switch%bind (SolutionLock.ofPath(proj.installSandbox, lockPath)) {
+ | Some(solution) =>
+ let* unPortableDependencies = EsyFetch.Solution.unPortableDependencies(solution);
+ let%lwt () =
+ switch(unPortableDependencies) {
+ | [] => Esy_logs_lwt.app(m => m("Is portable"));
+ | unsupportedPlatforms => {
+ let%lwt () = Esy_logs_lwt.app(m => m("The following packages are problematic and dont build on specified platform"));
+ let f = ((package, _platform)) => {
+ Esy_logs_lwt.app(m => m("Package %a", Package.pp, package));
+ };
+ List.map(~f, unsupportedPlatforms) |> Lwt.join
+ }
+ };
+ RunAsync.return();
+ | None =>
+ error(
+ "No lock found, therefore not solution to check for its portability.",
+ )
+ };
+};
+
let fetch = (proj: Project.t) => {
open RunAsync.Syntax;
let lockPath = SandboxSpec.solutionLockPath(proj.projcfg.spec);
@@ -2093,6 +2119,12 @@ let commandsConfig = {
$ pkgTerm
),
),
+ makeProjectCommand(
+ ~name="check-solution-portability",
+ ~doc="Check if lock file is portable on other platforms",
+ ~docs=lowLevelSection,
+ Term.(const(checkSolutionPortability)),
+ ),
makeProjectCommand(
~name="solve",
~doc="Solve dependencies and store the solution",
Introduction of available field in esy's internal representation
diff --git a/esy-fetch/Package.re b/esy-fetch/Package.re
index e5cee096d..bd50c5aac 100644
--- a/esy-fetch/Package.re
+++ b/esy-fetch/Package.re
@@ -10,7 +10,7 @@ type t = {
devDependencies: PackageId.Set.t,
installConfig: InstallConfig.t,
extraSources: list(ExtraSource.t),
- available: option(string),
+ available: AvailablePlatforms.t,
};
let compare = (a, b) => PackageId.compare(a.id, b.id);
@@ -47,19 +47,6 @@ let opam = pkg =>
}
);
-let evaluateOpamPackageAvailability = pkg => {
- /*
- Allowing sources here would let us resolve to github urls for
- npm dependencies. Atleast in theory. TODO: test this
- */
- switch (pkg.version, pkg.available) {
- | (Source(_), Some(availabilityFilter))
- | (Opam(_), Some(availabilityFilter)) =>
- EsyOpamLibs.Available.eval(availabilityFilter)
- | _ => true
- };
-};
-
module Map =
Map.Make({
type nonrec t = t;
diff --git a/esy-fetch/Package.rei b/esy-fetch/Package.rei
index be9a5a5f9..b59f80cd2 100644
--- a/esy-fetch/Package.rei
+++ b/esy-fetch/Package.rei
@@ -10,13 +10,12 @@ type t = {
devDependencies: PackageId.Set.t,
installConfig: InstallConfig.t, /* currently only tells if pnp is enabled or not */
extraSources: list(ExtraSource.t), /* See opam manual */
- available: option(string),
+ available: AvailablePlatforms.t,
};
let id: t => PackageId.t;
let extraSources: t => list(ExtraSource.t);
let opam: t => RunAsync.t(option((string, Version.t, OpamFile.OPAM.t)));
-let evaluateOpamPackageAvailability: t => bool;
include S.COMPARABLE with type t := t;
include S.PRINTABLE with type t := t;
How we figure out if a solution has unportable dependencies
diff --git a/esy-fetch/Solution.re b/esy-fetch/Solution.re
index 5c798fad5..c4bd54545 100644
--- a/esy-fetch/Solution.re
+++ b/esy-fetch/Solution.re
@@ -119,3 +119,20 @@ let findByNameVersion = (name, version, solution) => {
let%map (_id, pkg) = Graph.findBy(solution, f);
pkg;
};
+
+let unPortableDependencies = solution => {
+ open Package;
+ let f = (pkg) => {
+ let missingPlatforms =
+ AvailablePlatforms.missing(
+ ~expected=AvailablePlatforms.default,
+ ~actual=pkg.available,
+ )
+ if (AvailablePlatforms.isEmpty(missingPlatforms)) {
+ None;
+ } else {
+ Some((pkg, missingPlatforms));
+ }
+ };
+ nodes(solution) |> List.filter_map(~f) |> RunAsync.return;
+};
diff --git a/esy-fetch/Solution.rei b/esy-fetch/Solution.rei
index 1b8e7609c..2f2987a7e 100644
--- a/esy-fetch/Solution.rei
+++ b/esy-fetch/Solution.rei
@@ -55,3 +55,9 @@ let eval: (t, FetchDepSpec.t, PackageId.t) => PackageId.Set.t;
*/
let collect: (t, FetchDepSpec.t, PackageId.t) => PackageId.Set.t;
+
+/**
+ Returns a list of dependencies that don't,ca build on other platforms. The default list of platforms
+ (os, arch) tuples are list in [DefaultPlatforms]
+*/
+let unPortableDependencies: t => RunAsync.t(list((pkg, AvailablePlatforms.t)));
How esy evaluates the filters in opam file.
We only parameterise only on os and arch for now.
diff --git a/esy-opam/Available.re b/esy-opam/Available.re
index 5f9116f33..c0e4f1823 100644
--- a/esy-opam/Available.re
+++ b/esy-opam/Available.re
@@ -12,18 +12,19 @@ let parseOpamFilterString = filterString => {
opamFile.available;
};
-let evalAvailabilityFilter = filter => {
+let evalAvailabilityFilter = (~os, ~arch, filter) => {
let env = (var: OpamVariable.Full.t) => {
let scope = OpamVariable.Full.scope(var);
let name = OpamVariable.Full.variable(var);
switch (scope, OpamVariable.to_string(name)) {
| (OpamVariable.Full.Global, "arch") =>
- Some(OpamVariable.string(System.Arch.show(System.Arch.host)))
+ Some(OpamVariable.string(System.Arch.show(arch)))
| (OpamVariable.Full.Global, "os") =>
+ open System.Platform;
// We could have avoided the following altogether if the System.Platform implementation
// matched opam's. TODO
let sys =
- switch (System.Platform.host) {
+ switch (os) {
| Darwin => "macos"
| Linux => "linux"
| Cygwin => "cygwin"
@@ -41,6 +42,7 @@ let evalAvailabilityFilter = filter => {
OpamFilter.eval_to_bool(~default=true, env, filter);
};
-let eval = availabilityFilter => {
- parseOpamFilterString(availabilityFilter) |> evalAvailabilityFilter;
+let eval = (~os, ~arch, availabilityFilter) => {
+ parseOpamFilterString(availabilityFilter)
+ |> evalAvailabilityFilter(~os, ~arch);
};
diff --git a/esy-opam/Available.rei b/esy-opam/Available.rei
index d8589b807..c83f71a03 100644
--- a/esy-opam/Available.rei
+++ b/esy-opam/Available.rei
@@ -1,8 +1,8 @@
+let evalAvailabilityFilter: (~os: System.Platform.t, ~arch: System.Arch.t, OpamTypes.filter) => bool;
/**
- Evaluates available field (cached in lock file) and determines if the package is available.
+ [eval(~os, ~arch, filter)] evaluates availability filter, [filter] and
+ determines if the package is available.
- Assumes [available] is currently only an opam filter, which means this function must be a no-op on
- NPM packages. Atleast till designed takes into account [platform] and [arch] fields in NPM manifests
*/
-let eval: string => bool;
+let eval: (~os: System.Platform.t, ~arch: System.Arch.t, string) => bool;
Internal representation of AvailablePlatform.t
diff --git a/esy-package-config/AvailablePlatforms.re b/esy-package-config/AvailablePlatforms.re
new file mode 100644
index 000000000..5eab0713b
--- /dev/null
+++ b/esy-package-config/AvailablePlatforms.re
@@ -0,0 +1,111 @@
+module Set = Set.Make({
+ [@deriving ord]
+ type t = (System.Platform.t, System.Arch.t);
+});
+
+type t = Set.t;
+
+let of_yojson =
+ fun
+| `List(availablePlatforms) => {
+ let f = acc => fun
+ | `List([`String(os), `String(arch)]) as json => switch((acc, System.Platform.parse(os), System.Arch.parse(arch))) {
+ | (Ok(acc), Ok(os), arch) => Ok(Set.add((os, arch), acc))
+ | _ => Result.errorf("AvailablePlatforms.parse couldn't parse: %a", Json.Print.ppRegular, json);
+ }
+ | json => Result.errorf("AvailablePlatforms.parse couldn't parse: %a", Json.Print.ppRegular, json);
+ List.fold_left(~f, ~init=Ok(Set.empty), availablePlatforms);
+ }
+ | json =>
+ Result.errorf(
+ "Unexpected JSON %a where AvailablePlatforms.t was expected",
+ Json.Print.ppRegular,
+ json,
+ );
+
+let to_yojson = platforms => {
+ let f = ((os, arch)) =>
+ `List([System.Platform.to_yojson(os), System.Arch.to_yojson(arch)]);
+ `List(platforms |> Set.elements |> List.map(~f));
+};
+
+let default: t = Set.of_list([
+ (System.Platform.Windows, System.Arch.X86_64),
+ (System.Platform.Linux, System.Arch.X86_64),
+ (System.Platform.Darwin, System.Arch.Arm64),
+]);
+
+let filter = (availabilityFilter, platforms) => {
+ let f = ((os, arch)) => {
+ EsyOpamLibs.Available.evalAvailabilityFilter(~os, ~arch, availabilityFilter);
+ };
+ Set.filter(f, platforms);
+};
+
+let missing = (~expected, ~actual) => Set.diff(expected, actual);
+
+let isEmpty = v => Set.is_empty(v);
+let empty = Set.empty;
+let add = (~os, ~arch, v) => Set.add((os, arch), v);
+
+let ppEntry = (ppf, (os, arch)) =>
+ Fmt.pf(ppf, "%a %a", System.Platform.pp, os, System.Arch.pp, arch);
+
+let pp = (ppf, v) => {
+ let sep = Fmt.any(", ");
+ Fmt.hbox(Fmt.list(~sep, ppEntry), ppf, Set.elements(v));
+};
+
+let union = (a, b) => Set.union(a, b);
+
+module Map = {
+ include Map.Make({
+ type t = available;
+ let compare = compare;
+ });
+
+ let to_yojson = (v_to_yojson, map) => {
+ let items = {
+ let f = (k, v, items) => {
+ let (os, arch) = k;
+ let k =
+ Format.asprintf(
+ "%a+%a",
+ System.Platform.pp,
+ os,
+ System.Arch.pp,
+ arch,
+ );
+ [(k, v_to_yojson(v)), ...items];
+ };
+
+ fold(f, map, []);
+ };
+
+ `Assoc(items);
+ };
+
+ let of_yojson = v_of_yojson =>
+ Result.Syntax.(
+ fun
+ | `Assoc(items) => {
+ let f = (map, (k, v)) => {
+ let* (os, arch) =
+ switch (String.split_on_char('+', k)) {
+ | [os, arch] =>
+ switch (System.Platform.parse(os), System.Arch.parse(arch)) {
+ | (Ok(os), arch) => Ok((os, arch))
+ | _ =>
+ Result.errorf("Couldn't parse %s into os-arch tuple", k)
+ }
+ | _ => errorf("Expect key %s to be of syntax <os>+<arch>", k)
+ };
+ let* v = v_of_yojson(v);
+ return(add((os, arch), v, map));
+ };
+
+ Result.List.foldLeft(~f, ~init=empty, items);
+ }
+ | _ => error("expected an object")
+ );
+};
Introduction of available field in internal InstallManifest.re
diff --git a/esy-package-config/InstallManifest.re b/esy-package-config/InstallManifest.re
index ed561ac0f..6665e6504 100644
--- a/esy-package-config/InstallManifest.re
+++ b/esy-package-config/InstallManifest.re
@@ -150,7 +150,7 @@ type t = {
kind,
installConfig: InstallConfig.t,
extraSources: list(ExtraSource.t),
- available: option(string),
+ available: AvailablePlatforms.t,
}
and kind =
| Esy
diff --git a/esy-package-config/InstallManifest.rei b/esy-package-config/InstallManifest.rei
index a20ada9f4..9418b5722 100644
--- a/esy-package-config/InstallManifest.rei
+++ b/esy-package-config/InstallManifest.rei
@@ -56,7 +56,7 @@ type t = {
kind,
installConfig: InstallConfig.t,
extraSources: list(ExtraSource.t),
- available: option(string) /* contains opam filter serialised to be eval'd later */
+ available: AvailablePlatforms.t,
}
and kind =
| Esy
diff --git a/esy-package-config/OfPackageJson.re b/esy-package-config/OfPackageJson.re
index f42e18f7f..5eb961e54 100644
--- a/esy-package-config/OfPackageJson.re
+++ b/esy-package-config/OfPackageJson.re
@@ -172,7 +172,7 @@ module InstallManifestV1 = {
/* */
/********************************************************************/
- let available = None;
+ let available = AvailablePlatforms.default;
return((
{
InstallManifest.name,
+ let availableFilter = OpamFile.OPAM.available(manifest.opam);
let available =
- manifest.opam |> OpamFile.OPAM.available |> OpamFilter.to_string;
+ AvailablePlatforms.default |> AvailablePlatforms.filter(availableFilter);
return((
dependencies,
@@ -645,7 +646,7 @@ let opamManifestToInstallManifest = (~source=?, ~name, ~version, manifest) => {
resolutions: Resolutions.empty,
installConfig: InstallConfig.empty,
extraSources,
- available: Some(available),
+ available,
}),
);
};
@@ -684,7 +685,7 @@ let packageOfSource = (~name, ~overrides, source: Source.t, resolver) => {
RunAsync.ofRun(
{
let version = OpamPackage.Version.of_string("dev");
- EsyOpamLibs.OpamManifest.ofString(
+ OpamManifest.ofString(
~name=opamname,
~version,
data,
Parameterisation of os and arch
diff --git a/esy-solve/Resolver.re b/esy-solve/Resolver.re
index b2b762c4f..ea0c608d2 100644
--- a/esy-solve/Resolver.re
+++ b/esy-solve/Resolver.re
@@ -443,10 +443,11 @@ let opamHashToChecksum = opamHash => {
let contents = OpamHash.contents(opamHash);
(kind, contents);
};
-let convertDependencies = manifest => {
+
+let convertDependencies = (~os, ~arch, manifest) => {
open Result.Syntax;
- let filterOpamFormula = (~build, ~post, ~test, ~doc, ~dev, f) => {
+ let filterOpamFormula = (~os, ~arch, ~build, ~post, ~test, ~doc, ~dev, f) => {
let f = {
let env = var => {
switch (OpamVariable.Full.to_string(var)) {
@@ -456,6 +457,29 @@ let convertDependencies = manifest => {
| "with-dev-setup" => Some(OpamVariable.B(dev))
| "with-doc" => Some(OpamVariable.B(doc))
| "dev" => Some(OpamVariable.B(dev))
+ | "arch" =>
+ switch (arch) {
+ | Some(arch) => Some(OpamVariable.S(System.Arch.show(arch)))
+ | None => None
+ }
+ | "os" =>
+ switch (os) {
+ | Some(os) =>
+ open System.Platform;
+ // We could have avoided the following altogether if the System.Platform implementation
+ // matched opam's. TODO
+ let sys =
+ switch (os) {
+ | Darwin => "macos"
+ | Linux => "linux"
+ | Cygwin => "cygwin"
+ | Unix => "unix"
+ | Windows => "win32"
+ | Unknown => "unknown"
+ };
+ Some(OpamVariable.S(sys));
+ | None => None
+ }
| "version" =>
let version =
OpamPackage.Version.to_string(
@@ -486,8 +510,9 @@ let convertDependencies = manifest => {
};
};
- let filterAndConvertOpamFormula = (~build, ~post, ~test, ~doc, ~dev, f) => {
- let* f = filterOpamFormula(~build, ~post, ~test, ~doc, ~dev, f);
+ let filterAndConvertOpamFormula = (~os, ~build, ~post, ~test, ~doc, ~dev, f) => {
+ let* f =
+ filterOpamFormula(~os, ~arch, ~build, ~post, ~test, ~doc, ~dev, f);
convertOpamFormula(f);
};
@@ -498,6 +523,7 @@ let convertDependencies = manifest => {
let* dependencies = {
let* formula =
filterAndConvertOpamFormula(
+ ~os,
~build=true,
~post=false,
~test=false,
@@ -519,6 +545,7 @@ let convertDependencies = manifest => {
let* devDependencies = {
let* formula =
filterAndConvertOpamFormula(
+ ~os,
~build=false,
~post=false,
~test=true,
@@ -532,6 +559,8 @@ let convertDependencies = manifest => {
let* optDependencies = {
let* formula =
filterOpamFormula(
+ ~os,
+ ~arch,
~build=false,
~post=false,
~test=true,
@@ -575,7 +604,8 @@ let convertDependencies = manifest => {
));
};
-let opamManifestToInstallManifest = (~source=?, ~name, ~version, manifest) => {
+let opamManifestToInstallManifest =
+ (~source=?, ~os, ~arch, ~name, ~version, manifest) => {
open RunAsync.Syntax;
let converted = {
@@ -588,7 +618,7 @@ let opamManifestToInstallManifest = (~source=?, ~name, ~version, manifest) => {
extraSources,
available,
) =
- convertDependencies(manifest);
+ convertDependencies(~os, ~arch, manifest);
return((
source,
dependencies,
@@ -693,6 +723,8 @@ let packageOfSource = (~name, ~overrides, source: Source.t, resolver) => {
},
);
opamManifestToInstallManifest(
+ ~os=resolver.os,
+ ~arch=resolver.arch,
~name,
~version=Version.Source(source),
~source,
@@ -924,6 +956,8 @@ let package = (~resolution: Resolution.t, resolver) => {
| Some(manifest) =>
let* pkg_result =
opamManifestToInstallManifest(
+ ~os=resolver.os,
+ ~arch=resolver.arch,
~name=resolution.name,
~version=Version.Opam(version),
manifest,
E2E tests
--- a/test-e2e/esy-install/opam.test.js
+++ b/test-e2e/esy-install/opam.test.js
@@ -60,17 +60,38 @@ describe('opam available filter tests', () => {
esy: {},
});
- await defineOpamPackageOfFixture(p, 'pkg1', '1.0.0', 'pkg1', undefined, `os = "macos"`);
- await defineOpamPackageOfFixture(p, 'pkg2', '1.0.0', 'pkg2', undefined, `os = "win32"`);
- await defineOpamPackageOfFixture(p, 'pkg3', '1.0.0', 'pkg3', undefined, `os = "linux"`);
+ await defineOpamPackageOfFixture(p, 'dep-macos', '1.0.0', 'depMacos', undefined, `os = "macos"`);
+ await defineOpamPackageOfFixture(p, 'dep-win32', '1.0.0', 'depWin32', undefined, `os = "win32"`);
+ await defineOpamPackageOfFixture(p, 'dep-linux', '1.0.0', 'depLinux', undefined, `os = "linux"`);
+ await p.defineOpamPackageOfFixture(
+ {
+ name: "pkg-main",
+ version: "1.0.0",
+ opam: outdent`
+ opam-version: "2.0"
+ depends: [
+ "dep-macos" {= version & os = "macos"}
+ "dep-win32" {= version & os = "win32"}
+ "dep-linux" {= version & os = "linux"}
+ ]
+ build: [
+ ${helpers.buildCommandInOpam("pkg-main.js")}
+ ["cp" "pkg-main.cmd" "%{bin}%/pkg-main.cmd"]
+ ["cp" "pkg-main.js" "%{bin}%/pkg-main.js"]
+ ]
+ `,
+ },
+ [
+ helpers.dummyExecutable("pkg-main"),
+ ],
+ );
await p.fixture(
helpers.packageJson({
name: 'root',
+ available: [["windows", "x86_64"], ["linux", "x86_64"], ["darwin", "arm64"],["darwin", "x86_64"]],
dependencies: {
- '@opam/pkg1': '*',
- '@opam/pkg2': '*',
- '@opam/pkg3': '*',
+ '@opam/pkg-main': '*',
}
}),
);
@@ -78,35 +99,203 @@ describe('opam available filter tests', () => {
await p.esy();
-
// const solution = await helpers.readSolution(p.projectPath);
// expect(JSON.stringify(solution, null, 2)).toEqual(''); // Just a wait to print lock file
if (process.platform === 'darwin') {
{
- const { stdout } = await p.esy('x pkg1.cmd');
- expect(stdout.trim()).toEqual('__pkg1__');
+ const { stdout } = await p.esy('x depMacos.cmd');
+ expect(stdout.trim()).toEqual('__depMacos__');
+ }
+ await expect(p.esy('x depLinux.cmd')).rejects.toThrow();
+ await expect(p.esy('x depWin32.cmd')).rejects.toThrow();
+ }
+
+ if (process.platform === 'linux') {
+ {
+ const { stdout } = await p.esy('x depLinux.cmd');
+ expect(stdout.trim()).toEqual('__depLinux__');
+ }
+ await expect(p.esy('x depWin32.cmd')).rejects.toThrow();
+ await expect(p.esy('x depMacos.cmd')).rejects.toThrow();
+ }
+
+ if (process.platform === 'win32') {
+ {
+ const { stdout } = await p.esy('x depWin32.cmd');
+ expect(stdout.trim()).toEqual('__depWin32__');
+ }
+ await expect(p.esy('x depMacos.cmd')).rejects.toThrow();
+ await expect(p.esy('x depLinux.cmd')).rejects.toThrow();
+ }
+ });
+
+ it('ensure default solution works on the platforms when only one dependency is platform specific', async () => {
+ const p = await helpers.createTestSandbox();
+
+ await p.defineNpmPackage({
+ name: '@esy-ocaml/substs',
+ version: '0.0.0',
+ esy: {},
+ });
+
+ await defineOpamPackageOfFixture(p, 'dep-macos', '1.0.0', 'depMacos', undefined, `os = "macos"`);
+ await defineOpamPackageOfFixture(p, 'dep-win32', '1.0.0', 'depWin32', undefined, `true`);
+ await defineOpamPackageOfFixture(p, 'dep-linux', '1.0.0', 'depLinux', undefined, `true`);
+
+ await p.defineOpamPackageOfFixture(
+ {
+ name: "pkg-main",
+ version: "1.0.0",
+ opam: outdent`
+ opam-version: "2.0"
+ depends: [
+ "dep-macos" {= version & os = "macos"}
+ "dep-win32"
+ "dep-linux"
+ ]
+ build: [
+ ${helpers.buildCommandInOpam("pkg-main.js")}
+ ["cp" "pkg-main.cmd" "%{bin}%/pkg-main.cmd"]
+ ["cp" "pkg-main.js" "%{bin}%/pkg-main.js"]
+ ]
+ `,
+ },
+ [
+ helpers.dummyExecutable("pkg-main"),
+ ],
+ );
+
+ await p.fixture(
+ helpers.packageJson({
+ name: 'root',
+ dependencies: {
+ '@opam/pkg-main': '*',
+ }
+ }),
+ );
+
+ await p.esy();
+
+ // Since the multiplat solver config only guarantees solutions for windows-x64,
+ // For other platforms, default solution is used.
+ // Which means, unavailable packages could become available there
+ if (process.platform === 'darwin') {
+ {
+ const { stdout } = await p.esy('x depMacos.cmd');
+ expect(stdout.trim()).toEqual('__depMacos__');
+ }
+ {
+ const { stdout } = await p.esy('x depLinux.cmd');
+ expect(stdout.trim()).toEqual('__depLinux__');
+ }
+ {
+ const { stdout } = await p.esy('x depWin32.cmd');
+ expect(stdout.trim()).toEqual('__depWin32__');
+ }
+ }
+
+ if (process.platform === 'linux') {
+ {
+ const { stdout } = await p.esy('x depLinux.cmd');
+ expect(stdout.trim()).toEqual('__depLinux__');
+ }
+ {
+ const { stdout } = await p.esy('x depWin32.cmd');
+ expect(stdout.trim()).toEqual('__depWin32__');
+ }
+ await expect(p.esy('x depMacos.cmd')).rejects.toThrow();
+ }
+
+ if (process.platform === 'win32') {
+ {
+ const { stdout } = await p.esy('x depLinux.cmd');
+ expect(stdout.trim()).toEqual('__depLinux__');
+ }
+ {
+ const { stdout } = await p.esy('x depWin32.cmd');
+ expect(stdout.trim()).toEqual('__depWin32__');
+ }
+ await expect(p.esy('x depMacos.cmd')).rejects.toThrow();
+ }
+ });
+
+ it('ensure package.json/esy.json config is respected', async () => {
+ const p = await helpers.createTestSandbox();
+
+ await p.defineNpmPackage({
+ name: '@esy-ocaml/substs',
+ version: '0.0.0',
+ esy: {},
+ });
+
+ await defineOpamPackageOfFixture(p, 'dep-macos', '1.0.0', 'depMacos', undefined, `os = "macos"`);
+ await defineOpamPackageOfFixture(p, 'dep-win32', '1.0.0', 'depWin32', undefined, `os = "win32"`);
+ await defineOpamPackageOfFixture(p, 'dep-linux', '1.0.0', 'depLinux', undefined, `os = "linux"`);
+
+ await p.defineOpamPackageOfFixture(
+ {
+ name: "pkg-main",
+ version: "1.0.0",
+ opam: outdent`
+ opam-version: "2.0"
+ depends: [
+ "dep-macos" {= version & os = "macos"}
+ "dep-win32" {= version & os = "win32"}
+ "dep-linux" {= version & os = "linux"}
+ ]
+ build: [
+ ${helpers.buildCommandInOpam("pkg-main.js")}
+ ["cp" "pkg-main.cmd" "%{bin}%/pkg-main.cmd"]
+ ["cp" "pkg-main.js" "%{bin}%/pkg-main.js"]
+ ]
+ `,
+ },
+ [
+ helpers.dummyExecutable("pkg-main"),
+ ],
+ );
+
+ await p.fixture(
+ helpers.packageJson({
+ name: 'root',
+ available: [["windows", "x86_64"]],
+ dependencies: {
+ '@opam/pkg-main': '*',
+ }
+ }),
+ );
+
+ await p.esy();
+
+ // Since the multiplat solver config only guarantees solutions for windows-x64,
+ // For other platforms, default solution is used.
+ // Which means, unavailable packages could become available there
+ if (process.platform === 'darwin') {
+ {
+ const { stdout } = await p.esy('x depMacos.cmd');
+ expect(stdout.trim()).toEqual('__depMacos__');
}
- await expect(p.esy('x pkg2.cmd')).rejects.toThrow();
- await expect(p.esy('x pkg3.cmd')).rejects.toThrow();
+ await expect(p.esy('x depLinux.cmd'));
+ await expect(p.esy('x depWin32.cmd'));
}
if (process.platform === 'linux') {
{
- const { stdout } = await p.esy('x pkg3.cmd');
- expect(stdout.trim()).toEqual('__pkg3__');
+ const { stdout } = await p.esy('x depLinux.cmd');
+ expect(stdout.trim()).toEqual('__depLinux__');
}
- await expect(p.esy('x pkg1.cmd')).rejects.toThrow();
- await expect(p.esy('x pkg2.cmd')).rejects.toThrow();
+ await expect(p.esy('x depWin32.cmd'));
+ await expect(p.esy('x depMacos.cmd'));
}
if (process.platform === 'win32') {
{
- const { stdout } = await p.esy('x pkg2.cmd');
- expect(stdout.trim()).toEqual('__pkg2__');
+ const { stdout } = await p.esy('x depWin32.cmd');
+ expect(stdout.trim()).toEqual('__depWin32__');
}
- await expect(p.esy('x pkg1.cmd')).rejects.toThrow();
- await expect(p.esy('x pkg3.cmd')).rejects.toThrow();
+ await expect(p.esy('x depMacos.cmd')).rejects.toThrow();
+ await expect(p.esy('x depLinux.cmd')).rejects.toThrow();
}
});
});
Future work
Remove redundant tasks and better progress reporting
Remove the availability field from SolutionLock.re nodes
diff --git a/esy-fetch/SolutionLock.re b/esy-fetch/SolutionLock.re
index 18d7ffda2..22f494ef4 100644
--- a/esy-fetch/SolutionLock.re
+++ b/esy-fetch/SolutionLock.re
@@ -52,7 +52,7 @@ and node = {
installConfig: InstallConfig.t,
[@default []]
extraSources: list(ExtraSource.t),
- available: [@default None] option(string),
+ available: option(AvailablePlatforms.t),
};
let indexFilename = "index.json";
@@ -232,7 +232,7 @@ let writePackage = (sandbox, pkg: Package.t, gitUsername, gitPassword) => {
devDependencies: pkg.devDependencies,
installConfig: pkg.installConfig,
extraSources: pkg.extraSources,
- available: pkg.available,
+ available: Some(pkg.available),
});
};
@@ -250,6 +250,49 @@ let readPackage = (sandbox, node: node) => {
};
let* overrides = readOverrides(sandbox, node.overrides);
+
+ let* available =
+ switch (node.available) {
+ | Some(available) => RunAsync.return @@ available
+ | None =>
+ switch (node.source) {
+ | Link({path, manifest: Some((Opam, filename)), kind: _}) =>
+ let* opamfile = {
+ let path =
+ DistPath.(path / filename |> toPath(sandbox.Sandbox.spec.path));
+ let* data = Fs.readFile(path);
+ let filename =
+ OpamFile.make(OpamFilename.of_string(Path.show(path)));
+ try(return(OpamFile.OPAM.read_from_string(~filename, data))) {
+ | Failure(msg) =>
+ errorf("error parsing opam metadata %a: %s", Path.pp, path, msg)
+ | _ => error("error parsing opam metadata")
+ };
+ };
+ let availableFilter = OpamFile.OPAM.available(opamfile);
+ RunAsync.return @@
+ AvailablePlatforms.filter(availableFilter, AvailablePlatforms.default);
+
+ | Link(_) => RunAsync.return @@ AvailablePlatforms.default
+ | Install({source: _, opam: None}) =>
+ RunAsync.return @@ AvailablePlatforms.default
+ | Install({source: _, opam: Some(opam)}) =>
+ let* opamfile = {
+ let path = Path.(opam.path / "opam");
+ let* data = Fs.readFile(path);
+ let filename =
+ OpamFile.make(OpamFilename.of_string(Path.show(path)));
+ try(return(OpamFile.OPAM.read_from_string(~filename, data))) {
+ | Failure(msg) =>
+ errorf("error parsing opam metadata %a: %s", Path.pp, path, msg)
+ | _ => error("error parsing opam metadata")
+ };
+ };
+ let availableFilter = OpamFile.OPAM.available(opamfile);
+ RunAsync.return @@
+ AvailablePlatforms.filter(availableFilter, AvailablePlatforms.default);
+ }
+ };
return({
Package.id: node.id,
name: node.name,
@@ -260,7 +303,7 @@ let readPackage = (sandbox, node: node) => {
devDependencies: node.devDependencies,
installConfig: node.installConfig,
extraSources: node.extraSources,
- available: node.available,
+ available,
});
};
Thank you
I'm @ManasJayanth on x.com.
Email: manas [at] dining-philosophers [dot] in
These slides can be found at slides.manas-jayanth.com
