From 4d4269d3a68b36e968838507d19e138037427ebc Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Fri, 25 Oct 2024 19:00:10 +0200 Subject: [PATCH 1/4] Add a project argument to addprocs(::SSHManager) This is necessary since DistributedNext isn't in the default sysimage like Distributed is. --- src/managers.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/managers.jl b/src/managers.jl index b667675..02042a8 100644 --- a/src/managers.jl +++ b/src/managers.jl @@ -78,6 +78,10 @@ addprocs([ **Keyword arguments**: +* `project`: the Julia project to activate on the remote node. This *must* have + `DistributedNext` installed to work. Defaults to the currently active project + on the local node. + * `tunnel`: if `true` then SSH tunneling will be used to connect to the worker from the master process. Default is `false`. @@ -171,7 +175,8 @@ default_addprocs_params(::SSHManager) = :env => [], :tunnel => false, :multiplex => false, - :max_parallel => 10)) + :max_parallel => 10, + :project => Base.current_project())) function launch(manager::SSHManager, params::Dict, launched::Array, launch_ntfy::Condition) # Launch one worker on each unique host in parallel. Additional workers are launched later. @@ -238,8 +243,11 @@ function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, pa tunnel = params[:tunnel] multiplex = params[:multiplex] cmdline_cookie = params[:cmdline_cookie] + project = params[:project] env = Dict{String,String}(params[:env]) + exeflags = `--project=$project $exeflags` + # machine could be of the format [user@]host[:port] bind_addr[:bind_port] # machine format string is split on whitespace machine_bind = split(machine) From f02e734a293255861bf2f2772aaf618b85d8e96b Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Fri, 25 Oct 2024 01:16:07 +0200 Subject: [PATCH 2/4] Get the tests passing Though a few tests have been disabled because they test things like --worker, which is explicitly unsupported. --- src/{Distributed.jl => DistributedNext.jl} | 2 +- src/macros.jl | 2 +- src/managers.jl | 16 +- src/pmap.jl | 2 +- src/precompile.jl | 6 +- test/distributed_exec.jl | 209 +++++++++++---------- test/managers.jl | 4 +- test/splitrange.jl | 4 +- test/topology.jl | 8 +- 9 files changed, 139 insertions(+), 114 deletions(-) rename src/{Distributed.jl => DistributedNext.jl} (99%) diff --git a/src/Distributed.jl b/src/DistributedNext.jl similarity index 99% rename from src/Distributed.jl rename to src/DistributedNext.jl index dc2d9d4..f229519 100644 --- a/src/Distributed.jl +++ b/src/DistributedNext.jl @@ -3,7 +3,7 @@ """ Tools for distributed parallel processing. """ -module Distributed +module DistributedNext # imports for extension import Base: getindex, wait, put!, take!, fetch, isready, push!, length, diff --git a/src/macros.jl b/src/macros.jl index a4fec31..c89e69e 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -191,7 +191,7 @@ Similar to calling `remotecall_eval(Main, procs, expr)`, but with two extra feat """ macro everywhere(ex) procs = GlobalRef(@__MODULE__, :procs) - return esc(:($(Distributed).@everywhere $procs() $ex)) + return esc(:($(DistributedNext).@everywhere $procs() $ex)) end macro everywhere(procs, ex) diff --git a/src/managers.jl b/src/managers.jl index 02042a8..129b65c 100644 --- a/src/managers.jl +++ b/src/managers.jl @@ -234,6 +234,15 @@ function parse_machine(machine::AbstractString) (hoststr, portnum) end +function get_worker_arg(cookie=nothing) + if isnothing(cookie) + return `-E 'using DistributedNext; DistributedNext.start_worker()'` + else + code_str = "using DistributedNext; DistributedNext.start_worker(\"$(cookie)\")" + return `-E $(code_str)` + end +end + function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, params::Dict, launched::Array, launch_ntfy::Condition) shell = params[:shell] ssh = params[:ssh] @@ -257,10 +266,11 @@ function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, pa if length(machine_bind) > 1 exeflags = `--bind-to $(machine_bind[2]) $exeflags` end + if cmdline_cookie - exeflags = `$exeflags --worker=$(cluster_cookie())` + exeflags = `$exeflags $(get_worker_arg(cluster_cookie()))` else - exeflags = `$exeflags --worker` + exeflags = `$exeflags $(get_worker_arg())` end host, portnum = parse_machine(machine_bind[1]) @@ -518,7 +528,7 @@ function launch(manager::LocalManager, params::Dict, launched::Array, c::Conditi end for i in 1:manager.np - cmd = `$(julia_cmd(exename)) $exeflags --bind-to $bind_to --worker` + cmd = `$(julia_cmd(exename)) $exeflags --bind-to $bind_to $(get_worker_arg())` io = open(detach(setenv(addenv(cmd, env), dir=dir)), "r+") write_cookie(io) diff --git a/src/pmap.jl b/src/pmap.jl index 39acc4d..8716650 100644 --- a/src/pmap.jl +++ b/src/pmap.jl @@ -240,7 +240,7 @@ Return `head`: the first `n` elements of `c`; and `tail`: an iterator over the remaining elements. ```jldoctest -julia> b, c = Distributed.head_and_tail(1:10, 3) +julia> b, c = DistributedNext.head_and_tail(1:10, 3) ([1, 2, 3], Base.Iterators.Rest{UnitRange{Int64}, Int64}(1:10, 3)) julia> collect(c) diff --git a/src/precompile.jl b/src/precompile.jl index 87380f6..816905c 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,6 +1,6 @@ -precompile(Tuple{typeof(Distributed.remotecall),Function,Int,Module,Vararg{Any, 100}}) -precompile(Tuple{typeof(Distributed.procs)}) -precompile(Tuple{typeof(Distributed.finalize_ref), Distributed.Future}) +precompile(Tuple{typeof(DistributedNext.remotecall),Function,Int,Module,Vararg{Any, 100}}) +precompile(Tuple{typeof(DistributedNext.procs)}) +precompile(Tuple{typeof(DistributedNext.finalize_ref), DistributedNext.Future}) # This is disabled because it doesn't give much benefit # and the code in Distributed is poorly typed causing many invalidations # TODO: Maybe reenable now that Distributed is not in sysimage. diff --git a/test/distributed_exec.jl b/test/distributed_exec.jl index 71b6b36..00a3881 100644 --- a/test/distributed_exec.jl +++ b/test/distributed_exec.jl @@ -1,20 +1,13 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Distributed, Random, Serialization, Sockets -import Distributed: launch, manage - -sharedir = normpath(joinpath(Sys.BINDIR, "..", "share")) -if parse(Bool, get(ENV, "JULIA_DISTRIBUTED_TESTING_STANDALONE", "false")) - @test !startswith(pathof(Distributed), sharedir) -else - @test startswith(pathof(Distributed), sharedir) -end +using Test, DistributedNext, Random, Serialization, Sockets +import DistributedNext: launch, manage @test cluster_cookie() isa String include(joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testenv.jl")) -@test Distributed.extract_imports(:(begin; import Foo, Bar; let; using Baz; end; end)) == +@test DistributedNext.extract_imports(:(begin; import Foo, Bar; let; using Baz; end; end)) == Any[:(import Foo, Bar), :(using Baz)] # Test a few "remote" invocations when no workers are present @@ -53,11 +46,11 @@ id_me = myid() id_other = filter(x -> x != id_me, procs())[rand(1:(nprocs()-1))] # Test role -@everywhere using Distributed -@test Distributed.myrole() === :master +@everywhere using DistributedNext +@test DistributedNext.myrole() === :master for wid = workers() wrole = remotecall_fetch(wid) do - Distributed.myrole() + DistributedNext.myrole() end @test wrole === :worker end @@ -172,27 +165,27 @@ function include_thread_unsafe_tests() return true end -# Distributed GC tests for Futures +# DistributedNext GC tests for Futures function test_futures_dgc(id) f = remotecall(myid, id) fid = remoteref_id(f) # remote value should be deleted after a fetch - @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == true + @test remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, fid) == true @test f.v === nothing @test fetch(f) == id @test f.v !== nothing yield(); # flush gc msgs - @test poll_while(() -> remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid)) + @test poll_while(() -> remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, fid)) # if unfetched, it should be deleted after a finalize f = remotecall(myid, id) fid = remoteref_id(f) - @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == true + @test remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, fid) == true @test f.v === nothing finalize(f) yield(); # flush gc msgs - @test poll_while(() -> remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid)) + @test poll_while(() -> remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, fid)) end test_futures_dgc(id_me) @@ -208,23 +201,23 @@ fstore = RemoteChannel(wid2) put!(fstore, f) @test fetch(f) == wid1 -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == true remotecall_fetch(r->(fetch(fetch(r)); yield()), wid2, fstore) sleep(0.5) # to ensure that wid2 gc messages have been executed on wid1 -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == false # put! should release remote reference since it would have been cached locally f = Future(wid1) fid = remoteref_id(f) # should not be created remotely till accessed -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == false # create it remotely isready(f) -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == true put!(f, :OK) -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == false @test fetch(f) === :OK # RemoteException should be thrown on a put! when another process has set the value @@ -235,7 +228,7 @@ fstore = RemoteChannel(wid2) put!(fstore, f) # send f to wid2 put!(f, :OK) # set value from master -@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(DistributedNext.PGRP.refs, k), wid1, fid) == true testval = remotecall_fetch(wid2, fstore) do x try @@ -258,30 +251,30 @@ end end f = remotecall_wait(identity, id_other, ones(10)) -rrid = Distributed.RRID(f.whence, f.id) +rrid = DistributedNext.RRID(f.whence, f.id) remotecall_fetch(f25847, id_other, f) -@test BitSet([id_me]) == remotecall_fetch(()->Distributed.PGRP.refs[rrid].clientset, id_other) +@test BitSet([id_me]) == remotecall_fetch(()->DistributedNext.PGRP.refs[rrid].clientset, id_other) remotecall_fetch(f25847, id_other, f) -@test BitSet([id_me]) == remotecall_fetch(()->Distributed.PGRP.refs[rrid].clientset, id_other) +@test BitSet([id_me]) == remotecall_fetch(()->DistributedNext.PGRP.refs[rrid].clientset, id_other) finalize(f) yield() # flush gc msgs -@test poll_while(() -> remotecall_fetch(chk_rrid->(yield(); haskey(Distributed.PGRP.refs, chk_rrid)), id_other, rrid)) +@test poll_while(() -> remotecall_fetch(chk_rrid->(yield(); haskey(DistributedNext.PGRP.refs, chk_rrid)), id_other, rrid)) -# Distributed GC tests for RemoteChannels +# DistributedNext GC tests for RemoteChannels function test_remoteref_dgc(id) rr = RemoteChannel(id) put!(rr, :OK) rrid = remoteref_id(rr) # remote value should be deleted after finalizing the ref - @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid) == true + @test remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, rrid) == true @test fetch(rr) === :OK - @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid) == true + @test remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, rrid) == true finalize(rr) yield(); # flush gc msgs - @test poll_while(() -> remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid)) + @test poll_while(() -> remotecall_fetch(k->(yield();haskey(DistributedNext.PGRP.refs, k)), id, rrid)) end test_remoteref_dgc(id_me) test_remoteref_dgc(id_other) @@ -295,16 +288,16 @@ let wid1 = workers()[1], put!(fstore, rr) if include_thread_unsafe_tests() - @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == true + @test remotecall_fetch(k -> haskey(DistributedNext.PGRP.refs, k), wid1, rrid) == true end finalize(rr) # finalize locally yield() # flush gc msgs if include_thread_unsafe_tests() - @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == true + @test remotecall_fetch(k -> haskey(DistributedNext.PGRP.refs, k), wid1, rrid) == true end remotecall_fetch(r -> (finalize(take!(r)); yield(); nothing), wid2, fstore) # finalize remotely sleep(0.5) # to ensure that wid2 messages have been executed on wid1 - @test poll_while(() -> remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid)) + @test poll_while(() -> remotecall_fetch(k -> haskey(DistributedNext.PGRP.refs, k), wid1, rrid)) end # Tests for issue #23109 - should not hang. @@ -347,7 +340,7 @@ test_indexing(RemoteChannel()) test_indexing(RemoteChannel(id_other)) # Test ser/deser to non-ClusterSerializer objects. -function test_regular_io_ser(ref::Distributed.AbstractRemoteRef) +function test_regular_io_ser(ref::DistributedNext.AbstractRemoteRef) io = IOBuffer() serialize(io, ref) seekstart(io) @@ -687,7 +680,7 @@ end n = 10 as = [rand(4,4) for i in 1:n] bs = deepcopy(as) -cs = collect(Distributed.pgenerate(x->(sleep(rand()*0.1); svd(x)), bs)) +cs = collect(DistributedNext.pgenerate(x->(sleep(rand()*0.1); svd(x)), bs)) svdas = map(svd, as) for i in 1:n @test cs[i].U ≈ svdas[i].U @@ -738,16 +731,16 @@ clear!(wp) @test length(wp.map_obj2ref) == 0 # default_worker_pool! tests -wp_default = Distributed.default_worker_pool() +wp_default = DistributedNext.default_worker_pool() try local wp = CachingPool(workers()) - Distributed.default_worker_pool!(wp) + DistributedNext.default_worker_pool!(wp) @test [1:100...] == pmap(x->x, wp, 1:100) @test !isempty(wp.map_obj2ref) clear!(wp) @test isempty(wp.map_obj2ref) finally - Distributed.default_worker_pool!(wp_default) + DistributedNext.default_worker_pool!(wp_default) end # The below block of tests are usually run only on local development systems, since: @@ -771,8 +764,8 @@ if DoFullTest all_w = workers() # Test sending fake data to workers. The worker processes will print an # error message but should not terminate. - for w in Distributed.PGRP.workers - if isa(w, Distributed.Worker) + for w in DistributedNext.PGRP.workers + if isa(w, DistributedNext.Worker) local s = connect(w.config.host, w.config.port) write(s, randstring(32)) end @@ -981,7 +974,7 @@ end rc_unbuffered_other = RemoteChannel(()->Channel{Int}(0), id_other) close(rc_unbuffered_other) try; take!(rc_unbuffered_other); catch; end -@test !remotecall_fetch(rc -> islocked(Distributed.lookup_ref(remoteref_id(rc)).synctake), +@test !remotecall_fetch(rc -> islocked(DistributedNext.lookup_ref(remoteref_id(rc)).synctake), id_other, rc_unbuffered_other) # github PR #14456 @@ -1081,7 +1074,7 @@ end # Test calling @everywhere from a module not defined on the workers module LocalBar - using Distributed + using DistributedNext bar() = @everywhere new_bar()=myid() end LocalBar.bar() @@ -1143,7 +1136,7 @@ function get_remote_num_threads(processes_added) end function test_blas_config(pid, expected) - for worker in Distributed.PGRP.workers + for worker in DistributedNext.PGRP.workers if worker.id == pid @test worker.config.enable_threaded_blas == expected return @@ -1329,9 +1322,9 @@ global v4 = v3 # Global references to Types and Modules should work if they are locally defined global v5 = Int -global v6 = Distributed +global v6 = DistributedNext @test remotecall_fetch(()->v5, id_other) === Int -@test remotecall_fetch(()->v6, id_other) === Distributed +@test remotecall_fetch(()->v6, id_other) === DistributedNext struct FooStructLocal end module FooModLocal end @@ -1464,7 +1457,7 @@ wrapped_var_ser_tests() global ids_cleanup = fill(1., 6) global ids_func = ()->ids_cleanup -clust_ser = (Distributed.worker_from_id(id_other)).w_serializer +clust_ser = (DistributedNext.worker_from_id(id_other)).w_serializer @test remotecall_fetch(ids_func, id_other) == ids_cleanup # TODO Add test for cleanup from `clust_ser.glbs_in_tnobj` @@ -1634,10 +1627,10 @@ function launch(manager::WorkerArgTester, params::Dict, launched::Array, c::Cond exename = params[:exename] exeflags = params[:exeflags] - cmd = `$exename $exeflags --bind-to $(Distributed.LPROC.bind_addr) $(manager.worker_opt)` + cmd = `$exename $exeflags --bind-to $(DistributedNext.LPROC.bind_addr) $(manager.worker_opt)` cmd = pipeline(detach(setenv(cmd, dir=dir))) io = open(cmd, "r+") - manager.write_cookie && Distributed.write_cookie(io) + manager.write_cookie && DistributedNext.write_cookie(io) wconfig = WorkerConfig() wconfig.process = io @@ -1650,18 +1643,20 @@ manage(::WorkerArgTester, ::Integer, ::WorkerConfig, ::Symbol) = nothing nprocs()>1 && rmprocs(workers()) -npids = addprocs_with_testenv(WorkerArgTester(`--worker`, true)) -@test remotecall_fetch(myid, npids[1]) == npids[1] -rmprocs(npids) +## These tests are disabled because DistributedNext has no way of supporting the +## --worker argument. +# npids = addprocs_with_testenv(WorkerArgTester(`--worker`, true)) +# @test remotecall_fetch(myid, npids[1]) == npids[1] +# rmprocs(npids) -cluster_cookie("") # An empty string is a valid cookie -npids = addprocs_with_testenv(WorkerArgTester(`--worker=`, false)) -@test remotecall_fetch(myid, npids[1]) == npids[1] -rmprocs(npids) +# cluster_cookie("") # An empty string is a valid cookie +# npids = addprocs_with_testenv(WorkerArgTester(`--worker=`, false)) +# @test remotecall_fetch(myid, npids[1]) == npids[1] +# rmprocs(npids) -cluster_cookie("foobar") # custom cookie -npids = addprocs_with_testenv(WorkerArgTester(`--worker=foobar`, false)) -@test remotecall_fetch(myid, npids[1]) == npids[1] +# cluster_cookie("foobar") # custom cookie +# npids = addprocs_with_testenv(WorkerArgTester(`--worker=foobar`, false)) +# @test remotecall_fetch(myid, npids[1]) == npids[1] # tests for start_worker options to retain stdio (issue #31035) struct RetainStdioTester <: ClusterManager @@ -1674,8 +1669,8 @@ function launch(manager::RetainStdioTester, params::Dict, launched::Array, c::Co exename = params[:exename] exeflags = params[:exeflags] - jlcmd = "using Distributed; start_worker(\"\"; close_stdin=$(manager.close_stdin), stderr_to_stdout=$(manager.stderr_to_stdout));" - cmd = detach(setenv(`$exename $exeflags --bind-to $(Distributed.LPROC.bind_addr) -e $jlcmd`, dir=dir)) + jlcmd = "using DistributedNext; start_worker(\"\"; close_stdin=$(manager.close_stdin), stderr_to_stdout=$(manager.stderr_to_stdout));" + cmd = detach(setenv(`$exename $exeflags --bind-to $(DistributedNext.LPROC.bind_addr) -e $jlcmd`, dir=dir)) proc = open(cmd, "r+") wconfig = WorkerConfig() @@ -1717,7 +1712,7 @@ function reuseport_tests() remotecall_fetch(p) do ports_lower = [] # ports of pids lower than myid() ports_higher = [] # ports of pids higher than myid() - for w in Distributed.PGRP.workers + for w in DistributedNext.PGRP.workers w.id == myid() && continue port = Sockets._sockname(w.r_stream, true)[2] if (w.id == 1) @@ -1768,38 +1763,38 @@ end # issue #28966 let code = """ - import Distributed - Distributed.addprocs(1) - Distributed.@everywhere f() = myid() - for w in Distributed.workers() - @assert Distributed.remotecall_fetch(f, w) == w + import DistributedNext + DistributedNext.addprocs(1) + DistributedNext.@everywhere f() = myid() + for w in DistributedNext.workers() + @assert DistributedNext.remotecall_fetch(f, w) == w end """ @test success(`$(Base.julia_cmd()) --startup-file=no -e $code`) end -# PR 32431: tests for internal Distributed.head_and_tail -let (h, t) = Distributed.head_and_tail(1:10, 3) +# PR 32431: tests for internal DistributedNext.head_and_tail +let (h, t) = DistributedNext.head_and_tail(1:10, 3) @test h == 1:3 @test collect(t) == 4:10 end -let (h, t) = Distributed.head_and_tail(1:10, 0) +let (h, t) = DistributedNext.head_and_tail(1:10, 0) @test h == [] @test collect(t) == 1:10 end -let (h, t) = Distributed.head_and_tail(1:3, 5) +let (h, t) = DistributedNext.head_and_tail(1:3, 5) @test h == 1:3 @test collect(t) == [] end -let (h, t) = Distributed.head_and_tail(1:3, 3) +let (h, t) = DistributedNext.head_and_tail(1:3, 3) @test h == 1:3 @test collect(t) == [] end -let (h, t) = Distributed.head_and_tail(Int[], 3) +let (h, t) = DistributedNext.head_and_tail(Int[], 3) @test h == [] @test collect(t) == [] end -let (h, t) = Distributed.head_and_tail(Int[], 0) +let (h, t) = DistributedNext.head_and_tail(Int[], 0) @test h == [] @test collect(t) == [] end @@ -1823,9 +1818,10 @@ end # Propagation of package environments for local workers (#28781) let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp + pkg_project = joinpath(Base.pkgdir(DistributedNext), "Project.toml") project = mkdir(joinpath(tmp, "project")) depots = [mkdir(joinpath(tmp, "depot1")), mkdir(joinpath(tmp, "depot2"))] - load_path = [mkdir(joinpath(tmp, "load_path")), "@stdlib", "@"] + load_path = [mkdir(joinpath(tmp, "load_path")), "@stdlib", "@", pkg_project] pathsep = Sys.iswindows() ? ";" : ":" env = Dict( "JULIA_DEPOT_PATH" => join(depots, pathsep), @@ -1834,14 +1830,22 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp # CI system where `TMPDIR` is special. "TMPDIR" => dirname(tmp), ) - setupcode = """ - using Distributed, Test + + funcscode = """ + using Test + @everywhere begin depot_path() = DEPOT_PATH load_path() = LOAD_PATH active_project() = Base.ACTIVE_PROJECT[] end """ + + setupcode = """ + using DistributedNext + addprocs(1) + """ * funcscode + testcode = setupcode * """ for w in workers() @test remotecall_fetch(depot_path, w) == DEPOT_PATH @@ -1851,14 +1855,17 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp @test remotecall_fetch(Base.active_project, w) == Base.active_project() end """ - # No active project - extracode = """ - for w in workers() - @test remotecall_fetch(active_project, w) === Base.ACTIVE_PROJECT[] === nothing - end - """ - cmd = setenv(`$(julia) -p1 -e $(testcode * extracode)`, env) - @test success(cmd) + + # No active project. This test is disabled because it won't work with + # DistributedNext since the package isn't a stdlib. + # extracode = """ + # for w in workers() + # @test remotecall_fetch(active_project, w) === Base.ACTIVE_PROJECT[] === nothing + # end + # """ + # cmd = setenv(`$(julia) -e $(testcode * extracode)`, env) + # @test success(cmd) + # --project extracode = """ for w in workers() @@ -1866,16 +1873,16 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp $(repr(project)) end """ - cmd = setenv(`$(julia) --project=$(project) -p1 -e $(testcode * extracode)`, env) + cmd = setenv(`$(julia) --project=$(project) -e $(testcode * extracode)`, env) @test success(cmd) # JULIA_PROJECT - cmd = setenv(`$(julia) -p1 -e $(testcode * extracode)`, + cmd = setenv(`$(julia) -e $(testcode * extracode)`, (env["JULIA_PROJECT"] = project; env)) @test success(cmd) # Pkg.activate(...) activateish = """ Base.ACTIVE_PROJECT[] = $(repr(project)) - using Distributed + using DistributedNext addprocs(1) """ cmd = setenv(`$(julia) -e $(activateish * testcode * extracode)`, env) @@ -1888,7 +1895,7 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp append!(empty!(LOAD_PATH), l) """ addcode = """ - using Distributed + using DistributedNext addprocs(1) # after shuffling """ extracode = """ @@ -1899,31 +1906,39 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp """ cmd = setenv(`$(julia) -e $(shufflecode * addcode * testcode * extracode)`, env) @test success(cmd) - # Mismatch when shuffling after proc addition - failcode = shufflecode * setupcode * """ + # Mismatch when shuffling after proc addition. Note that the use of + # `addcode` mimics the behaviour of -p1 as the first worker is started + # before `shufflecode` executes. + failcode = addcode * shufflecode * funcscode * """ + @show workers() for w in workers() @test remotecall_fetch(load_path, w) == reverse(LOAD_PATH) == $(repr(load_path)) @test remotecall_fetch(depot_path, w) == reverse(DEPOT_PATH) == $(repr(depots)) end """ - cmd = setenv(`$(julia) -p1 -e $(failcode)`, env) + cmd = setenv(`$(julia) -e $(failcode)`, env) @test success(cmd) + + # Hideous hack to double escape path separators on Windows so that it gets + # interpolated into the string (and then Cmd) correctly. + escaped_pkg_project = Sys.iswindows() ? replace(pkg_project, "\\" => "\\\\") : pkg_project + # Passing env or exeflags to addprocs(...) to override defaults envcode = """ - using Distributed + using DistributedNext project = mktempdir() env = Dict( - "JULIA_LOAD_PATH" => string(LOAD_PATH[1], $(repr(pathsep)), "@stdlib"), + "JULIA_LOAD_PATH" => string(LOAD_PATH[1], $(repr(pathsep)), "@stdlib", $(repr(pathsep)), "$(escaped_pkg_project)"), "JULIA_DEPOT_PATH" => DEPOT_PATH[1], "TMPDIR" => ENV["TMPDIR"], ) addprocs(1; env = env, exeflags = `--project=\$(project)`) env["JULIA_PROJECT"] = project addprocs(1; env = env) - """ * setupcode * """ + """ * funcscode * """ for w in workers() @test remotecall_fetch(depot_path, w) == [DEPOT_PATH[1]] - @test remotecall_fetch(load_path, w) == [LOAD_PATH[1], "@stdlib"] + @test remotecall_fetch(load_path, w) == [LOAD_PATH[1], "@stdlib", "$(escaped_pkg_project)"] @test remotecall_fetch(active_project, w) == project @test remotecall_fetch(Base.active_project, w) == joinpath(project, "Project.toml") end diff --git a/test/managers.jl b/test/managers.jl index 7971222..54ca3a5 100644 --- a/test/managers.jl +++ b/test/managers.jl @@ -1,9 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test -using Distributed +using DistributedNext using Sockets -using Distributed: parse_machine, SSHManager, LocalManager +using DistributedNext: parse_machine, SSHManager, LocalManager @test parse_machine("127.0.0.1") == ("127.0.0.1", nothing) @test parse_machine("127.0.0.1:80") == ("127.0.0.1", 80) diff --git a/test/splitrange.jl b/test/splitrange.jl index 1cb12e1..bbb8284 100644 --- a/test/splitrange.jl +++ b/test/splitrange.jl @@ -1,8 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test -using Distributed -using Distributed: splitrange +using DistributedNext +using DistributedNext: splitrange @test splitrange(1, 11, 1) == Array{UnitRange{Int64},1}([1:11]) @test splitrange(0, 10, 1) == Array{UnitRange{Int64},1}([0:10]) diff --git a/test/topology.jl b/test/topology.jl index a24efb2..66cc78d 100644 --- a/test/topology.jl +++ b/test/topology.jl @@ -43,11 +43,11 @@ function launch(manager::TopoTestManager, params::Dict, launched::Array, c::Cond exename = params[:exename] exeflags = params[:exeflags] - cmd = `$exename $exeflags --bind-to $(Distributed.LPROC.bind_addr) --worker` + cmd = `$exename $exeflags --bind-to $(DistributedNext.LPROC.bind_addr) $(DistributedNext.get_worker_arg())` cmd = pipeline(detach(setenv(cmd, dir=dir))) for i in 1:manager.np io = open(cmd, "r+") - Distributed.write_cookie(io) + DistributedNext.write_cookie(io) wconfig = WorkerConfig() wconfig.process = io @@ -98,8 +98,8 @@ remove_workers_and_test() # test `lazy` connection setup function def_count_conn() @everywhere function count_connected_workers() - count(x -> isa(x, Distributed.Worker) && isdefined(x, :r_stream) && isopen(x.r_stream), - Distributed.PGRP.workers) + count(x -> isa(x, DistributedNext.Worker) && isdefined(x, :r_stream) && isopen(x.r_stream), + DistributedNext.PGRP.workers) end end From 6241f27f249295c9ceea42f64bf727c00806d617 Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Fri, 25 Oct 2024 19:02:31 +0200 Subject: [PATCH 3/4] Use LibSSH.jl to set up an SSH server for the SSHManager tests Also enables it by default. --- Project.toml | 5 +- test/distributed_exec.jl | 150 ++++++++++++++++++++------------------- 2 files changed, 81 insertions(+), 74 deletions(-) diff --git a/Project.toml b/Project.toml index dd09740..97fc7e9 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DistributedNext" uuid = "fab6aee4-877b-4bac-a744-3eca44acbb6f" -version = "1" +version = "1.0.0" [deps] Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -8,8 +8,9 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" [extras] +LibSSH = "00483490-30f8-4353-8aba-35b82f51f4d0" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["LinearAlgebra", "Test"] +test = ["LinearAlgebra", "Test", "LibSSH"] diff --git a/test/distributed_exec.jl b/test/distributed_exec.jl index 00a3881..a5aea3e 100644 --- a/test/distributed_exec.jl +++ b/test/distributed_exec.jl @@ -3,6 +3,9 @@ using Test, DistributedNext, Random, Serialization, Sockets import DistributedNext: launch, manage +import LibSSH as ssh +import LibSSH.Demo: DemoServer + @test cluster_cookie() isa String include(joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testenv.jl")) @@ -743,11 +746,9 @@ finally DistributedNext.default_worker_pool!(wp_default) end -# The below block of tests are usually run only on local development systems, since: -# - tests which print errors -# - addprocs tests are memory intensive -# - ssh addprocs requires sshd to be running locally with passwordless login enabled. -# The test block is enabled by defining env JULIA_TESTFULL=1 +# The below block of tests are usually run only on local development systems, +# since they print errors. The test block is enabled by defining env +# JULIA_TESTFULL=1. DoFullTest = Base.get_bool_env("JULIA_TESTFULL", false) @@ -772,8 +773,10 @@ if DoFullTest end @test workers() == all_w @test all([p == remotecall_fetch(myid, p) for p in all_w]) +end -if Sys.isunix() # aka have ssh +# LibSSH.jl currently only works on 64bit unixes +if Sys.isunix() && Sys.WORD_SIZE == 64 function test_n_remove_pids(new_pids) for p in new_pids w_in_remote = sort(remotecall_fetch(workers, p)) @@ -791,75 +794,78 @@ if Sys.isunix() # aka have ssh remotecall_fetch(rmprocs, 1, new_pids) end - print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n") - print("Please ensure: \n") - print("1) sshd is running locally with passwordless login enabled.\n") - print("2) Env variable USER is defined and is the ssh user.\n") - print("3) Port 9300 is not in use.\n") - - sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR ` - #Issue #9951 - hosts=[] - localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"] - num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9"))) - - for i in 1:(num_workers/length(localhost_aliases)) - append!(hosts, localhost_aliases) - end - - print("\nTesting SSH addprocs with $(length(hosts)) workers...\n") - new_pids = addprocs_with_testenv(hosts; sshflags=sshflags) - @test length(new_pids) == length(hosts) - test_n_remove_pids(new_pids) - - print("\nMixed ssh addprocs with :auto\n") - new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags) - @test length(new_pids) == (2 + Sys.CPU_THREADS) - test_n_remove_pids(new_pids) - - print("\nMixed ssh addprocs with numeric counts\n") - new_pids = addprocs_with_testenv([("localhost", 2), ("127.0.0.1", 2), "localhost"]; sshflags=sshflags) - @test length(new_pids) == 5 - test_n_remove_pids(new_pids) - - print("\nssh addprocs with tunnel\n") - new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, sshflags=sshflags) - @test length(new_pids) == num_workers - test_n_remove_pids(new_pids) - - print("\nssh addprocs with tunnel (SSH multiplexing)\n") - new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, multiplex=true, sshflags=sshflags) - @test length(new_pids) == num_workers - controlpath = joinpath(homedir(), ".ssh", "julia-$(ENV["USER"])@localhost:22") - @test issocket(controlpath) - test_n_remove_pids(new_pids) - @test :ok == timedwait(()->!issocket(controlpath), 10.0; pollint=0.5) - - print("\nAll supported formats for hostname\n") - h1 = "localhost" - user = ENV["USER"] - h2 = "$user@$h1" - h3 = "$h2:22" - h4 = "$h3 $(string(getipaddr()))" - h5 = "$h4:9300" - - new_pids = addprocs_with_testenv([h1, h2, h3, h4, h5]; sshflags=sshflags) - @test length(new_pids) == 5 - test_n_remove_pids(new_pids) - - print("\nkeyword arg exename\n") - for exename in [`$(joinpath(Sys.BINDIR, Base.julia_exename()))`, "$(joinpath(Sys.BINDIR, Base.julia_exename()))"] - for addp_func in [()->addprocs_with_testenv(["localhost"]; exename=exename, exeflags=test_exeflags, sshflags=sshflags), - ()->addprocs_with_testenv(1; exename=exename, exeflags=test_exeflags)] - - local new_pids = addp_func() - @test length(new_pids) == 1 - test_n_remove_pids(new_pids) + println("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.") + println("Please ensure port 9300 and 2222 are not in use.") + + DemoServer(2222; auth_methods=[ssh.AuthMethod_None], allow_auth_none=true, verbose=false, timeout=3600) do + sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR -p 2222 ` + #Issue #9951 + hosts=[] + localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"] + num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9"))) + + for i in 1:(num_workers/length(localhost_aliases)) + append!(hosts, localhost_aliases) end - end + # CI machines sometimes don't already have a .ssh directory + ssh_dir = joinpath(homedir(), ".ssh") + if !isdir(ssh_dir) + mkdir(ssh_dir) + end + + print("\nTesting SSH addprocs with $(length(hosts)) workers...\n") + new_pids = addprocs_with_testenv(hosts; sshflags=sshflags) + @test length(new_pids) == length(hosts) + test_n_remove_pids(new_pids) + + print("\nMixed ssh addprocs with :auto\n") + new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags) + @test length(new_pids) == (2 + Sys.CPU_THREADS) + test_n_remove_pids(new_pids) + + print("\nMixed ssh addprocs with numeric counts\n") + new_pids = addprocs_with_testenv([("localhost", 2), ("127.0.0.1", 2), "localhost"]; sshflags=sshflags) + @test length(new_pids) == 5 + test_n_remove_pids(new_pids) + + print("\nssh addprocs with tunnel\n") + new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, sshflags=sshflags) + @test length(new_pids) == num_workers + test_n_remove_pids(new_pids) + + print("\nssh addprocs with tunnel (SSH multiplexing)\n") + new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, multiplex=true, sshflags=sshflags) + @test length(new_pids) == num_workers + controlpath = joinpath(ssh_dir, "julia-$(ENV["USER"])@localhost:2222") + @test issocket(controlpath) + test_n_remove_pids(new_pids) + @test :ok == timedwait(()->!issocket(controlpath), 10.0; pollint=0.5) + + print("\nAll supported formats for hostname\n") + h1 = "localhost" + user = ENV["USER"] + h2 = "$user@$h1" + h3 = "$h2:2222" + h4 = "$h3 $(string(getipaddr()))" + h5 = "$h4:9300" + + new_pids = addprocs_with_testenv([h1, h2, h3, h4, h5]; sshflags=sshflags) + @test length(new_pids) == 5 + test_n_remove_pids(new_pids) + + print("\nkeyword arg exename\n") + for exename in [`$(joinpath(Sys.BINDIR, Base.julia_exename()))`, "$(joinpath(Sys.BINDIR, Base.julia_exename()))"] + for addp_func in [()->addprocs_with_testenv(["localhost"]; exename=exename, exeflags=test_exeflags, sshflags=sshflags), + ()->addprocs_with_testenv(1; exename=exename, exeflags=test_exeflags)] + + local new_pids = addp_func() + @test length(new_pids) == 1 + test_n_remove_pids(new_pids) + end + end + end end # unix-only -end # full-test let t = @task 42 schedule(t, ErrorException(""), error=true) From 56b0b6c278b718e6878e0b0ecdf5ed03adf894c5 Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Fri, 25 Oct 2024 19:32:25 +0200 Subject: [PATCH 4/4] Get the docs building and add a changelog Also add Julia 1 to the CI --- .github/workflows/ci.yml | 17 +++--- .gitignore | 1 + docs/Project.toml | 7 +++ docs/make.jl | 50 ++++++++++++----- docs/src/_changelog.md | 14 +++++ docs/src/index.md | 114 +++++++++++++++++++-------------------- src/DistributedNext.jl | 3 +- 7 files changed, 128 insertions(+), 78 deletions(-) create mode 100644 .gitignore create mode 100644 docs/src/_changelog.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6a1e35..d4104fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ concurrency: # Cancel intermediate builds: only pull request builds group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }} cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -23,6 +24,7 @@ jobs: matrix: version: - 'nightly' + - '1' os: - ubuntu-latest - macOS-latest @@ -58,18 +60,17 @@ jobs: with: file: lcov.info token: ${{ secrets.CODECOV_TOKEN }} + docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - # version: '1.6' - version: 'nightly' - - name: Generate docs - run: | - julia --color=yes -e 'write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"47e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\n"));' - julia --project --color=yes -e 'using Pkg; Pkg.activate("docs"); Pkg.instantiate(); Pkg.develop(PackageSpec(path = pwd()))' - julia --project=docs --color=yes docs/make.jl pdf + version: '1' + - name: Install dependencies + run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - name: Build and deploy env: - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia --project=docs/ docs/make.jl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..de6b1c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docs/src/changelog.md diff --git a/docs/Project.toml b/docs/Project.toml index dfa65cd..bdc70e9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,9 @@ [deps] +Changelog = "5217a498-cd5d-4ec6-b8c2-9b85a09b6e3e" +DistributedNext = "fab6aee4-877b-4bac-a744-3eca44acbb6f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" + +[sources] +DistributedNext = {path = ".."} diff --git a/docs/make.jl b/docs/make.jl index 9b652ca..3046f93 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,16 +1,42 @@ -using Distributed -using Documenter: DocMeta, makedocs, deploydocs +import Revise +import Changelog +using DistributedNext +import Documenter +using Documenter: Remotes, DocMeta, makedocs, deploydocs -DocMeta.setdocmeta!(Distributed, :DocTestSetup, :(using Distributed); recursive=true) +DocMeta.setdocmeta!(DistributedNext, :DocTestSetup, :(using DistributedNext); recursive=true) -makedocs(; - modules = [Distributed], - sitename = "Distributed", - pages = Any[ - "Distributed" => "index.md", - ], - checkdocs = :exports, - warnonly = [:cross_references], +# Always trigger a revise to pick up the latest docstrings. This is useful when +# working with servedocs(). If you are using servedocs(), run it like this: +# +# julia> servedocs(; include_dirs=["src"], skip_files=["docs/src/changelog.md"]) +# +# Otherwise it'll get into an infinite loop as the changelog is constantly +# regenerated and triggering LiveServer. +Revise.revise() + +# Build the changelog. Note that _changelog.md is the source and changelog.md is +# the destination. It's named that way for the vain reason of a nicer URL. +Changelog.generate( + Changelog.Documenter(), + joinpath(@__DIR__, "src/_changelog.md"), + joinpath(@__DIR__, "src/changelog.md"), + repo="JuliaParallel/DistributedNext.jl" ) -deploydocs(repo = "github.com/JuliaLang/Distributed.jl.git") +makedocs(; + repo = Remotes.GitHub("JuliaParallel", "DistributedNext.jl"), + format = Documenter.HTML( + prettyurls=get(ENV, "CI", "false") == "true", + size_threshold_warn=500_000, + size_threshold=600_000), + modules = [DistributedNext], + sitename = "DistributedNext", + pages = [ + "DistributedNext" => "index.md", + "changelog.md" + ], + warnonly = [:missing_docs, :cross_references], + ) + +deploydocs(repo = "github.com/JuliaParallel/DistributedNext.jl.git") diff --git a/docs/src/_changelog.md b/docs/src/_changelog.md new file mode 100644 index 0000000..014863b --- /dev/null +++ b/docs/src/_changelog.md @@ -0,0 +1,14 @@ +```@meta +CurrentModule = DistributedNext +``` + +# Changelog + +This documents notable changes in DistributedNext.jl. The format is based on +[Keep a Changelog](https://keepachangelog.com). + +## Unreleased + +### Changed +- Added a `project` argument to [`addprocs(::AbstractVector)`](@ref) to specify + the project of a remote worker ([#2]). diff --git a/docs/src/index.md b/docs/src/index.md index 22d63ce..fd88707 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,53 +1,53 @@ # [Distributed Computing](@id man-distributed) ```@docs -Distributed -Distributed.addprocs -Distributed.nprocs -Distributed.nworkers -Distributed.procs() -Distributed.procs(::Integer) -Distributed.workers -Distributed.rmprocs -Distributed.interrupt -Distributed.myid -Distributed.pmap -Distributed.RemoteException -Distributed.ProcessExitedException -Distributed.Future -Distributed.RemoteChannel -Distributed.fetch(::Distributed.Future) -Distributed.fetch(::RemoteChannel) -Distributed.remotecall(::Any, ::Integer, ::Any...) -Distributed.remotecall_wait(::Any, ::Integer, ::Any...) -Distributed.remotecall_fetch(::Any, ::Integer, ::Any...) -Distributed.remote_do(::Any, ::Integer, ::Any...) -Distributed.put!(::RemoteChannel, ::Any...) -Distributed.put!(::Distributed.Future, ::Any) -Distributed.take!(::RemoteChannel, ::Any...) -Distributed.isready(::RemoteChannel, ::Any...) -Distributed.isready(::Distributed.Future) -Distributed.AbstractWorkerPool -Distributed.WorkerPool -Distributed.CachingPool -Distributed.default_worker_pool -Distributed.clear! -Distributed.remote -Distributed.remotecall(::Any, ::AbstractWorkerPool, ::Any...) -Distributed.remotecall_wait(::Any, ::AbstractWorkerPool, ::Any...) -Distributed.remotecall_fetch(::Any, ::AbstractWorkerPool, ::Any...) -Distributed.remote_do(::Any, ::AbstractWorkerPool, ::Any...) -Distributed.@spawn -Distributed.@spawnat -Distributed.@fetch -Distributed.@fetchfrom -Distributed.@distributed -Distributed.@everywhere -Distributed.remoteref_id -Distributed.channel_from_id -Distributed.worker_id_from_socket -Distributed.cluster_cookie() -Distributed.cluster_cookie(::Any) +DistributedNext +DistributedNext.addprocs +DistributedNext.nprocs +DistributedNext.nworkers +DistributedNext.procs() +DistributedNext.procs(::Integer) +DistributedNext.workers +DistributedNext.rmprocs +DistributedNext.interrupt +DistributedNext.myid +DistributedNext.pmap +DistributedNext.RemoteException +DistributedNext.ProcessExitedException +DistributedNext.Future +DistributedNext.RemoteChannel +DistributedNext.fetch(::DistributedNext.Future) +DistributedNext.fetch(::RemoteChannel) +DistributedNext.remotecall(::Any, ::Integer, ::Any...) +DistributedNext.remotecall_wait(::Any, ::Integer, ::Any...) +DistributedNext.remotecall_fetch(::Any, ::Integer, ::Any...) +DistributedNext.remote_do(::Any, ::Integer, ::Any...) +DistributedNext.put!(::RemoteChannel, ::Any...) +DistributedNext.put!(::DistributedNext.Future, ::Any) +DistributedNext.take!(::RemoteChannel, ::Any...) +DistributedNext.isready(::RemoteChannel, ::Any...) +DistributedNext.isready(::DistributedNext.Future) +DistributedNext.AbstractWorkerPool +DistributedNext.WorkerPool +DistributedNext.CachingPool +DistributedNext.default_worker_pool +DistributedNext.clear! +DistributedNext.remote +DistributedNext.remotecall(::Any, ::AbstractWorkerPool, ::Any...) +DistributedNext.remotecall_wait(::Any, ::AbstractWorkerPool, ::Any...) +DistributedNext.remotecall_fetch(::Any, ::AbstractWorkerPool, ::Any...) +DistributedNext.remote_do(::Any, ::AbstractWorkerPool, ::Any...) +DistributedNext.@spawn +DistributedNext.@spawnat +DistributedNext.@fetch +DistributedNext.@fetchfrom +DistributedNext.@distributed +DistributedNext.@everywhere +DistributedNext.remoteref_id +DistributedNext.channel_from_id +DistributedNext.worker_id_from_socket +DistributedNext.cluster_cookie() +DistributedNext.cluster_cookie(::Any) ``` ## Cluster Manager Interface @@ -58,14 +58,14 @@ same host, and `SSHManager`, for launching on remote hosts via `ssh`. TCP/IP soc and transport messages between processes. It is possible for Cluster Managers to provide a different transport. ```@docs -Distributed.ClusterManager -Distributed.WorkerConfig -Distributed.launch -Distributed.manage -Distributed.kill(::ClusterManager, ::Int, ::WorkerConfig) -Distributed.connect(::ClusterManager, ::Int, ::WorkerConfig) -Distributed.init_worker -Distributed.start_worker -Distributed.process_messages -Distributed.default_addprocs_params +DistributedNext.ClusterManager +DistributedNext.WorkerConfig +DistributedNext.launch +DistributedNext.manage +DistributedNext.kill(::ClusterManager, ::Int, ::WorkerConfig) +DistributedNext.connect(::ClusterManager, ::Int, ::WorkerConfig) +DistributedNext.init_worker +DistributedNext.start_worker +DistributedNext.process_messages +DistributedNext.default_addprocs_params ``` diff --git a/src/DistributedNext.jl b/src/DistributedNext.jl index f229519..d5b66e9 100644 --- a/src/DistributedNext.jl +++ b/src/DistributedNext.jl @@ -1,7 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license """ -Tools for distributed parallel processing. +Tools for distributed parallel processing. This is a soft fork of Distributed.jl +for the purposes of testing new things before merging upstream. Here be dragons! """ module DistributedNext