build fails due to missing libpq

[davepacheco]

tl;dr: On Helios, to build "main" after #192, you should:

• install pkg install library/postgresql-13
• make sure /opt/ooce/bin is on your PATH
• build with RUSTFLAGS="-Clink-args=-R$(pg_config --libdir)" in the environment

Below are my notes from debugging this. Much thanks to @rmustacc @wesolows @jclulow @luqmana and @bnaecker for help with debugging this and finding possible workarounds. At the end there's a discussion of different approaches we could take, but my conclusion for now is we should just tell people to set RUSTFLAGS.

Debugging the problem

On Helios, a full build-and-test from a clean clone currently fails.

Here's an example from @bnaecker:

bnaecker@shale:~/omicron $ cargo build
   Compiling omicron-common v0.1.0 (/home/bnaecker/omicron/omicron-common)
   Compiling oximeter v0.1.0 (/home/bnaecker/omicron/oximeter/oximeter)
   Compiling omicron-bootstrap-agent v0.1.0 (/home/bnaecker/omicron/omicron-bootstrap-agent)
   Compiling omicron-sled-agent v0.1.0 (/home/bnaecker/omicron/omicron-sled-agent)
   Compiling omicron-nexus v0.1.0 (/home/bnaecker/omicron/omicron-nexus)
error: linking with `gcc` failed: exit status: 1
  |
  = note: "gcc" "-m64" "-std=c99" "-L" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.124wgd4g5jpd1ng9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1449uq9hbxse03sj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.14bc0js9nzygb57e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.15rf7spb3mzibb7c.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.164y1n0fr90gophb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.16zojk3gzndy5uot.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.18cdlh4tr5uuvlge.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.19enev67dprqsv2s.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.19q3746jf7kx3y3o.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1bbckr20cjanztqg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1ciw7qh2nadlb1vz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1crj13yjoin6jm0p.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1hxtl05dcq0zoov.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1hy54k16rr43ewq7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1j1bhbinpg0j38yt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1jpdz0bfvlvgu22l.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1k4hiapz6qhyabsz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1kba2k0zfshs5mym.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1l5ucwjrfzn0j4jv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1lkfg7laeedq96y6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1or3748zay3rnx68.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1p1x0nybg6i3u52x.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1pke2x3txi440ptm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1qfvfhi601bfh6yr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1tds0jdjd45g7xo2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1tn60kjscvtm81wb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1va4c0o1asibbjwx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1vxsolyuliyyudsc.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1w9sig5oxo3gspwe.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1wl0mwslitoialqc.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1y00lm5yjx5b0qr1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.200z3x0laxds9j6v.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.209vru8sa59sehth.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.20wzrbljst5egohk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.21orgahydcusrzcm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.227ne0fi1tk7ivvb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.22iaspdomkc1rtzk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.22pdteaax7fchmkl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.22zh3ufxbdpl4iy3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.241f16ejkzn02gjo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.24ajyfval4uuhmwh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.25llpvr7fu4cesdu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2698g0rvc0be1mrd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.27g8xbb5ih6bj8ke.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.27hmex7ufqe8gq8e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.28hyp15z04mjb8d9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.294v4pzgbtp0j8nn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.29bxtrjg0vd1cbuj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.29l448kb4hpg3zgk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2a7q1bo8hv6lekk5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2bceg3ykst4vd9vk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2ch6cdb6l1asrbny.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2dgygjv769psg8l3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2dl7rs4gz06gq5fq.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2e6tdsjl15hpcp6r.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2f225dugk0he0muf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2g45ikmeloq3nzzy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2gfi9q51ur7lgra9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2h2x0bo2opfxdyn8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2h8j56hq2kihzlnt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2htksv4lie50g17i.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2jqaehfdg6l1g5oy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2lw8vcy2klefbrmt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2m6os8wqyydqky3z.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2n7ap7l3te6nxm2q.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2obj8foy5e2zgdo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2paj4599au0o646d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2qdv3b9hnecm22zt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2qesoxigvnbmg70g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2qnow36nmkvpterh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2r7ftrt0r24p0ngj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2t2wn6x4zchj8awk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2tj97zqv7gog6yqy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2v83qlz7nc2evk2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2vnd224ikzg1s8jz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2vyumedzsmra2r71.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2w6d4jtvazaju7hd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2wxhzepf0s6i1rj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2wzx1gsmcfnm4ocr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.2zekk6jb22idux94.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.30jyje4djgx5kchw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.30uc12mwlbf4nigc.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.318f0nda1rx5ciyx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.31gwp237s3876cwd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.31hajbk8vr9ntynl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.321w5bf28auf9kf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3303yr8m7vg3j0cp.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.337ihaxn7nxrslui.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.33w5rlwm2d81tm3x.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.343mtsfe6mzjvax8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.34ovfh5x08zqas4g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.35m64j33j8s3unr3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3c3ftjy161jq650l.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3cpf4hy0cko2vh62.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3d3ty6yj7vw3t12i.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3d80k5jlpw7ckibt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3e531grps1hmsvut.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3hhkuzc3t58dugca.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3hyxx8zr80qjt4kz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3iri2i99q1w2m4ps.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3jg3ij2j2uqx6rjf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3l9ncw12std8sjk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3n87d4u14o9m9e5n.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3pkoa0nvwt7yj0mr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3r1defcw413a0eb1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3tt325hg89wkerw9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3tz187izqjny6ye9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3ua1w7vim0tagvvq.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3v2wvrx3hk5pcqwg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3v3deauy6dix4r3a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3vvxc5gaf84juxje.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3xer1cduar50d8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3ymcozh6xk4v5mdv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.3ytemfufldtf8idw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.40uokr506qhgti40.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4113iq6sdffc1rks.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.42pgiy9n6tjovl56.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.431pgohfrn3bykv1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.441kz40wyjwhyszr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.44aqtboxpb4ax2fl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.45l4z3dn35sxas7f.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.45t7bumudarn58ws.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.47b1pmf37x3gxnhy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.48176794rzq4qrbb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4903wabjd6znsuhz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4ajytmclhwyyen6e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4cx4zsl13s1b2ce0.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4cxzpl8zwtdbea5f.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4e2yvyz44xa87xif.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4fztcc7iirlbll13.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4gkbmfizml238i4h.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4gxfejiz0xru352j.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4h62otxfmhzre52q.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4hpyjw6wlozpwz09.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4j38daagapkc8di0.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4j41v1iqqnhgr68s.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4jl3ve41l3ysh46g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4jvlwp2eezxkl2p1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4ka7so3gjkftg3zl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4kez24qb4gxeues5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4kkk8gdhl5drtqdg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4l5rpazbbmntt035.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4l7h4xokbl6h4hsj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4ll36cjba33o569b.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4m0veq0vni3zye91.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4n8e4zjsk7muulwm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4npe6jgkm9tkcnf8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4om4ad4kfta6yhee.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4rhvb0m7i2suxa32.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4tgal9qddogxfalq.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4u4ko0iqtbzfm4uu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4vthtdmayxmiwjef.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4y3kubm7s9dm5vfb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4ydx7crz1woxyjcj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4ytlj5ei6zf65p5e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.4zp9xjb4fxddcvx3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.51lrhnomqaz5y07r.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.51pb8v17ouj6mprf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.52f9bhgld9ccb3a9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.52k5glbhjsmzdhka.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.52y50o6ejqsrufdo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.53zkti5jtwa2pxvm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.54l082sjfy8id0ve.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.54l7d8zpk38vt1id.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.55jsworqzc157frl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.57i9y2d5ugfh3rr9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.57pv037ssg3p9wxl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.57xiypmuz5n6j915.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.586oqnyk44kk2r77.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.58jux48nb07pgill.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.59k4wpn5z8k221lo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5btdi2kcmmaoahcz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5e2c56ruc05434gm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5ebhreeguy7xw8p7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5fa96f9g3w4tb5cr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5g4wbk0z0qj9vv0.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.5gatzvtutahs47js.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.6ouenthz3vjxrjb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.6uel83ves86f76i.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.88yl3pwnymv0mlf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.8t39oa5s7z59r42.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.8vkc63x5d7kql4s.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.a0jh4ygt9cnldvw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.auu6qxr9yvkod5w.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.c1amo1pw93cz7t5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.ddmntzyzuzh2hii.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.dypyv2uoyae4tn3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.hexos2cmooubt8w.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.hknl6p01hemjal4.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.i72vw11gp4st65x.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.icc3uecyqlgpz3z.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.j50v8ijeuui9776.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.jgf64tdky4fjro9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.k9ta3e6mfu6tgqh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.lemq8ujj4tk808b.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.lqsbbamzn2px2ei.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.lxblw7vu4nzfbsf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.mudvgaskxec2tbr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.o50b3piondcb1k9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.ovl66hi9o1oqh22.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.rdlqjhxcyqxut0m.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.rqonzqkzjr4xw17.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.ttdh361mdfv1l7h.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.usmsk9dgxtim6m3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.wdh8bkpy8pzvms1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.yuyfyavgbg3qrkl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.zi3879wnzg8jsj8.rcgu.o" "-o" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0" "/home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0.1uqp2y998ymu42lw.rcgu.o" "-Wl,-zignore" "-nodefaultlibs" "-L" "/home/bnaecker/omicron/target/debug/deps" "-L" "/home/bnaecker/omicron/target/debug/build/ring-8d289739009b00bd/out" "-L" "/usr/ssl-1.1/lib" "-L" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-Wl,-Bstatic" "/home/bnaecker/omicron/target/debug/deps/libstructopt-9f20633c43b28019.rlib" "/home/bnaecker/omicron/target/debug/deps/libclap-c34a1430fa5c37ff.rlib" "/home/bnaecker/omicron/target/debug/deps/libvec_map-cd0a81e016d57752.rlib" "/home/bnaecker/omicron/target/debug/deps/libtextwrap-f19935ff84ca11bf.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_width-e46d4015063b3364.rlib" "/home/bnaecker/omicron/target/debug/deps/libstrsim-0f5d63f378ea1870.rlib" "/home/bnaecker/omicron/target/debug/deps/libansi_term-45d599bfbb0c270b.rlib" "/home/bnaecker/omicron/target/debug/deps/libsignal_hook_tokio-3e27c51296c1a5a9.rlib" "/home/bnaecker/omicron/target/debug/deps/libsignal_hook-8b0f69604a4d4a4a.rlib" "/home/bnaecker/omicron/target/debug/deps/libomicron_common-54fa11c084e38cd7.rlib" "/home/bnaecker/omicron/target/debug/deps/libmacaddr-f724f028e465f028.rlib" "/home/bnaecker/omicron/target/debug/deps/libipnet-5c0ac3e128800b4d.rlib" "/home/bnaecker/omicron/target/debug/deps/libsteno-902d75c55bf0b383.rlib" "/home/bnaecker/omicron/target/debug/deps/libpetgraph-79b4d6540678be71.rlib" "/home/bnaecker/omicron/target/debug/deps/libfixedbitset-d158b3fceb12ad06.rlib" "/home/bnaecker/omicron/target/debug/deps/libnewtype_derive-4444218dacba388d.rlib" "/home/bnaecker/omicron/target/debug/deps/libring-9db14b3618cce0b5.rlib" "/home/bnaecker/omicron/target/debug/deps/libspin-3759a12329edbdec.rlib" "/home/bnaecker/omicron/target/debug/deps/libuntrusted-bc9e05a9fae88c2a.rlib" "/home/bnaecker/omicron/target/debug/deps/libsubprocess-2553e18f15501c55.rlib" "/home/bnaecker/omicron/target/debug/deps/libtempfile-1bee9b43c6057436.rlib" "/home/bnaecker/omicron/target/debug/deps/libremove_dir_all-ac2657924204f37c.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio_postgres-6c28ac73b06aded6.rlib" "/home/bnaecker/omicron/target/debug/deps/libphf-2613fb0a9a094551.rlib" "/home/bnaecker/omicron/target/debug/deps/libphf_shared-9ee20954771c2137.rlib" "/home/bnaecker/omicron/target/debug/deps/libsiphasher-03daf110bd37b3ea.rlib" "/home/bnaecker/omicron/target/debug/deps/libpostgres_types-b4d583a31b613da0.rlib" "/home/bnaecker/omicron/target/debug/deps/libpostgres_protocol-886ccd08c0a4916c.rlib" "/home/bnaecker/omicron/target/debug/deps/libstringprep-e55cee23e72a5835.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_normalization-657069cd74b63e2f.rlib" "/home/bnaecker/omicron/target/debug/deps/libtinyvec-16f944bfad7e1580.rlib" "/home/bnaecker/omicron/target/debug/deps/libtinyvec_macros-e8bd53e72ecab495.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_bidi-6f2d2e1d10f0358b.rlib" "/home/bnaecker/omicron/target/debug/deps/libsha2-b94e0ba17a955240.rlib" "/home/bnaecker/omicron/target/debug/deps/libcpufeatures-49243c9df55c42ef.rlib" "/home/bnaecker/omicron/target/debug/deps/libhmac-95bd764875ee3c13.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrypto_mac-d8339cbf80e69737.rlib" "/home/bnaecker/omicron/target/debug/deps/libsubtle-7be91b857cfa9bae.rlib" "/home/bnaecker/omicron/target/debug/deps/libmd5-a448ef870f9aaa34.rlib" "/home/bnaecker/omicron/target/debug/deps/libopaque_debug-ddfa65a0176c4908.rlib" "/home/bnaecker/omicron/target/debug/deps/libblock_buffer-dfc932147799a878.rlib" "/home/bnaecker/omicron/target/debug/deps/libdigest-6fc485ca5b3b37a9.rlib" "/home/bnaecker/omicron/target/debug/deps/libgeneric_array-24e18cc1d23d45f9.rlib" "/home/bnaecker/omicron/target/debug/deps/libtypenum-9c3dcce0439a808e.rlib" "/home/bnaecker/omicron/target/debug/deps/libfallible_iterator-d066a3ad3e582b70.rlib" "/home/bnaecker/omicron/target/debug/deps/libthiserror-e701737b1ff01205.rlib" "/home/bnaecker/omicron/target/debug/deps/libbackoff-e9b0dc36dfebe9e3.rlib" "/home/bnaecker/omicron/target/debug/deps/librand-daa076f93226fb1a.rlib" "/home/bnaecker/omicron/target/debug/deps/librand_chacha-a94c54cea89d3a15.rlib" "/home/bnaecker/omicron/target/debug/deps/libppv_lite86-41dff59578f37541.rlib" "/home/bnaecker/omicron/target/debug/deps/librand_core-158f4979726c1db0.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_project-3e52f7404649584a.rlib" "/home/bnaecker/omicron/target/debug/deps/libdropshot-e7dc0583b215377d.rlib" "/home/bnaecker/omicron/target/debug/deps/libtoml-893967043edf29cf.rlib" "/home/bnaecker/omicron/target/debug/deps/libsyn-a0905caa51afb7ce.rlib" "/home/bnaecker/omicron/target/debug/deps/libquote-f8c0c574325bf255.rlib" "/home/bnaecker/omicron/target/debug/deps/libproc_macro2-73efbf0f09edb6f2.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_xid-d141cb0e6378bd71.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libproc_macro-1d10511f0a97d80e.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_bunyan-215cd05af18b3926.rlib" "/home/bnaecker/omicron/target/debug/deps/libhostname-7c113b066be926ed.rlib" "/home/bnaecker/omicron/target/debug/deps/libmatch_cfg-1edd338f76a4b015.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_json-1200909a70e5e588.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_async-bdeaab40f25d5ce9.rlib" "/home/bnaecker/omicron/target/debug/deps/libthread_local-5a0547b1544c03d2.rlib" "/home/bnaecker/omicron/target/debug/deps/libtake_mut-07d86cedf76aca5f.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrossbeam_channel-3edbb1981e8e8cef.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrossbeam_utils-3b3c06bce6711af3.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_term-47dc108d9b65610f.rlib" "/home/bnaecker/omicron/target/debug/deps/libatty-de8289824cdd33aa.rlib" "/home/bnaecker/omicron/target/debug/deps/libterm-2c0f8123065f814a.rlib" "/home/bnaecker/omicron/target/debug/deps/libdirs_next-77f68d2964953e14.rlib" "/home/bnaecker/omicron/target/debug/deps/libdirs_sys_next-11db7b621e7e47cd.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde_urlencoded-dee161de7e9e81e6.rlib" "/home/bnaecker/omicron/target/debug/deps/libform_urlencoded-b1150ee1b1182401.rlib" "/home/bnaecker/omicron/target/debug/deps/libmatches-53cf20fa7291f940.rlib" "/home/bnaecker/omicron/target/debug/deps/libopenapiv3-fae7c8529c77ed3e.rlib" "/home/bnaecker/omicron/target/debug/deps/libpercent_encoding-448ccbc6cfd352e5.rlib" "/home/bnaecker/omicron/target/debug/deps/libbase64-40796d4e8bc68190.rlib" "/home/bnaecker/omicron/target/debug/deps/libschemars-2de580150a84bdd6.rlib" "/home/bnaecker/omicron/target/debug/deps/libdyn_clone-85ba9ccd3c0dcc50.rlib" "/home/bnaecker/omicron/target/debug/deps/libhyper-e9810f741c1a1e68.rlib" "/home/bnaecker/omicron/target/debug/deps/libwant-14c79d1515589199.rlib" "/home/bnaecker/omicron/target/debug/deps/libtry_lock-999f30278c74b492.rlib" "/home/bnaecker/omicron/target/debug/deps/libsocket2-1f24d20d040eb0e7.rlib" "/home/bnaecker/omicron/target/debug/deps/libh2-5316b96afc4dbfa0.rlib" "/home/bnaecker/omicron/target/debug/deps/libindexmap-44ecfb14b3456c29.rlib" "/home/bnaecker/omicron/target/debug/deps/libhashbrown-62109a9f42df0bb7.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio_util-854bc67dbe14f0ac.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttparse-0d8306f5d84b761e.rlib" "/home/bnaecker/omicron/target/debug/deps/libtower_service-02d19c5cf52ac3a9.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttp_body-49a893dd1fc758ad.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio-4e5cf89f53d74dd5.rlib" "/home/bnaecker/omicron/target/debug/deps/libsignal_hook_registry-f44108f4e30e9ed3.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_cpus-6a7f94224b87ddd1.rlib" "/home/bnaecker/omicron/target/debug/deps/libonce_cell-983e777de2dffb91.rlib" "/home/bnaecker/omicron/target/debug/deps/libmio-c0e4839e4abbecd2.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttpdate-781215c87d248ee6.rlib" "/home/bnaecker/omicron/target/debug/deps/libtracing-1e6821b1349344f4.rlib" "/home/bnaecker/omicron/target/debug/deps/libtracing_core-c7feb64068d61463.rlib" "/home/bnaecker/omicron/target/debug/deps/liblazy_static-72b18b40aaa861f8.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttp-a92060c26bf96314.rlib" "/home/bnaecker/omicron/target/debug/deps/libbytes-e618e8ebf2ad59d5.rlib" "/home/bnaecker/omicron/target/debug/deps/libfnv-0710946dcf4dccf9.rlib" "/home/bnaecker/omicron/target/debug/deps/libdiesel-b9299802248d6700.rlib" "/home/bnaecker/omicron/target/debug/deps/libuuid-f66f932814fb8768.rlib" "/home/bnaecker/omicron/target/debug/deps/libgetrandom-c44851db43db6307.rlib" "/home/bnaecker/omicron/target/debug/deps/libpq_sys-8c0a8dc7cb4868e4.rlib" "/home/bnaecker/omicron/target/debug/deps/libipnetwork-6e1a7c38b21ac09a.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde_json-3abaa3c687f3e99d.rlib" "/home/bnaecker/omicron/target/debug/deps/libryu-82c19bf17a24b785.rlib" "/home/bnaecker/omicron/target/debug/deps/libitoa-b5d0dacd5d002ed8.rlib" "/home/bnaecker/omicron/target/debug/deps/libchrono-54bfc2d3fc4d0448.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde-70086fb5d9665b56.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_integer-8eedf15583bd566b.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_traits-ea87f16353a7ab96.rlib" "/home/bnaecker/omicron/target/debug/deps/libtime-dd811fb935d1c058.rlib" "/home/bnaecker/omicron/target/debug/deps/libr2d2-6a0c6cca6cffa9eb.rlib" "/home/bnaecker/omicron/target/debug/deps/libscheduled_thread_pool-03984c0ce22780a3.rlib" "/home/bnaecker/omicron/target/debug/deps/libparking_lot-464dfbd9492b5b3b.rlib" "/home/bnaecker/omicron/target/debug/deps/libparking_lot_core-4c808711e4fcbb12.rlib" "/home/bnaecker/omicron/target/debug/deps/liblibc-be707306429f040c.rlib" "/home/bnaecker/omicron/target/debug/deps/libsmallvec-348db80a3e827aa0.rlib" "/home/bnaecker/omicron/target/debug/deps/liblock_api-bfa80e01a6cac024.rlib" "/home/bnaecker/omicron/target/debug/deps/libscopeguard-31a2cdce41286a7b.rlib" "/home/bnaecker/omicron/target/debug/deps/libinstant-ae2c96832c229c11.rlib" "/home/bnaecker/omicron/target/debug/deps/liblog-409a956358a69afe.rlib" "/home/bnaecker/omicron/target/debug/deps/libcfg_if-363f0f165151a264.rlib" "/home/bnaecker/omicron/target/debug/deps/libbyteorder-fd80698730bdd288.rlib" "/home/bnaecker/omicron/target/debug/deps/libbitflags-680fe52bdfbe201e.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog-39ca96abbd588a0d.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures-fe717c0ade44bf0c.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_executor-be1450fb00b7d55d.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_util-4d37b0d16b9220e0.rlib" "/home/bnaecker/omicron/target/debug/deps/libmemchr-4597f23a46861c74.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_io-991c30c196480e3e.rlib" "/home/bnaecker/omicron/target/debug/deps/libslab-78cbfc437c8c17b8.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_channel-e76b0616938b79b0.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_project_lite-d5107c8292b276ac.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_sink-19205779bcb95883.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_task-3ed7cd0636938d03.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_utils-0b876b1a68196637.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_core-7c23d4ac2632764f.rlib" "/home/bnaecker/omicron/target/debug/deps/libanyhow-067e0166c6c1f99f.rlib" "-Wl,--start-group" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd-df551240bd83d316.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libpanic_abort-0c5f20c3095dc6fd.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libminiz_oxide-d28cbea3beaa15a4.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libadler-422dd4d469ec938c.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libobject-261e9a8c17cad15e.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libaddr2line-bef829f29e00713d.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgimli-a090674e9ae21902.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd_detect-426e08f9caf54663.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_demangle-ca65e69f4f320e51.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libhashbrown-38da2ffd4ba6e106.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_alloc-ad8091dae1980f64.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunwind-390453163eb4e652.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcfg_if-de11336c0160e7a6.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liblibc-066d8902c3f391d2.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liballoc-40d2c2423d4f0bf4.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_core-45d83066ecf05e1b.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcore-2a95b4624d21aa7e.rlib" "-Wl,--end-group" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcompiler_builtins-13e55be1ea183f77.rlib" "-Wl,-Bdynamic" "-lpq" "-lsocket" "-lposix4" "-lpthread" "-lresolv" "-lnsl" "-lumem" "-lgcc_s" "-lc" "-lm" "-lrt" "-lpthread" "-lssp"
  = note: ld: fatal: library -lpq: not found
          ld: fatal: file processing errors. No output written to /home/bnaecker/omicron/target/debug/deps/omicron_dev-607734382f38c2b0
          collect2: error: ld returned 1 exit status


error: linking with `gcc` failed: exit status: 1
  |
  = note: "gcc" "-m64" "-std=c99" "-L" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.10rv1ciosdb3wwsv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.10t3n7zral44bhqf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.10wyagpnc7ivjxdb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.10z0z9sm6fp9kwzt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1134zvzqjsyo1106.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.11y0y4voa7a5csy8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.11zq8dayu6hpjona.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.12de94o2b1lmt1m6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.13xg6sc2m73krxpn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.13yazsm2c1g72z65.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.13zoafmnr8akv3zj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.15lcy2idpfmvpg2g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.16htr7c1f5oy5ipd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.16k8gdflenqw9qd1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.17v00fncxadmr0rj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.18bspk38400r57dt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.18iha4tdf3qqrjpi.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.18qoneot903f32sy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1atmw7uwyaxjclbr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1b4a2559yzd60xwm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1c9fp2s4vfceftxo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1clbirvz7ie92woh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1dc1emhe168ejto3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1dkm921o8pndcioj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1dveaymfxsyl3ygb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1eotytjdgg5e37a1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1fiz7ip3k281293j.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1fv84h5nyc3f05c6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1g2cf9obcw8q4fcw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1hjcn883r5awz3tp.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1hlitzs3imr40hel.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1hx8xc16r07bnx1d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1j70npmi05qtap8d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1j7d7oydhdj1094i.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1j85djz1bjtwlgoa.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1knuo2bfdppj6u70.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1lreck71mnj4yxhm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1lrpbuni2dntnsep.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1lvly65txdk2n9xn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1nc72vnx556e0kgk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1nlcyam9lg1fwwkk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1nmrw4mofsl8iyyz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1rd5chy0d59ajpp8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1vphsx6bq7g3wazv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1vqtckpv3wnafjdo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1wmgqzg6hsenuw1z.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1x5x7i4y7tstzdnn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1yg8erxzgwr3hh5h.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1yr0lx1giahlgfmn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1yvt8snxkm06k4o1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1z7rbua2dl76hqjs.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1zvhztxwjh2731vr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.1zw8tqaed8fxso1c.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.20okwt2ytuawg1m6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.21xrw0q3afs3qxf9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.21ygmmjwrml9dssp.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.227k28spy1o3vkgx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.24e8yvbmznoubem1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.253ecwbwbezyml95.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.263c01lkfrwm2mqz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.26u3gchmozimhaed.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.26wy6qqbv8gaynep.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.284w7drrq8gosj0a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.28yd10io985drv6a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2al7m2lhwgj6j2xd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2d5iug0rk9udf50v.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2dqxcwknmyt0jpgr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2ern5n8nadaeb2rh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2exa4r591c1xbxt0.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2flgs6rjlzmuqf83.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2hbpyglq58pcywca.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2jctjrwykkpwcqc9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2ju843i9j5heu0vb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2k1dayjdf527659e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2l2qjhz0qrcixu6b.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2l5hy2pgb91er7d3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2lbgu84cmdlc2a15.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2ljylos4loeo75tl.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2mccn0i8s1ut3pc0.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2o5mfuj8amckaabm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2o82v86ontyqm9uz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2ptvo69atpslbg9p.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2q7ffi52hl59phze.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2qb0s16vcifnh7c6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2qlrjpuqktgy9zso.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2qsz0r628vsv1jf2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2s1de79inkaqbckb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2si0vv25j0rej1fu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2skqwndbekafatl7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2twnc21sj37uvxvv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2vdmnhoq0joon6d4.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2vj7ol9gls7dv94e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2y6uopmx73xvv4z1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2yk1xg0les51varb.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.2zkwiyvlomze7arx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.30ba4pwuumz1hqv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.30jk7qx3zzdp6x7d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.30z5sxpszelugnnk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.31fe83zhe91r8xe7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.31pypijg6qwr1koe.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.328hb4jn4ab49qt6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.330qgvi1w8gp90e5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3431xh6sebcm7ut6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.34mxmp8acbmiaxph.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.36onicla5fj1c6t.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.37cveknvcd9nt1h1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.37kgftzn6dsn89ig.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.37to3q5ejyqtzli2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.37vwl1zjx83ylh34.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.38d5vs7hdzttmpch.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3aoitzfdiz59u5te.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3aozuypbjx2pi1b8.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3c7mvvfc219t74lh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3crmx10cvasu07tw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3e3l0sod8flj4o4d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3h85rtlagere5l7d.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3hl4gbntqde319bu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3hoqm7j45d041oeg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3jx9oi8hz4crnccm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3ksmxlzmm3lj4egg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3l9r203it4lyzkr1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3m3gkg2e8nhqydhe.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3nbi8nfurcl0ik0s.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3nojygec7irin3of.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3p4rqx54yddj3xk1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3pcvkfvnqwcittx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3r9ax5fjzcunr16p.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3rbdgisnf2x440oa.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3s5quh56jiolcccy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3t7rpk9xml2zjtpr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3u8yqunsojmut43g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3uqph7uxagqic9pj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3x6u6xenip4nawgm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3xf30wdlyv4ync0e.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3y360mmzs0qf66hk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3z7mu2d4kqivcgjk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3znybi37zkv6j8q2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.3zqfls0bvenw04kn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.403wfpoz8zawyl37.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.40b8sydx7tnuvbwm.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.424qwtchc7zqrai9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.452cp7a1tlvj0hng.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.45gfv8bcfiniirqt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.45hbvpzymjjosqf9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.48n8amiyqxi0nyry.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4c0qmuyx6u8nmu2w.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4dlb24hbir36svgf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4dlq4id5jkkilbsz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4eqfv7gpm69eh47p.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4f931e9f7n58rqoa.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4g6nu8a1d1pdaokk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4guv3n6e01d7a8a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4gvfxhizbkbt8coc.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4ij76xw5msaam6u7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4jh2j60em9aex9vs.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4kjcr9otepee9znw.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4kp9yc1i71705y8g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4leztvpcl6zfdm2q.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4lvh7vocdvbduekf.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4nraswrxhj66xs3b.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4pvw8bpjilg9p7yv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4q0m4bpge0tsps7q.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4qwb6rcfrp9up6v7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4tg740oxsp88g6ga.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4tka5o3tmi2zxqj3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4u0ls1vq3cxdnbyq.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4ubg2yiynxar1r43.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4w0f70tvnrrcgep5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4wmr8na2ek30in9a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4ybo0wo4iqh7j8ez.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4yyzqtu32fsxd1oy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4zptzcxot0hv7gqu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4ztqp77ey1mm2zx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.51adn2h7pz3nnr1k.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.51ouyibflqb31bu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.51sz23n09w0zj5mu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.52yn5b7b0i9m7okd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.54v2bm220qz97tr6.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.55x8jf9m6onnkow9.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.56pum671bwcpeoh7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.56wobgsdjmc06nqc.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.57w5luu0l91m8z2c.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.58e20k35d48opg5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.58qs2v9w6emw0hh1.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.59x5eed39yjlx67g.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5bbo33yaffgjv71c.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5bcob1l05mudvyjg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5becxh98k9wuk85x.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5c42vrwzmnhi7gyk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5c6jw0prlaqfyx35.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5cbdxa6ay37z647j.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5dhlybtuj5nngzov.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5e0klay01k95y3nd.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5eknc63oibo84678.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5eudmcr1y5ohi8wy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5f37m8wx7d1t5cwt.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.5fja70134a5di3cu.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.6gl09hzpm66muf2.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.6yiglerhkk9yqen.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.90ulnf5md1qtl0k.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.9cbfqa8zsy1vxyn.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.9yhir2s7swofx6z.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.b7njxy1p76begy5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.c1i7i4copufv7y7.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.dnzh58wjxx7bfxg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.es8b8dt9ncwnkjv.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.f5qa80j01l08ksa.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.f64l5p5mx8r5ahy.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.famz1ul96pgrmrx.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.fpavtf48f8a6d6x.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.h5adb7ab86m4d2f.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.hnl92sdbn52n0hg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.ibwey2j5ppqgquo.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.kxxrxwgrhdue9kr.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.ljj1zdv5c96gn6a.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.lys2ac80hqlm9gs.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.p9zfaspblpt59mk.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.q2563bkfh601lle.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.qfvxm8wz60y1jhh.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.rqvim4gkkaxvjlz.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.sltlci4306k4y2v.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.tc3onvizkv1hkf3.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.tgtia4c5l36i3wp.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.tx2flwfah444weg.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.vkhsz41b1wq8ir5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.vpdoe3qia82n7d5.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.xdf0bn2mjx7luj.rcgu.o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.y80copem9fwu5e3.rcgu.o" "-o" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef" "/home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef.4b5ohv6lsg759cjy.rcgu.o" "-Wl,-zignore" "-nodefaultlibs" "-L" "/home/bnaecker/omicron/target/debug/deps" "-L" "/home/bnaecker/omicron/target/debug/build/ring-8d289739009b00bd/out" "-L" "/usr/ssl-1.1/lib" "-L" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-Wl,-Bstatic" "/home/bnaecker/omicron/target/debug/deps/libstructopt-9f20633c43b28019.rlib" "/home/bnaecker/omicron/target/debug/deps/libclap-c34a1430fa5c37ff.rlib" "/home/bnaecker/omicron/target/debug/deps/libvec_map-cd0a81e016d57752.rlib" "/home/bnaecker/omicron/target/debug/deps/libtextwrap-f19935ff84ca11bf.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_width-e46d4015063b3364.rlib" "/home/bnaecker/omicron/target/debug/deps/libstrsim-0f5d63f378ea1870.rlib" "/home/bnaecker/omicron/target/debug/deps/libansi_term-45d599bfbb0c270b.rlib" "/home/bnaecker/omicron/target/debug/deps/libomicron_bootstrap_agent-b016f5edc255ed06.rlib" "/home/bnaecker/omicron/target/debug/deps/libsmf-853a06e81c643f14.rlib" "/home/bnaecker/omicron/target/debug/deps/libtar-d7165835c4042453.rlib" "/home/bnaecker/omicron/target/debug/deps/libxattr-120b2bc311b38fc3.rlib" "/home/bnaecker/omicron/target/debug/deps/libfiletime-5a5a47e43ef42d4e.rlib" "/home/bnaecker/omicron/target/debug/deps/libomicron_common-54fa11c084e38cd7.rlib" "/home/bnaecker/omicron/target/debug/deps/libmacaddr-f724f028e465f028.rlib" "/home/bnaecker/omicron/target/debug/deps/libipnet-5c0ac3e128800b4d.rlib" "/home/bnaecker/omicron/target/debug/deps/libsteno-902d75c55bf0b383.rlib" "/home/bnaecker/omicron/target/debug/deps/libpetgraph-79b4d6540678be71.rlib" "/home/bnaecker/omicron/target/debug/deps/libfixedbitset-d158b3fceb12ad06.rlib" "/home/bnaecker/omicron/target/debug/deps/libnewtype_derive-4444218dacba388d.rlib" "/home/bnaecker/omicron/target/debug/deps/libring-9db14b3618cce0b5.rlib" "/home/bnaecker/omicron/target/debug/deps/libspin-3759a12329edbdec.rlib" "/home/bnaecker/omicron/target/debug/deps/libuntrusted-bc9e05a9fae88c2a.rlib" "/home/bnaecker/omicron/target/debug/deps/libsubprocess-2553e18f15501c55.rlib" "/home/bnaecker/omicron/target/debug/deps/libtempfile-1bee9b43c6057436.rlib" "/home/bnaecker/omicron/target/debug/deps/libremove_dir_all-ac2657924204f37c.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio_postgres-6c28ac73b06aded6.rlib" "/home/bnaecker/omicron/target/debug/deps/libphf-2613fb0a9a094551.rlib" "/home/bnaecker/omicron/target/debug/deps/libphf_shared-9ee20954771c2137.rlib" "/home/bnaecker/omicron/target/debug/deps/libsiphasher-03daf110bd37b3ea.rlib" "/home/bnaecker/omicron/target/debug/deps/libpostgres_types-b4d583a31b613da0.rlib" "/home/bnaecker/omicron/target/debug/deps/libpostgres_protocol-886ccd08c0a4916c.rlib" "/home/bnaecker/omicron/target/debug/deps/libstringprep-e55cee23e72a5835.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_normalization-657069cd74b63e2f.rlib" "/home/bnaecker/omicron/target/debug/deps/libtinyvec-16f944bfad7e1580.rlib" "/home/bnaecker/omicron/target/debug/deps/libtinyvec_macros-e8bd53e72ecab495.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_bidi-6f2d2e1d10f0358b.rlib" "/home/bnaecker/omicron/target/debug/deps/libsha2-b94e0ba17a955240.rlib" "/home/bnaecker/omicron/target/debug/deps/libcpufeatures-49243c9df55c42ef.rlib" "/home/bnaecker/omicron/target/debug/deps/libhmac-95bd764875ee3c13.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrypto_mac-d8339cbf80e69737.rlib" "/home/bnaecker/omicron/target/debug/deps/libsubtle-7be91b857cfa9bae.rlib" "/home/bnaecker/omicron/target/debug/deps/libmd5-a448ef870f9aaa34.rlib" "/home/bnaecker/omicron/target/debug/deps/libopaque_debug-ddfa65a0176c4908.rlib" "/home/bnaecker/omicron/target/debug/deps/libblock_buffer-dfc932147799a878.rlib" "/home/bnaecker/omicron/target/debug/deps/libdigest-6fc485ca5b3b37a9.rlib" "/home/bnaecker/omicron/target/debug/deps/libgeneric_array-24e18cc1d23d45f9.rlib" "/home/bnaecker/omicron/target/debug/deps/libtypenum-9c3dcce0439a808e.rlib" "/home/bnaecker/omicron/target/debug/deps/libfallible_iterator-d066a3ad3e582b70.rlib" "/home/bnaecker/omicron/target/debug/deps/libthiserror-e701737b1ff01205.rlib" "/home/bnaecker/omicron/target/debug/deps/libbackoff-e9b0dc36dfebe9e3.rlib" "/home/bnaecker/omicron/target/debug/deps/librand-daa076f93226fb1a.rlib" "/home/bnaecker/omicron/target/debug/deps/librand_chacha-a94c54cea89d3a15.rlib" "/home/bnaecker/omicron/target/debug/deps/libppv_lite86-41dff59578f37541.rlib" "/home/bnaecker/omicron/target/debug/deps/librand_core-158f4979726c1db0.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_project-3e52f7404649584a.rlib" "/home/bnaecker/omicron/target/debug/deps/libanyhow-067e0166c6c1f99f.rlib" "/home/bnaecker/omicron/target/debug/deps/libdiesel-b9299802248d6700.rlib" "/home/bnaecker/omicron/target/debug/deps/libpq_sys-8c0a8dc7cb4868e4.rlib" "/home/bnaecker/omicron/target/debug/deps/libipnetwork-6e1a7c38b21ac09a.rlib" "/home/bnaecker/omicron/target/debug/deps/libr2d2-6a0c6cca6cffa9eb.rlib" "/home/bnaecker/omicron/target/debug/deps/libscheduled_thread_pool-03984c0ce22780a3.rlib" "/home/bnaecker/omicron/target/debug/deps/libbyteorder-fd80698730bdd288.rlib" "/home/bnaecker/omicron/target/debug/deps/libbitflags-680fe52bdfbe201e.rlib" "/home/bnaecker/omicron/target/debug/deps/libdropshot-e7dc0583b215377d.rlib" "/home/bnaecker/omicron/target/debug/deps/libtoml-893967043edf29cf.rlib" "/home/bnaecker/omicron/target/debug/deps/libsyn-a0905caa51afb7ce.rlib" "/home/bnaecker/omicron/target/debug/deps/libquote-f8c0c574325bf255.rlib" "/home/bnaecker/omicron/target/debug/deps/libproc_macro2-73efbf0f09edb6f2.rlib" "/home/bnaecker/omicron/target/debug/deps/libunicode_xid-d141cb0e6378bd71.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libproc_macro-1d10511f0a97d80e.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_bunyan-215cd05af18b3926.rlib" "/home/bnaecker/omicron/target/debug/deps/libhostname-7c113b066be926ed.rlib" "/home/bnaecker/omicron/target/debug/deps/libmatch_cfg-1edd338f76a4b015.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_json-1200909a70e5e588.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_async-bdeaab40f25d5ce9.rlib" "/home/bnaecker/omicron/target/debug/deps/libthread_local-5a0547b1544c03d2.rlib" "/home/bnaecker/omicron/target/debug/deps/libtake_mut-07d86cedf76aca5f.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrossbeam_channel-3edbb1981e8e8cef.rlib" "/home/bnaecker/omicron/target/debug/deps/libcrossbeam_utils-3b3c06bce6711af3.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog_term-47dc108d9b65610f.rlib" "/home/bnaecker/omicron/target/debug/deps/libatty-de8289824cdd33aa.rlib" "/home/bnaecker/omicron/target/debug/deps/libterm-2c0f8123065f814a.rlib" "/home/bnaecker/omicron/target/debug/deps/libdirs_next-77f68d2964953e14.rlib" "/home/bnaecker/omicron/target/debug/deps/libdirs_sys_next-11db7b621e7e47cd.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde_urlencoded-dee161de7e9e81e6.rlib" "/home/bnaecker/omicron/target/debug/deps/libform_urlencoded-b1150ee1b1182401.rlib" "/home/bnaecker/omicron/target/debug/deps/libmatches-53cf20fa7291f940.rlib" "/home/bnaecker/omicron/target/debug/deps/libopenapiv3-fae7c8529c77ed3e.rlib" "/home/bnaecker/omicron/target/debug/deps/libpercent_encoding-448ccbc6cfd352e5.rlib" "/home/bnaecker/omicron/target/debug/deps/libbase64-40796d4e8bc68190.rlib" "/home/bnaecker/omicron/target/debug/deps/libschemars-2de580150a84bdd6.rlib" "/home/bnaecker/omicron/target/debug/deps/libdyn_clone-85ba9ccd3c0dcc50.rlib" "/home/bnaecker/omicron/target/debug/deps/libuuid-f66f932814fb8768.rlib" "/home/bnaecker/omicron/target/debug/deps/libgetrandom-c44851db43db6307.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde_json-3abaa3c687f3e99d.rlib" "/home/bnaecker/omicron/target/debug/deps/libryu-82c19bf17a24b785.rlib" "/home/bnaecker/omicron/target/debug/deps/libchrono-54bfc2d3fc4d0448.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_integer-8eedf15583bd566b.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_traits-ea87f16353a7ab96.rlib" "/home/bnaecker/omicron/target/debug/deps/libtime-dd811fb935d1c058.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures-fe717c0ade44bf0c.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_executor-be1450fb00b7d55d.rlib" "/home/bnaecker/omicron/target/debug/deps/libhyper-e9810f741c1a1e68.rlib" "/home/bnaecker/omicron/target/debug/deps/libwant-14c79d1515589199.rlib" "/home/bnaecker/omicron/target/debug/deps/libtry_lock-999f30278c74b492.rlib" "/home/bnaecker/omicron/target/debug/deps/libsocket2-1f24d20d040eb0e7.rlib" "/home/bnaecker/omicron/target/debug/deps/libh2-5316b96afc4dbfa0.rlib" "/home/bnaecker/omicron/target/debug/deps/libindexmap-44ecfb14b3456c29.rlib" "/home/bnaecker/omicron/target/debug/deps/libhashbrown-62109a9f42df0bb7.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio_util-854bc67dbe14f0ac.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttparse-0d8306f5d84b761e.rlib" "/home/bnaecker/omicron/target/debug/deps/libtower_service-02d19c5cf52ac3a9.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttp_body-49a893dd1fc758ad.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_util-4d37b0d16b9220e0.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_io-991c30c196480e3e.rlib" "/home/bnaecker/omicron/target/debug/deps/libslab-78cbfc437c8c17b8.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_channel-e76b0616938b79b0.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_sink-19205779bcb95883.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_task-3ed7cd0636938d03.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_utils-0b876b1a68196637.rlib" "/home/bnaecker/omicron/target/debug/deps/libfutures_core-7c23d4ac2632764f.rlib" "/home/bnaecker/omicron/target/debug/deps/libtokio-4e5cf89f53d74dd5.rlib" "/home/bnaecker/omicron/target/debug/deps/libsignal_hook_registry-f44108f4e30e9ed3.rlib" "/home/bnaecker/omicron/target/debug/deps/libnum_cpus-6a7f94224b87ddd1.rlib" "/home/bnaecker/omicron/target/debug/deps/libmemchr-4597f23a46861c74.rlib" "/home/bnaecker/omicron/target/debug/deps/libonce_cell-983e777de2dffb91.rlib" "/home/bnaecker/omicron/target/debug/deps/libmio-c0e4839e4abbecd2.rlib" "/home/bnaecker/omicron/target/debug/deps/liblog-409a956358a69afe.rlib" "/home/bnaecker/omicron/target/debug/deps/libparking_lot-464dfbd9492b5b3b.rlib" "/home/bnaecker/omicron/target/debug/deps/libparking_lot_core-4c808711e4fcbb12.rlib" "/home/bnaecker/omicron/target/debug/deps/liblibc-be707306429f040c.rlib" "/home/bnaecker/omicron/target/debug/deps/libsmallvec-348db80a3e827aa0.rlib" "/home/bnaecker/omicron/target/debug/deps/liblock_api-bfa80e01a6cac024.rlib" "/home/bnaecker/omicron/target/debug/deps/libscopeguard-31a2cdce41286a7b.rlib" "/home/bnaecker/omicron/target/debug/deps/libinstant-ae2c96832c229c11.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttpdate-781215c87d248ee6.rlib" "/home/bnaecker/omicron/target/debug/deps/libtracing-1e6821b1349344f4.rlib" "/home/bnaecker/omicron/target/debug/deps/libpin_project_lite-d5107c8292b276ac.rlib" "/home/bnaecker/omicron/target/debug/deps/libtracing_core-c7feb64068d61463.rlib" "/home/bnaecker/omicron/target/debug/deps/liblazy_static-72b18b40aaa861f8.rlib" "/home/bnaecker/omicron/target/debug/deps/libcfg_if-363f0f165151a264.rlib" "/home/bnaecker/omicron/target/debug/deps/libhttp-a92060c26bf96314.rlib" "/home/bnaecker/omicron/target/debug/deps/libitoa-b5d0dacd5d002ed8.rlib" "/home/bnaecker/omicron/target/debug/deps/libbytes-e618e8ebf2ad59d5.rlib" "/home/bnaecker/omicron/target/debug/deps/libserde-70086fb5d9665b56.rlib" "/home/bnaecker/omicron/target/debug/deps/libfnv-0710946dcf4dccf9.rlib" "/home/bnaecker/omicron/target/debug/deps/libslog-39ca96abbd588a0d.rlib" "-Wl,--start-group" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd-df551240bd83d316.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libpanic_abort-0c5f20c3095dc6fd.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libminiz_oxide-d28cbea3beaa15a4.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libadler-422dd4d469ec938c.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libobject-261e9a8c17cad15e.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libaddr2line-bef829f29e00713d.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgimli-a090674e9ae21902.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd_detect-426e08f9caf54663.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_demangle-ca65e69f4f320e51.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libhashbrown-38da2ffd4ba6e106.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_alloc-ad8091dae1980f64.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunwind-390453163eb4e652.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcfg_if-de11336c0160e7a6.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liblibc-066d8902c3f391d2.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liballoc-40d2c2423d4f0bf4.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_core-45d83066ecf05e1b.rlib" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcore-2a95b4624d21aa7e.rlib" "-Wl,--end-group" "/home/bnaecker/.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcompiler_builtins-13e55be1ea183f77.rlib" "-Wl,-Bdynamic" "-lpq" "-lsocket" "-lposix4" "-lpthread" "-lresolv" "-lnsl" "-lumem" "-lgcc_s" "-lc" "-lm" "-lrt" "-lpthread" "-lssp"
  = note: ld: fatal: library -lpq: not found
          ld: fatal: file processing errors. No output written to /home/bnaecker/omicron/target/debug/deps/bootstrap_agent-c5f009f3b9abf0ef
          collect2: error: ld returned 1 exit status


error: aborting due to previous error

error: aborting due to previous error

error: could not compile `omicron-common`

Since this came in with the recent Diesel change, and libpq is a PostgreSQL-related library, I figured there's probably some new package that's pulling that in. So I looked at cargo tree:

    dap@helios omicron $ cargo tree
    omicron-bootstrap-agent v0.1.0 (/home/dap/omicron/omicron-bootstrap-agent)
    ├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15)
    │   ├── async-trait v0.1.51 (proc-macro)
    │   │   ├── proc-macro2 v1.0.28
    │   │   │   └── unicode-xid v0.2.2
    │   │   ├── quote v1.0.9
    │   │   │   └── proc-macro2 v1.0.28 (*)
    │   │   └── syn v1.0.74
    │   │       ├── proc-macro2 v1.0.28 (*)
    │   │       ├── quote v1.0.9 (*)
    │   │       └── unicode-xid v0.2.2
    │   ├── base64 v0.13.0
    │   ├── bytes v1.0.1
    │   │   └── serde v1.0.127
    │   │       └── serde_derive v1.0.127 (proc-macro)
    │   │           ├── proc-macro2 v1.0.28 (*)
    │   │           ├── quote v1.0.9 (*)
    │   │           └── syn v1.0.74 (*)
    │   ├── chrono v0.4.19
    │   │   ├── libc v0.2.98
    │   │   ├── num-integer v0.1.44
    │   │   │   └── num-traits v0.2.14
    │   │   │       [build-dependencies]
    │   │   │       └── autocfg v1.0.1
    │   │   │   [build-dependencies]
    │   │   │   └── autocfg v1.0.1
    │   │   ├── num-traits v0.2.14 (*)
    │   │   ├── serde v1.0.127 (*)
    │   │   └── time v0.1.44
    │   │       └── libc v0.2.98
    │   ├── dropshot_endpoint v0.5.2-dev (proc-macro) (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15)
    │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   ├── quote v1.0.9 (*)
    │   │   ├── serde v1.0.127
    │   │   │   └── serde_derive v1.0.127 (proc-macro) (*)
    │   │   ├── serde_tokenstream v0.1.2
    │   │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   ├── serde v1.0.127 (*)
    │   │   │   └── syn v1.0.74 (*)
    │   │   └── syn v1.0.74 (*)
    │   ├── futures v0.3.15
    │   │   ├── futures-channel v0.3.15
    │   │   │   ├── futures-core v0.3.15
    │   │   │   └── futures-sink v0.3.15
    │   │   ├── futures-core v0.3.15
    │   │   ├── futures-executor v0.3.15
    │   │   │   ├── futures-core v0.3.15
    │   │   │   ├── futures-task v0.3.15
    │   │   │   └── futures-util v0.3.15
    │   │   │       ├── futures-channel v0.3.15 (*)
    │   │   │       ├── futures-core v0.3.15
    │   │   │       ├── futures-io v0.3.15
    │   │   │       ├── futures-macro v0.3.15 (proc-macro)
    │   │   │       │   ├── proc-macro-hack v0.5.19 (proc-macro)
    │   │   │       │   ├── proc-macro2 v1.0.28 (*)
    │   │   │       │   ├── quote v1.0.9 (*)
    │   │   │       │   └── syn v1.0.74 (*)
    │   │   │       │   [build-dependencies]
    │   │   │       │   └── autocfg v1.0.1
    │   │   │       ├── futures-sink v0.3.15
    │   │   │       ├── futures-task v0.3.15
    │   │   │       ├── memchr v2.4.0
    │   │   │       ├── pin-project-lite v0.2.6
    │   │   │       ├── pin-utils v0.1.0
    │   │   │       ├── proc-macro-hack v0.5.19 (proc-macro)
    │   │   │       ├── proc-macro-nested v0.1.7
    │   │   │       └── slab v0.4.3
    │   │   │       [build-dependencies]
    │   │   │       └── autocfg v1.0.1
    │   │   ├── futures-io v0.3.15
    │   │   ├── futures-sink v0.3.15
    │   │   ├── futures-task v0.3.15
    │   │   └── futures-util v0.3.15 (*)
    │   ├── hostname v0.3.1
    │   │   ├── libc v0.2.98
    │   │   └── match_cfg v0.1.0
    │   ├── http v0.2.4
    │   │   ├── bytes v1.0.1 (*)
    │   │   ├── fnv v1.0.7
    │   │   └── itoa v0.4.7
    │   ├── hyper v0.14.10
    │   │   ├── bytes v1.0.1 (*)
    │   │   ├── futures-channel v0.3.15 (*)
    │   │   ├── futures-core v0.3.15
    │   │   ├── futures-util v0.3.15 (*)
    │   │   ├── h2 v0.3.3
    │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   ├── fnv v1.0.7
    │   │   │   ├── futures-core v0.3.15
    │   │   │   ├── futures-sink v0.3.15
    │   │   │   ├── futures-util v0.3.15 (*)
    │   │   │   ├── http v0.2.4 (*)
    │   │   │   ├── indexmap v1.6.2
    │   │   │   │   ├── hashbrown v0.9.1
    │   │   │   │   └── serde v1.0.127 (*)
    │   │   │   │   [build-dependencies]
    │   │   │   │   └── autocfg v1.0.1
    │   │   │   ├── slab v0.4.3
    │   │   │   ├── tokio v1.10.0
    │   │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   │   ├── libc v0.2.98
    │   │   │   │   ├── memchr v2.4.0
    │   │   │   │   ├── mio v0.7.11
    │   │   │   │   │   ├── libc v0.2.98
    │   │   │   │   │   └── log v0.4.14
    │   │   │   │   │       └── cfg-if v1.0.0
    │   │   │   │   ├── num_cpus v1.13.0
    │   │   │   │   │   └── libc v0.2.98
    │   │   │   │   ├── once_cell v1.7.2
    │   │   │   │   ├── parking_lot v0.11.1
    │   │   │   │   │   ├── instant v0.1.9
    │   │   │   │   │   │   └── cfg-if v1.0.0
    │   │   │   │   │   ├── lock_api v0.4.4
    │   │   │   │   │   │   └── scopeguard v1.1.0
    │   │   │   │   │   └── parking_lot_core v0.8.3
    │   │   │   │   │       ├── cfg-if v1.0.0
    │   │   │   │   │       ├── instant v0.1.9 (*)
    │   │   │   │   │       ├── libc v0.2.98
    │   │   │   │   │       └── smallvec v1.6.1
    │   │   │   │   ├── pin-project-lite v0.2.6
    │   │   │   │   ├── signal-hook-registry v1.4.0
    │   │   │   │   │   └── libc v0.2.98
    │   │   │   │   └── tokio-macros v1.2.0 (proc-macro)
    │   │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       ├── quote v1.0.9 (*)
    │   │   │   │       └── syn v1.0.74 (*)
    │   │   │   │   [build-dependencies]
    │   │   │   │   └── autocfg v1.0.1
    │   │   │   ├── tokio-util v0.6.7
    │   │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   │   ├── futures-core v0.3.15
    │   │   │   │   ├── futures-sink v0.3.15
    │   │   │   │   ├── log v0.4.14 (*)
    │   │   │   │   ├── pin-project-lite v0.2.6
    │   │   │   │   └── tokio v1.10.0 (*)
    │   │   │   └── tracing v0.1.26
    │   │   │       ├── cfg-if v1.0.0
    │   │   │       ├── pin-project-lite v0.2.6
    │   │   │       └── tracing-core v0.1.18
    │   │   │           └── lazy_static v1.4.0
    │   │   ├── http v0.2.4 (*)
    │   │   ├── http-body v0.4.2
    │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   ├── http v0.2.4 (*)
    │   │   │   └── pin-project-lite v0.2.6
    │   │   ├── httparse v1.4.1
    │   │   ├── httpdate v1.0.1
    │   │   ├── itoa v0.4.7
    │   │   ├── pin-project-lite v0.2.6
    │   │   ├── socket2 v0.4.0
    │   │   │   └── libc v0.2.98
    │   │   ├── tokio v1.10.0 (*)
    │   │   ├── tower-service v0.3.1
    │   │   ├── tracing v0.1.26 (*)
    │   │   └── want v0.3.0
    │   │       ├── log v0.4.14 (*)
    │   │       └── try-lock v0.2.3
    │   ├── indexmap v1.6.2 (*)
    │   ├── openapiv3 v0.5.0
    │   │   ├── indexmap v1.6.2 (*)
    │   │   ├── serde v1.0.127 (*)
    │   │   ├── serde_json v1.0.64
    │   │   │   ├── itoa v0.4.7
    │   │   │   ├── ryu v1.0.5
    │   │   │   └── serde v1.0.127 (*)
    │   │   └── serde_yaml v0.8.17
    │   │       ├── dtoa v0.4.8
    │   │       ├── linked-hash-map v0.5.4
    │   │       ├── serde v1.0.127 (*)
    │   │       └── yaml-rust v0.4.5
    │   │           └── linked-hash-map v0.5.4
    │   ├── paste v1.0.5 (proc-macro)
    │   ├── percent-encoding v2.1.0
    │   ├── proc-macro2 v1.0.28 (*)
    │   ├── schemars v0.8.3
    │   │   ├── bytes v1.0.1 (*)
    │   │   ├── chrono v0.4.19 (*)
    │   │   ├── dyn-clone v1.0.4
    │   │   ├── schemars_derive v0.8.3 (proc-macro)
    │   │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   ├── quote v1.0.9 (*)
    │   │   │   ├── serde_derive_internals v0.25.0
    │   │   │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   │   ├── quote v1.0.9 (*)
    │   │   │   │   └── syn v1.0.74 (*)
    │   │   │   └── syn v1.0.74 (*)
    │   │   ├── serde v1.0.127 (*)
    │   │   ├── serde_json v1.0.64 (*)
    │   │   └── uuid v0.8.2
    │   │       ├── getrandom v0.2.3
    │   │       │   ├── cfg-if v1.0.0
    │   │       │   └── libc v0.2.98
    │   │       └── serde v1.0.127 (*)
    │   ├── serde v1.0.127 (*)
    │   ├── serde_json v1.0.64 (*)
    │   ├── serde_urlencoded v0.7.0
    │   │   ├── form_urlencoded v1.0.1
    │   │   │   ├── matches v0.1.8
    │   │   │   └── percent-encoding v2.1.0
    │   │   ├── itoa v0.4.7
    │   │   ├── ryu v1.0.5
    │   │   └── serde v1.0.127 (*)
    │   ├── slog v2.7.0
    │   ├── slog-async v2.6.0
    │   │   ├── crossbeam-channel v0.5.1
    │   │   │   ├── cfg-if v1.0.0
    │   │   │   └── crossbeam-utils v0.8.5
    │   │   │       ├── cfg-if v1.0.0
    │   │   │       └── lazy_static v1.4.0
    │   │   ├── slog v2.7.0
    │   │   ├── take_mut v0.2.2
    │   │   └── thread_local v1.1.3
    │   │       └── once_cell v1.7.2
    │   ├── slog-bunyan v2.3.0
    │   │   ├── chrono v0.4.19 (*)
    │   │   ├── hostname v0.3.1 (*)
    │   │   ├── slog v2.7.0
    │   │   └── slog-json v2.3.0
    │   │       ├── chrono v0.4.19 (*)
    │   │       ├── serde v1.0.127 (*)
    │   │       ├── serde_json v1.0.64 (*)
    │   │       └── slog v2.7.0
    │   ├── slog-json v2.3.0 (*)
    │   ├── slog-term v2.8.0
    │   │   ├── atty v0.2.14
    │   │   │   └── libc v0.2.98
    │   │   ├── chrono v0.4.19 (*)
    │   │   ├── slog v2.7.0
    │   │   ├── term v0.7.0
    │   │   │   └── dirs-next v2.0.0
    │   │   │       ├── cfg-if v1.0.0
    │   │   │       └── dirs-sys-next v0.1.2
    │   │   │           └── libc v0.2.98
    │   │   └── thread_local v1.1.3 (*)
    │   ├── syn v1.0.74
    │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   ├── quote v1.0.9 (*)
    │   │   └── unicode-xid v0.2.2
    │   ├── tokio v1.10.0 (*)
    │   ├── toml v0.5.8
    │   │   └── serde v1.0.127 (*)
    │   └── uuid v0.8.2 (*)
    ├── http v0.2.4 (*)
    ├── hyper v0.14.10 (*)
    ├── omicron-common v0.1.0 (/home/dap/omicron/omicron-common)
    │   ├── anyhow v1.0.42
    │   ├── api_identity v0.1.0 (proc-macro) (/home/dap/omicron/api_identity)
    │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   ├── quote v1.0.9 (*)
    │   │   └── syn v1.0.74 (*)
    │   ├── async-trait v0.1.51 (proc-macro) (*)
    │   ├── backoff v0.3.0
    │   │   ├── futures-core v0.3.15
    │   │   ├── getrandom v0.2.3 (*)
    │   │   ├── instant v0.1.9 (*)
    │   │   ├── pin-project v1.0.7
    │   │   │   └── pin-project-internal v1.0.7 (proc-macro)
    │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │       ├── quote v1.0.9 (*)
    │   │   │       └── syn v1.0.74 (*)
    │   │   ├── rand v0.8.3
    │   │   │   ├── libc v0.2.98
    │   │   │   ├── rand_chacha v0.3.0
    │   │   │   │   ├── ppv-lite86 v0.2.10
    │   │   │   │   └── rand_core v0.6.2
    │   │   │   │       └── getrandom v0.2.3 (*)
    │   │   │   └── rand_core v0.6.2 (*)
    │   │   └── tokio v1.10.0 (*)
    │   ├── chrono v0.4.19 (*)
    │   ├── diesel v2.0.0 (https://github.com/diesel-rs/diesel?rev=a39dd2e#a39dd2eb)
    │   │   ├── bitflags v1.2.1
    │   │   ├── byteorder v1.4.3
    │   │   ├── chrono v0.4.19 (*)
    │   │   ├── diesel_derives v2.0.0 (proc-macro) (https://github.com/diesel-rs/diesel?rev=a39dd2e#a39dd2eb)
    │   │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   ├── quote v1.0.9 (*)
    │   │   │   └── syn v1.0.74 (*)
    │   │   ├── ipnetwork v0.18.0
    │   │   │   └── serde v1.0.127 (*)
    │   │   ├── itoa v0.4.7
    │   │   ├── libc v0.2.98
    │   │   ├── pq-sys v0.4.6
    │   │   ├── r2d2 v0.8.9
    │   │   │   ├── log v0.4.14 (*)
    │   │   │   ├── parking_lot v0.11.1 (*)
    │   │   │   └── scheduled-thread-pool v0.2.5
    │   │   │       └── parking_lot v0.11.1 (*)
    │   │   ├── serde_json v1.0.64 (*)
    │   │   └── uuid v0.8.2 (*)
    │   ├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15) (*)
    │   ├── futures v0.3.15 (*)
    │   ├── http v0.2.4 (*)
    │   ├── hyper v0.14.10 (*)
    │   ├── ipnet v2.3.1
    │   │   └── serde v1.0.127 (*)
    │   ├── libc v0.2.98
    │   ├── macaddr v1.0.1
    │   │   └── serde v1.0.127 (*)
    │   ├── postgres-protocol v0.6.1
    │   │   ├── base64 v0.13.0
    │   │   ├── byteorder v1.4.3
    │   │   ├── bytes v1.0.1 (*)
    │   │   ├── fallible-iterator v0.2.0
    │   │   ├── hmac v0.10.1
    │   │   │   ├── crypto-mac v0.10.0
    │   │   │   │   ├── generic-array v0.14.4
    │   │   │   │   │   └── typenum v1.13.0
    │   │   │   │   │   [build-dependencies]
    │   │   │   │   │   └── version_check v0.9.3
    │   │   │   │   └── subtle v2.4.0
    │   │   │   └── digest v0.9.0
    │   │   │       └── generic-array v0.14.4 (*)
    │   │   ├── md-5 v0.9.1
    │   │   │   ├── block-buffer v0.9.0
    │   │   │   │   └── generic-array v0.14.4 (*)
    │   │   │   ├── digest v0.9.0 (*)
    │   │   │   └── opaque-debug v0.3.0
    │   │   ├── memchr v2.4.0
    │   │   ├── rand v0.8.3 (*)
    │   │   ├── sha2 v0.9.5
    │   │   │   ├── block-buffer v0.9.0 (*)
    │   │   │   ├── cfg-if v1.0.0
    │   │   │   ├── cpufeatures v0.1.4
    │   │   │   ├── digest v0.9.0 (*)
    │   │   │   └── opaque-debug v0.3.0
    │   │   └── stringprep v0.1.2
    │   │       ├── unicode-bidi v0.3.5
    │   │       │   └── matches v0.1.8
    │   │       └── unicode-normalization v0.1.19
    │   │           └── tinyvec v1.2.0
    │   │               └── tinyvec_macros v0.1.0
    │   ├── propolis-server v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   ├── anyhow v1.0.42
    │   │   ├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15) (*)
    │   │   ├── futures v0.3.15 (*)
    │   │   ├── hyper v0.14.10 (*)
    │   │   ├── propolis v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   │   ├── bhyve_api v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   │   │   ├── libc v0.2.98
    │   │   │   │   └── num_enum v0.5.1
    │   │   │   │       ├── derivative v2.2.0 (proc-macro)
    │   │   │   │       │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       │   ├── quote v1.0.9 (*)
    │   │   │   │       │   └── syn v1.0.74 (*)
    │   │   │   │       └── num_enum_derive v0.5.1 (proc-macro)
    │   │   │   │           ├── proc-macro-crate v0.1.5
    │   │   │   │           │   └── toml v0.5.8 (*)
    │   │   │   │           ├── proc-macro2 v1.0.28 (*)
    │   │   │   │           ├── quote v1.0.9 (*)
    │   │   │   │           └── syn v1.0.74 (*)
    │   │   │   ├── bitflags v1.2.1
    │   │   │   ├── byteorder v1.4.3
    │   │   │   ├── dladm v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   │   │   ├── libc v0.2.98
    │   │   │   │   └── num_enum v0.5.1 (*)
    │   │   │   ├── lazy_static v1.4.0
    │   │   │   ├── libc v0.2.98
    │   │   │   ├── num_enum v0.5.1 (*)
    │   │   │   ├── thiserror v1.0.26
    │   │   │   │   └── thiserror-impl v1.0.26 (proc-macro)
    │   │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       ├── quote v1.0.9 (*)
    │   │   │   │       └── syn v1.0.74 (*)
    │   │   │   ├── usdt v0.1.13
    │   │   │   │   ├── dtrace-parser v0.1.9
    │   │   │   │   │   ├── pest v2.1.3
    │   │   │   │   │   │   └── ucd-trie v0.1.3
    │   │   │   │   │   ├── pest_derive v2.1.0 (proc-macro)
    │   │   │   │   │   │   ├── pest v2.1.3 (*)
    │   │   │   │   │   │   └── pest_generator v2.1.3
    │   │   │   │   │   │       ├── pest v2.1.3 (*)
    │   │   │   │   │   │       ├── pest_meta v2.1.3
    │   │   │   │   │   │       │   ├── maplit v1.0.2
    │   │   │   │   │   │       │   └── pest v2.1.3 (*)
    │   │   │   │   │   │       │   [build-dependencies]
    │   │   │   │   │   │       │   └── sha-1 v0.8.2
    │   │   │   │   │   │       │       ├── block-buffer v0.7.3
    │   │   │   │   │   │       │       │   ├── block-padding v0.1.5
    │   │   │   │   │   │       │       │   │   └── byte-tools v0.3.1
    │   │   │   │   │   │       │       │   ├── byte-tools v0.3.1
    │   │   │   │   │   │       │       │   ├── byteorder v1.4.3
    │   │   │   │   │   │       │       │   └── generic-array v0.12.4
    │   │   │   │   │   │       │       │       └── typenum v1.13.0
    │   │   │   │   │   │       │       ├── digest v0.8.1
    │   │   │   │   │   │       │       │   └── generic-array v0.12.4 (*)
    │   │   │   │   │   │       │       ├── fake-simd v0.1.2
    │   │   │   │   │   │       │       └── opaque-debug v0.2.3
    │   │   │   │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │   │   │   │       ├── quote v1.0.9 (*)
    │   │   │   │   │   │       └── syn v1.0.74 (*)
    │   │   │   │   │   └── thiserror v1.0.26 (*)
    │   │   │   │   ├── usdt-impl v0.1.8
    │   │   │   │   │   ├── byteorder v1.4.3
    │   │   │   │   │   ├── dof v0.1.5
    │   │   │   │   │   │   ├── thiserror v1.0.26 (*)
    │   │   │   │   │   │   └── zerocopy v0.3.0
    │   │   │   │   │   │       ├── byteorder v1.4.3
    │   │   │   │   │   │       └── zerocopy-derive v0.2.0 (proc-macro)
    │   │   │   │   │   │           ├── proc-macro2 v1.0.28 (*)
    │   │   │   │   │   │           ├── syn v1.0.74 (*)
    │   │   │   │   │   │           └── synstructure v0.12.4
    │   │   │   │   │   │               ├── proc-macro2 v1.0.28 (*)
    │   │   │   │   │   │               ├── quote v1.0.9 (*)
    │   │   │   │   │   │               ├── syn v1.0.74 (*)
    │   │   │   │   │   │               └── unicode-xid v0.2.2
    │   │   │   │   │   ├── dtrace-parser v0.1.9 (*)
    │   │   │   │   │   ├── libc v0.2.98
    │   │   │   │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   │   │   ├── quote v1.0.9 (*)
    │   │   │   │   │   ├── serde v1.0.127 (*)
    │   │   │   │   │   ├── syn v1.0.74 (*)
    │   │   │   │   │   └── thiserror v1.0.26 (*)
    │   │   │   │   └── usdt-macro v0.1.9 (proc-macro)
    │   │   │   │       ├── dtrace-parser v0.1.9 (*)
    │   │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       ├── quote v1.0.9 (*)
    │   │   │   │       ├── serde_tokenstream v0.1.2 (*)
    │   │   │   │       ├── syn v1.0.74 (*)
    │   │   │   │       └── usdt-impl v0.1.8
    │   │   │   │           ├── byteorder v1.4.3
    │   │   │   │           ├── dof v0.1.5 (*)
    │   │   │   │           ├── dtrace-parser v0.1.9 (*)
    │   │   │   │           ├── libc v0.2.98
    │   │   │   │           ├── proc-macro2 v1.0.28 (*)
    │   │   │   │           ├── quote v1.0.9 (*)
    │   │   │   │           ├── serde v1.0.127 (*)
    │   │   │   │           ├── syn v1.0.74 (*)
    │   │   │   │           └── thiserror v1.0.26 (*)
    │   │   │   └── viona_api v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   ├── propolis-client v0.1.0 (https://github.com/oxidecomputer/propolis?rev=b6da043d#b6da043d)
    │   │   │   ├── reqwest v0.11.4
    │   │   │   │   ├── base64 v0.13.0
    │   │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   │   ├── encoding_rs v0.8.28
    │   │   │   │   │   └── cfg-if v1.0.0
    │   │   │   │   ├── futures-core v0.3.15
    │   │   │   │   ├── futures-util v0.3.15 (*)
    │   │   │   │   ├── http v0.2.4 (*)
    │   │   │   │   ├── http-body v0.4.2 (*)
    │   │   │   │   ├── hyper v0.14.10 (*)
    │   │   │   │   ├── hyper-rustls v0.22.1
    │   │   │   │   │   ├── futures-util v0.3.15 (*)
    │   │   │   │   │   ├── hyper v0.14.10 (*)
    │   │   │   │   │   ├── log v0.4.14 (*)
    │   │   │   │   │   ├── rustls v0.19.1
    │   │   │   │   │   │   ├── base64 v0.13.0
    │   │   │   │   │   │   ├── log v0.4.14 (*)
    │   │   │   │   │   │   ├── ring v0.16.20
    │   │   │   │   │   │   │   ├── once_cell v1.7.2
    │   │   │   │   │   │   │   ├── spin v0.5.2
    │   │   │   │   │   │   │   └── untrusted v0.7.1
    │   │   │   │   │   │   │   [build-dependencies]
    │   │   │   │   │   │   │   └── cc v1.0.68
    │   │   │   │   │   │   ├── sct v0.6.1
    │   │   │   │   │   │   │   ├── ring v0.16.20 (*)
    │   │   │   │   │   │   │   └── untrusted v0.7.1
    │   │   │   │   │   │   └── webpki v0.21.4
    │   │   │   │   │   │       ├── ring v0.16.20 (*)
    │   │   │   │   │   │       └── untrusted v0.7.1
    │   │   │   │   │   ├── tokio v1.10.0 (*)
    │   │   │   │   │   ├── tokio-rustls v0.22.0
    │   │   │   │   │   │   ├── rustls v0.19.1 (*)
    │   │   │   │   │   │   ├── tokio v1.10.0 (*)
    │   │   │   │   │   │   └── webpki v0.21.4 (*)
    │   │   │   │   │   └── webpki v0.21.4 (*)
    │   │   │   │   ├── hyper-tls v0.5.0
    │   │   │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   │   │   ├── hyper v0.14.10 (*)
    │   │   │   │   │   ├── native-tls v0.2.7
    │   │   │   │   │   │   ├── log v0.4.14 (*)
    │   │   │   │   │   │   ├── openssl v0.10.34
    │   │   │   │   │   │   │   ├── bitflags v1.2.1
    │   │   │   │   │   │   │   ├── cfg-if v1.0.0
    │   │   │   │   │   │   │   ├── foreign-types v0.3.2
    │   │   │   │   │   │   │   │   └── foreign-types-shared v0.1.1
    │   │   │   │   │   │   │   ├── libc v0.2.98
    │   │   │   │   │   │   │   ├── once_cell v1.7.2
    │   │   │   │   │   │   │   └── openssl-sys v0.9.63
    │   │   │   │   │   │   │       └── libc v0.2.98
    │   │   │   │   │   │   │       [build-dependencies]
    │   │   │   │   │   │   │       ├── autocfg v1.0.1
    │   │   │   │   │   │   │       ├── cc v1.0.68
    │   │   │   │   │   │   │       └── pkg-config v0.3.19
    │   │   │   │   │   │   ├── openssl-probe v0.1.4
    │   │   │   │   │   │   └── openssl-sys v0.9.63 (*)
    │   │   │   │   │   ├── tokio v1.10.0 (*)
    │   │   │   │   │   └── tokio-native-tls v0.3.0
    │   │   │   │   │       ├── native-tls v0.2.7 (*)
    │   │   │   │   │       └── tokio v1.10.0 (*)
    │   │   │   │   ├── ipnet v2.3.1 (*)
    │   │   │   │   ├── lazy_static v1.4.0
    │   │   │   │   ├── log v0.4.14 (*)
    │   │   │   │   ├── mime v0.3.16
    │   │   │   │   ├── native-tls v0.2.7 (*)
    │   │   │   │   ├── percent-encoding v2.1.0
    │   │   │   │   ├── pin-project-lite v0.2.6
    │   │   │   │   ├── rustls v0.19.1 (*)
    │   │   │   │   ├── serde v1.0.127 (*)
    │   │   │   │   ├── serde_json v1.0.64 (*)
    │   │   │   │   ├── serde_urlencoded v0.7.0 (*)
    │   │   │   │   ├── tokio v1.10.0 (*)
    │   │   │   │   ├── tokio-native-tls v0.3.0 (*)
    │   │   │   │   ├── tokio-rustls v0.22.0 (*)
    │   │   │   │   ├── url v2.2.2
    │   │   │   │   │   ├── form_urlencoded v1.0.1 (*)
    │   │   │   │   │   ├── idna v0.2.3
    │   │   │   │   │   │   ├── matches v0.1.8
    │   │   │   │   │   │   ├── unicode-bidi v0.3.5 (*)
    │   │   │   │   │   │   └── unicode-normalization v0.1.19 (*)
    │   │   │   │   │   ├── matches v0.1.8
    │   │   │   │   │   └── percent-encoding v2.1.0
    │   │   │   │   └── webpki-roots v0.21.1
    │   │   │   │       └── webpki v0.21.4 (*)
    │   │   │   ├── ring v0.16.20 (*)
    │   │   │   ├── schemars v0.8.3 (*)
    │   │   │   ├── serde v1.0.127 (*)
    │   │   │   ├── serde_json v1.0.64 (*)
    │   │   │   ├── slog v2.7.0
    │   │   │   ├── structopt v0.3.22
    │   │   │   │   ├── clap v2.33.3
    │   │   │   │   │   ├── ansi_term v0.11.0
    │   │   │   │   │   ├── atty v0.2.14 (*)
    │   │   │   │   │   ├── bitflags v1.2.1
    │   │   │   │   │   ├── strsim v0.8.0
    │   │   │   │   │   ├── textwrap v0.11.0
    │   │   │   │   │   │   └── unicode-width v0.1.8
    │   │   │   │   │   ├── unicode-width v0.1.8
    │   │   │   │   │   └── vec_map v0.8.2
    │   │   │   │   ├── lazy_static v1.4.0
    │   │   │   │   └── structopt-derive v0.4.15 (proc-macro)
    │   │   │   │       ├── heck v0.3.3
    │   │   │   │       │   └── unicode-segmentation v1.7.1
    │   │   │   │       ├── proc-macro-error v1.0.4
    │   │   │   │       │   ├── proc-macro-error-attr v1.0.4 (proc-macro)
    │   │   │   │       │   │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       │   │   └── quote v1.0.9 (*)
    │   │   │   │       │   │   [build-dependencies]
    │   │   │   │       │   │   └── version_check v0.9.3
    │   │   │   │       │   ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       │   ├── quote v1.0.9 (*)
    │   │   │   │       │   └── syn v1.0.74 (*)
    │   │   │   │       │   [build-dependencies]
    │   │   │   │       │   └── version_check v0.9.3
    │   │   │   │       ├── proc-macro2 v1.0.28 (*)
    │   │   │   │       ├── quote v1.0.9 (*)
    │   │   │   │       └── syn v1.0.74 (*)
    │   │   │   ├── thiserror v1.0.26 (*)
    │   │   │   └── uuid v0.8.2 (*)
    │   │   ├── serde v1.0.127 (*)
    │   │   ├── serde_derive v1.0.127 (proc-macro) (*)
    │   │   ├── slog v2.7.0
    │   │   ├── structopt v0.3.22 (*)
    │   │   ├── thiserror v1.0.26 (*)
    │   │   ├── tokio v1.10.0 (*)
    │   │   ├── tokio-tungstenite v0.14.0
    │   │   │   ├── futures-util v0.3.15 (*)
    │   │   │   ├── log v0.4.14 (*)
    │   │   │   ├── pin-project v1.0.7 (*)
    │   │   │   ├── tokio v1.10.0 (*)
    │   │   │   └── tungstenite v0.13.0
    │   │   │       ├── base64 v0.13.0
    │   │   │       ├── byteorder v1.4.3
    │   │   │       ├── bytes v1.0.1 (*)
    │   │   │       ├── http v0.2.4 (*)
    │   │   │       ├── httparse v1.4.1
    │   │   │       ├── input_buffer v0.4.0
    │   │   │       │   └── bytes v1.0.1 (*)
    │   │   │       ├── log v0.4.14 (*)
    │   │   │       ├── rand v0.8.3 (*)
    │   │   │       ├── sha-1 v0.9.6
    │   │   │       │   ├── block-buffer v0.9.0 (*)
    │   │   │       │   ├── cfg-if v1.0.0
    │   │   │       │   ├── cpufeatures v0.1.4
    │   │   │       │   ├── digest v0.9.0 (*)
    │   │   │       │   └── opaque-debug v0.3.0
    │   │   │       ├── thiserror v1.0.26 (*)
    │   │   │       ├── url v2.2.2 (*)
    │   │   │       └── utf-8 v0.7.6
    │   │   ├── toml v0.5.8 (*)
    │   │   └── uuid v0.8.2 (*)
    │   ├── rayon v1.5.1
    │   │   ├── crossbeam-deque v0.8.0
    │   │   │   ├── cfg-if v1.0.0
    │   │   │   ├── crossbeam-epoch v0.9.5
    │   │   │   │   ├── cfg-if v1.0.0
    │   │   │   │   ├── crossbeam-utils v0.8.5 (*)
    │   │   │   │   ├── lazy_static v1.4.0
    │   │   │   │   ├── memoffset v0.6.4
    │   │   │   │   │   [build-dependencies]
    │   │   │   │   │   └── autocfg v1.0.1
    │   │   │   │   └── scopeguard v1.1.0
    │   │   │   └── crossbeam-utils v0.8.5 (*)
    │   │   ├── either v1.6.1
    │   │   └── rayon-core v1.9.1
    │   │       ├── crossbeam-channel v0.5.1 (*)
    │   │       ├── crossbeam-deque v0.8.0 (*)
    │   │       ├── crossbeam-utils v0.8.5 (*)
    │   │       ├── lazy_static v1.4.0
    │   │       └── num_cpus v1.13.0 (*)
    │   │   [build-dependencies]
    │   │   └── autocfg v1.0.1
    │   ├── reqwest v0.11.4 (*)
    │   ├── ring v0.16.20 (*)
    │   ├── schemars v0.8.3 (*)
    │   ├── serde v1.0.127 (*)
    │   ├── serde_derive v1.0.127 (proc-macro) (*)
    │   ├── serde_json v1.0.64 (*)
    │   ├── signal-hook v0.3.9
    │   │   ├── libc v0.2.98
    │   │   └── signal-hook-registry v1.4.0 (*)
    │   ├── signal-hook-tokio v0.3.0
    │   │   ├── futures-core v0.3.15
    │   │   ├── libc v0.2.98
    │   │   ├── signal-hook v0.3.9 (*)
    │   │   └── tokio v1.10.0 (*)
    │   ├── slog v2.7.0
    │   ├── smf v0.2.1
    │   │   └── thiserror v1.0.26 (*)
    │   ├── steno v0.1.0 (https://github.com/oxidecomputer/steno?branch=main#8da3eda3)
    │   │   ├── anyhow v1.0.42
    │   │   ├── async-trait v0.1.51 (proc-macro) (*)
    │   │   ├── chrono v0.4.19 (*)
    │   │   ├── futures v0.3.15 (*)
    │   │   ├── newtype_derive v0.1.6
    │   │   │   [build-dependencies]
    │   │   │   └── rustc_version v0.1.7
    │   │   │       └── semver v0.1.20
    │   │   ├── petgraph v0.5.1
    │   │   │   ├── fixedbitset v0.2.0
    │   │   │   └── indexmap v1.6.2 (*)
    │   │   ├── schemars v0.8.3 (*)
    │   │   ├── serde v1.0.127 (*)
    │   │   ├── serde_json v1.0.64 (*)
    │   │   ├── slog v2.7.0
    │   │   ├── thiserror v1.0.26 (*)
    │   │   ├── tokio v1.10.0 (*)
    │   │   └── uuid v0.8.2 (*)
    │   ├── structopt v0.3.22 (*)
    │   ├── subprocess v0.2.7
    │   │   └── libc v0.2.98
    │   ├── tar v0.4.35
    │   │   ├── filetime v0.2.14
    │   │   │   ├── cfg-if v1.0.0
    │   │   │   └── libc v0.2.98
    │   │   ├── libc v0.2.98
    │   │   └── xattr v0.2.2
    │   │       └── libc v0.2.98
    │   ├── tempfile v3.2.0
    │   │   ├── cfg-if v1.0.0
    │   │   ├── libc v0.2.98
    │   │   ├── rand v0.8.3 (*)
    │   │   └── remove_dir_all v0.5.3
    │   ├── thiserror v1.0.26 (*)
    │   ├── tokio v1.10.0 (*)
    │   ├── tokio-postgres v0.7.2
    │   │   ├── async-trait v0.1.51 (proc-macro) (*)
    │   │   ├── byteorder v1.4.3
    │   │   ├── bytes v1.0.1 (*)
    │   │   ├── fallible-iterator v0.2.0
    │   │   ├── futures v0.3.15 (*)
    │   │   ├── log v0.4.14 (*)
    │   │   ├── parking_lot v0.11.1 (*)
    │   │   ├── percent-encoding v2.1.0
    │   │   ├── phf v0.8.0
    │   │   │   └── phf_shared v0.8.0
    │   │   │       └── siphasher v0.3.5
    │   │   ├── pin-project-lite v0.2.6
    │   │   ├── postgres-protocol v0.6.1 (*)
    │   │   ├── postgres-types v0.2.1
    │   │   │   ├── bytes v1.0.1 (*)
    │   │   │   ├── chrono v0.4.19 (*)
    │   │   │   ├── fallible-iterator v0.2.0
    │   │   │   ├── postgres-protocol v0.6.1 (*)
    │   │   │   ├── serde v1.0.127 (*)
    │   │   │   ├── serde_json v1.0.64 (*)
    │   │   │   └── uuid v0.8.2 (*)
    │   │   ├── socket2 v0.4.0 (*)
    │   │   ├── tokio v1.10.0 (*)
    │   │   └── tokio-util v0.6.7 (*)
    │   ├── toml v0.5.8 (*)
    │   ├── uuid v0.8.2 (*)
    │   └── walkdir v2.3.2
    │       └── same-file v1.0.6
    │   [dev-dependencies]
    │   ├── expectorate v1.0.3
    │   │   ├── difference v2.0.0
    │   │   └── newline-converter v0.1.0
    │   ├── serde_urlencoded v0.7.0 (*)
    │   └── tokio v1.10.0 (*)
    ├── serde v1.0.127 (*)
    ├── serde_json v1.0.64 (*)
    ├── slog v2.7.0
    ├── smf v0.2.1 (*)
    ├── structopt v0.3.22 (*)
    ├── tar v0.4.35 (*)
    ├── thiserror v1.0.26 (*)
    ├── tokio v1.10.0 (*)
    ├── toml v0.5.8 (*)
    └── uuid v0.8.2 (*)
omicron-common v0.1.0 (/home/dap/omicron/omicron-common) (*)

omicron-nexus v0.1.0 (/home/dap/omicron/omicron-nexus)
├── anyhow v1.0.42
├── async-bb8-diesel v0.1.0 (https://github.com/oxidecomputer/async-bb8-diesel?rev=0a6d535#0a6d535f)
│   ├── async-trait v0.1.51 (proc-macro) (*)
│   ├── bb8 v0.7.1
│   │   ├── async-trait v0.1.51 (proc-macro) (*)
│   │   ├── futures-channel v0.3.15 (*)
│   │   ├── futures-util v0.3.15 (*)
│   │   ├── parking_lot v0.11.1 (*)
│   │   └── tokio v1.10.0 (*)
│   ├── diesel v2.0.0 (https://github.com/diesel-rs/diesel?rev=a39dd2e#a39dd2eb) (*)
│   └── tokio v1.10.0 (*)
├── async-trait v0.1.51 (proc-macro) (*)
├── bb8 v0.7.1 (*)
├── chrono v0.4.19 (*)
├── diesel v2.0.0 (https://github.com/diesel-rs/diesel?rev=a39dd2e#a39dd2eb) (*)
├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15) (*)
├── futures v0.3.15 (*)
├── http v0.2.4 (*)
├── hyper v0.14.10 (*)
├── ipnetwork v0.18.0 (*)
├── lazy_static v1.4.0
├── libc v0.2.98
├── newtype_derive v0.1.6 (*)
├── omicron-common v0.1.0 (/home/dap/omicron/omicron-common) (*)
├── omicron-sled-agent v0.1.0 (/home/dap/omicron/omicron-sled-agent)
│   ├── async-trait v0.1.51 (proc-macro) (*)
│   ├── cfg-if v1.0.0
│   ├── chrono v0.4.19 (*)
│   ├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15) (*)
│   ├── futures v0.3.15 (*)
│   ├── http v0.2.4 (*)
│   ├── hyper v0.14.10 (*)
│   ├── ipnet v2.3.1 (*)
│   ├── omicron-common v0.1.0 (/home/dap/omicron/omicron-common) (*)
│   ├── propolis-client v0.1.0 (https://github.com/oxidecomputer/propolis?rev=3d58a6398#3d58a639)
│   │   ├── reqwest v0.11.4 (*)
│   │   ├── ring v0.16.20 (*)
│   │   ├── schemars v0.8.3 (*)
│   │   ├── serde v1.0.127 (*)
│   │   ├── serde_json v1.0.64 (*)
│   │   ├── slog v2.7.0
│   │   ├── structopt v0.3.22 (*)
│   │   ├── thiserror v1.0.26 (*)
│   │   └── uuid v0.8.2 (*)
│   ├── schemars v0.8.3 (*)
│   ├── serde v1.0.127 (*)
│   ├── serde_json v1.0.64 (*)
│   ├── slog v2.7.0
│   ├── smf v0.2.1 (*)
│   ├── structopt v0.3.22 (*)
│   ├── tokio v1.10.0 (*)
│   ├── uuid v0.8.2 (*)
│   └── zone v0.1.7
│       ├── itertools v0.10.1
│       │   └── either v1.6.1
│       ├── thiserror v1.0.26 (*)
│       └── zone_cfg_derive v0.1.2 (proc-macro)
│           ├── heck v0.3.3 (*)
│           ├── proc-macro-error v1.0.4 (*)
│           ├── proc-macro2 v1.0.28 (*)
│           ├── quote v1.0.9 (*)
│           └── syn v1.0.74 (*)
│   [dev-dependencies]
│   ├── expectorate v1.0.3 (*)
│   ├── mockall v0.10.2
│   │   ├── cfg-if v1.0.0
│   │   ├── downcast v0.10.0
│   │   ├── fragile v1.0.0
│   │   ├── lazy_static v1.4.0
│   │   ├── mockall_derive v0.10.2 (proc-macro)
│   │   │   ├── cfg-if v1.0.0
│   │   │   ├── proc-macro2 v1.0.28 (*)
│   │   │   ├── quote v1.0.9 (*)
│   │   │   └── syn v1.0.74 (*)
│   │   ├── predicates v1.0.8
│   │   │   ├── difference v2.0.0
│   │   │   ├── float-cmp v0.8.0
│   │   │   │   └── num-traits v0.2.14 (*)
│   │   │   ├── normalize-line-endings v0.3.0
│   │   │   ├── predicates-core v1.0.2
│   │   │   └── regex v1.5.4
│   │   │       ├── aho-corasick v0.7.18
│   │   │       │   └── memchr v2.4.0
│   │   │       ├── memchr v2.4.0
│   │   │       └── regex-syntax v0.6.25
│   │   └── predicates-tree v1.0.2
│   │       ├── predicates-core v1.0.2
│   │       └── treeline v0.1.0
│   ├── serial_test v0.5.1
│   │   ├── lazy_static v1.4.0
│   │   ├── parking_lot v0.11.1 (*)
│   │   └── serial_test_derive v0.5.1 (proc-macro)
│   │       ├── proc-macro2 v1.0.28 (*)
│   │       ├── quote v1.0.9 (*)
│   │       └── syn v1.0.74 (*)
│   └── subprocess v0.2.7 (*)
├── schemars v0.8.3 (*)
├── serde v1.0.127 (*)
├── serde_json v1.0.64 (*)
├── serde_with v1.9.4
│   ├── rustversion v1.0.5 (proc-macro)
│   ├── serde v1.0.127 (*)
│   └── serde_with_macros v1.4.2 (proc-macro)
│       ├── darling v0.13.0
│       │   ├── darling_core v0.13.0
│       │   │   ├── fnv v1.0.7
│       │   │   ├── ident_case v1.0.1
│       │   │   ├── proc-macro2 v1.0.28 (*)
│       │   │   ├── quote v1.0.9 (*)
│       │   │   ├── strsim v0.10.0
│       │   │   └── syn v1.0.74 (*)
│       │   └── darling_macro v0.13.0 (proc-macro)
│       │       ├── darling_core v0.13.0 (*)
│       │       ├── quote v1.0.9 (*)
│       │       └── syn v1.0.74 (*)
│       ├── proc-macro2 v1.0.28 (*)
│       ├── quote v1.0.9 (*)
│       └── syn v1.0.74 (*)
├── slog v2.7.0
├── steno v0.1.0 (https://github.com/oxidecomputer/steno?branch=main#8da3eda3) (*)
├── structopt v0.3.22 (*)
├── thiserror v1.0.26 (*)
├── tokio v1.10.0 (*)
├── tokio-postgres v0.7.2 (*)
├── toml v0.5.8 (*)
└── uuid v0.8.2 (*)
[dev-dependencies]
├── bytes v1.0.1 (*)
├── expectorate v1.0.3 (*)
├── openapi-lint v0.1.0 (https://github.com/oxidecomputer/openapi-lint?branch=main#cbf584ba)
│   ├── indexmap v1.6.2 (*)
│   └── openapiv3 v0.5.0 (*)
├── openapiv3 v0.5.0 (*)
├── oximeter v0.1.0 (/home/dap/omicron/oximeter/oximeter)
│   ├── anyhow v1.0.42
│   ├── bytes v1.0.1 (*)
│   ├── chrono v0.4.19 (*)
│   ├── dropshot v0.5.2-dev (https://github.com/oxidecomputer/dropshot?branch=main#1c6f4d15) (*)
│   ├── hyper v0.14.10 (*)
│   ├── num-traits v0.2.14 (*)
│   ├── omicron-common v0.1.0 (/home/dap/omicron/omicron-common) (*)
│   ├── oximeter-macro-impl v0.1.0 (proc-macro) (/home/dap/omicron/oximeter/oximeter-macro-impl)
│   │   ├── bytes v1.0.1 (*)
│   │   ├── proc-macro2 v1.0.28 (*)
│   │   ├── quote v1.0.9 (*)
│   │   └── syn v1.0.74 (*)
│   ├── reqwest v0.11.4 (*)
│   ├── schemars v0.8.3 (*)
│   ├── serde v1.0.127 (*)
│   ├── serde_json v1.0.64 (*)
│   ├── slog v2.7.0
│   ├── slog-async v2.6.0 (*)
│   ├── slog-term v2.8.0 (*)
│   ├── structopt v0.3.22 (*)
│   ├── thiserror v1.0.26 (*)
│   ├── tokio v1.10.0 (*)
│   ├── toml v0.5.8 (*)
│   └── uuid v0.8.2 (*)
│   [dev-dependencies]
│   └── trybuild v1.0.44
│       ├── glob v0.3.0
│       ├── lazy_static v1.4.0
│       ├── serde v1.0.127 (*)
│       ├── serde_json v1.0.64 (*)
│       ├── termcolor v1.1.2
│       └── toml v0.5.8 (*)
└── subprocess v0.2.7 (*)

omicron-sled-agent v0.1.0 (/home/dap/omicron/omicron-sled-agent) (*)

oximeter v0.1.0 (/home/dap/omicron/oximeter/oximeter) (*)

oximeter-macro-impl v0.1.0 (proc-macro) (/home/dap/omicron/oximeter/oximeter-macro-impl) (*)

One of the crates there is pq-sys, depended-on by the recently-added "diesel", so that's promising:

$ cargo tree
...
├── omicron-common v0.1.0 (/home/dap/omicron/omicron-common)
...
│   ├── diesel v2.0.0 (https://github.com/diesel-rs/diesel?rev=a39dd2e#a39dd2eb)
...
│   │   ├── pq-sys v0.4.6
...

That's this crate, source here. It's a bindgen-generated binding for libpq. As expected, its Cargo.toml includes "links = "pq":

https://github.com/sgrif/pq-sys/blob/3e367d53019a2740054d5dc6946e07931f1fb70b/Cargo.toml#L8

And build.rs uses the rustc-link-lib instruction like you'd expect. Now, the README for pq-sys has notes:

• You can set environment variable PQ_LIB_DIR explicitly to use that directory for the libraries.
• Otherwise it will try to use pkg-config.
• Otherwise it will try to use pg_config.

Now on Helios we'd expect the library itself to be delivered by pkg:/ooce/library/postgresql-13. Sure enough, that delivers pg_config as well:

$ pkg search file:basename:pg_config
...
basename file opt/ooce/pgsql-13/bin/pg_config pkg:/ooce/library/postgresql-13@13.0-1.0
...

It puts it into /opt/ooce/pgsql-13/bin. @jclulow found with pkg search link:basename:pg_config that it also installs a mediated link into /opt/ooce/bin. One of these directories needs to be in your PATH at build time for the pq-sys build to find it. If we do set PATH accordingly, then the library is correctly found at build time and the build succeeds. But the binaries fail at runtime instead:

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running unittests (target/debug/deps/omicron_common-72d5f08d50c41ded)
ld.so.1: omicron_common-72d5f08d50c41ded: fatal: libpq.so.5: open failed: No such file or directory
error: test failed, to rerun pass '-p omicron-common --lib'

Caused by:
  process didn't exit successfully: `/home/dap/omicron/target/debug/deps/omicron_common-72d5f08d50c41ded` (signal: 9, SIGKILL: kill)

Usually if ld.so can't find a library like this, the library is NEEDED (e.g., ldd would show it) but not found in the standard locations or the RUNPATH. Sure enough, ldd shows that it's needed and not found:

$ ldd /home/dap/omicron/target/debug/deps/omicron_common-72d5f08d50c41ded 
        libpq.so.5 =>    (file not found)
        libsocket.so.1 =>        /lib/64/libsocket.so.1
        librt.so.1 =>    /lib/64/librt.so.1
        libpthread.so.1 =>       /lib/64/libpthread.so.1
        libnsl.so.1 =>   /lib/64/libnsl.so.1
        libumem.so.1 =>  /lib/64/libumem.so.1
        libgcc_s.so.1 =>         /usr/gcc/10/lib/amd64/libgcc_s.so.1
        libc.so.1 =>     /lib/64/libc.so.1
        libm.so.2 =>     /lib/64/libm.so.2
        libmp.so.2 =>    /lib/64/libmp.so.2
        libmd.so.1 =>    /lib/64/libmd.so.1

elfdump -d shows details:

$ elfdump -d /home/dap/omicron/target/debug/deps/omicron_common-72d5f08d50c41ded

Dynamic Section:  .dynamic
     index  tag                value
       [0]  NEEDED            0x8dd24e            libpq.so.5
       [1]  NEEDED            0x8dd11d            libsocket.so.1
       [2]  NEEDED            0x8dd15c            librt.so.1
       [3]  NEEDED            0x8dd170            libpthread.so.1
       [4]  NEEDED            0x8dd189            libnsl.so.1
       [5]  NEEDED            0x8dd1a5            libumem.so.1
       [6]  NEEDED            0x8dd1b2            libgcc_s.so.1
       [7]  NEEDED            0x8dd1da            libc.so.1
       [8]  NEEDED            0x8dd244            libm.so.2
       [9]  INIT              0x2411cf0           
      [10]  FINI              0x2411d00           
      [11]  RUNPATH           0x8dd259            /usr/gcc/10/lib/amd64
      [12]  RPATH             0x8dd259            /usr/gcc/10/lib/amd64
      [13]  HASH              0x8395a8            
      [14]  STRTAB            0xb260e8            
      [15]  STRSZ             0x8dd46f            
      [16]  SYMTAB            0x962a50            
      [17]  SYMENT            0x18                
      [18]  SUNW_SYMTAB       0x8cfd38            
      [19]  SUNW_SYMSZ        0x2563b0            
      [20]  SUNW_SORTENT      0x4                 
      [21]  SUNW_SYMSORT      0x1429180           
      [22]  SUNW_SYMSORTSZ    0x61e78             
      [23]  CHECKSUM          0xa22e              
      [24]  VERNEED           0x1403558           
      [25]  VERNEEDNUM        0x8                 
      [26]  FINI_ARRAY        0x258b3e8           
      [27]  FINI_ARRAYSZ      0x8                 
      [28]  INIT_ARRAY        0x258b3f0           
      [29]  INIT_ARRAYSZ      0x8                 
      [30]  PLTRELSZ          0x1218              
      [31]  PLTREL            0x7                 
      [32]  JMPREL            0x148b088           
      [33]  RELA              0x148aff8           
      [34]  RELASZ            0x12a8              
      [35]  RELAENT           0x18                
      [36]  DEBUG             0                   
      [37]  FLAGS             0                   0
      [38]  FLAGS_1           0                   0
      [39]  SUNW_STRPAD       0x200               
      [40]  SUNW_LDMACH       0x3e                EM_AMD64
      [41]  PLTGOT            0x25892f8           
[42-52]  NULL              0                

We can see that the RUNPATH/RPATH does not include the directory containing libpq. What directory is this? The build script gets it with:

$ pg_config --libdir
/opt/ooce/pgsql-13/lib/amd64

It seems like the cargo build did not update the RUNPATH. It seems like it should do that, right?

I've spent quite a while over the last two days trying to figure out how this is supposed to work in Cargo and my conclusion is that it doesn't. The clearest evidence is rust-lang/cargo#5077. It's pretty clear that a lot of people have wanted to be able to do this for some time, but it hasn't been implemented.

It should be noted that although this only affects Oxide employees running on illumos, the problem is not illumos-specific at all -- it only has to do with libraries installed in a directory that's not on ld.so.1's default search path.

Failed solutions / workarounds

Codegen option "rpath"

There's a Rust codgen option -C rpath, which is documented to "enable rpath". It's not clear what this means. I did rebuild with RUSTFLAGS='-C rpath=yes', and I got the same failure mode:

     Running unittests (target/debug/deps/omicron_common-72d5f08d50c41ded)
ld.so.1: omicron_common-72d5f08d50c41ded: fatal: libpq.so.5: open failed: No such file or directory
error: test failed, to rerun pass '-p omicron-common --lib'

Caused by:
  process didn't exit successfully: `/home/dap/omicron/target/debug/deps/omicron_common-72d5f08d50c41ded` (signal: 9, SIGKILL: kill)

though the runpath was different (arguably more broken):

dap@helios omicron $ elfdump -d /home/dap/omicron/target/debug/deps/omicron_common-72d5f08d50c41ded | grep -i path
      [11]  RUNPATH           0x8dd259            /usr/gcc/10/lib/amd64:$ORIGIN/../../../../.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib:$ORIGIN/:/home/dap/omicron/lib/rustlib/x86_64-unknown-illumos/lib
      [12]  RPATH             0x8dd259            /usr/gcc/10/lib/amd64:$ORIGIN/../../../../.rustup/toolchains/nightly-2021-04-24-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib:$ORIGIN/:/home/dap/omicron/lib/rustlib/x86_64-unknown-illumos/lib

It's not clear why these directories got added, but they're certainly not right for us here. We abandoned this approach.

Cargo's rustc-link-arg instruction

Keith pointed out that Cargo has grown a feature that could help us:

The rustc-link-arg instruction tells Cargo to pass the [-C link-arg=FLAG
option][link-arg] to the compiler, but only when building supported targets
(benchmarks, binaries, cdylib crates, examples, and tests). Its usage is
highly platform specific. It is useful to set the shared library version or
linker script.

We could use this to pass -R to the linker. pq-sys could emit cargo:rustc-link-arg=-Wl,R,/the/library/path or maybe we do this in Omicron. (The -Wl, prefix is what you pass to the compiler driver to pass the rest of it to the linker.) This is somewhat platform-specific (it probably won't work on Windows, for example).

The docs imply that this will be in 1.56. Current stable is 1.54. So this won't be available in stable Cargo for a few months (!). Now, Omicron is on "nightly", so we could use it there. But where would we put this directive? I think it would have to go into pq-sys, whose consumers are not necessarily on "nightly". I abandoned this path at this point, though we could create our own fork of pq-sys or else try to upstream a change that uses this flag conditional on it being available.

Statically linking libpq

pq-sys can statically link against libpq. That would obviate the need to get the RPATH of the binary right. To test this out, I switched over to a build of "pq-sys" directly, rather than trying to do this inside "omicron".

If we build with PQ_LIB_STATIC, we get a whole bunch of linker errors for undefined symbols:

    dap@helios pq-sys $ PQ_LIB_STATIC=yes cargo test
       Compiling pq-sys v0.4.6 (/home/dap/pq-sys)
    error: linking with `gcc` failed: exit status: 1
      |
      = note: "gcc" "-m64" "-std=c99" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.28zc43ngthik1xi0.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.2ae9uey3x8spvh2b.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.37ak78lfd7i8lgrk.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.3ly6jh3378tbmq2z.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.3pvg9mos4au2z44z.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.3q6bskioxmtlfjp9.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.3t9wgvj2kqufeobn.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.4fjphvztobxrfil.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.4ga1rzu3lsuqgy9o.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.5ghmmeuvdumo07ij.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.kycxi9ya29lqvrp.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.pj0395ry7x85v9d.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.sepdlklc4pe8hnh.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b.153ebt2aq433ew1r.rcgu.o" "-Wl,-z" "-Wl,ignore" "-L" "/home/dap/pq-sys/target/debug/deps" "-L" "/opt/ooce/pgsql-13/lib/amd64" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-Wl,-Bstatic" "-Wl,--whole-archive" "-lpq" "-Wl,--no-whole-archive" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libtest-535dee8e4f3c883b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libterm-be39c4177d57d29f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgetopts-47c81854ba90f851.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunicode_width-b8d4734d4eda0e2a.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_std-67f67341054cb595.rlib" "-Wl,--start-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd-50bed294e78c95c2.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libpanic_unwind-068f7a96e5aa9c3e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libminiz_oxide-79be3b483094292f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libadler-24b6fed8d1040d6b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libobject-cc9fa5d3172400da.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libaddr2line-5b1c430684c7f47b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgimli-2d01878f44f162f6.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd_detect-0a4773aa77217f2e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_demangle-b4f6a6a789a585b0.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libhashbrown-eda90539ad5b3632.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_alloc-168e3556278a7b7d.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunwind-23043d223378f39c.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcfg_if-389ca2ce1f2d8f07.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liblibc-04d9200ebe50adaf.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liballoc-30bf7d00f2682790.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_core-9bad5249b1907a01.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcore-5d546fb1aabd96b7.rlib" "-Wl,--end-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcompiler_builtins-5d2b8a810efe2040.rlib" "-Wl,-Bdynamic" "-lsocket" "-lposix4" "-lpthread" "-lresolv" "-lnsl" "-lumem" "-lgcc_s" "-lm" "-lrt" "-lpthread" "-lc" "-lssp" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-o" "/home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b" "-nodefaultlibs"
      = note: Undefined                 first referenced
               symbol                        in file
              ERR_clear_error                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_set_default_passwd_cb       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              OPENSSL_sk_pop_free                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_set_options                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_use_PrivateKey                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_get_peer_certificate            /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_set_flags                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CIPHER_get_bits                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CIPHER_get_name                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_link_canary_is_frontend          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              SSL_use_PrivateKey_file             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_ServerKey                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              scram_HMAC_final                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              BIO_int_ctrl                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_vsnprintf                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-exec.o)
              OPENSSL_sk_num                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_write                           /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_set_default_passwd_cb_userdata /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_write                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_get_new_index                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_get_error                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ENGINE_init                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ENGINE_free                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_clear_flags                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_s_socket                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_b64_encode                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              pg_b64_decode                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              X509_get_ext_d2i                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ENGINE_load_private_key             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_free                           /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_set_options                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ERR_get_error                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ERR_put_error                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              EVP_get_digestbyname                /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_build_secret                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              OPENSSL_sk_value                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_snprintf                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              BIO_meth_set_callback_ctrl          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_HMAC_update                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              BIO_meth_get_callback_ctrl          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_digest                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_STORE_set_flags                /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_SaltedPassword                /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              TLS_method                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              OBJ_nid2sn                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_use_certificate_chain_file  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_char_to_encoding                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-exec.o)
              pg_strip_crlf                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              EVP_sha256                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              getpeereid                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              BIO_new                             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_pending                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_b64_enc_len                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              pg_md5_encrypt                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth.o)
              SSL_CTX_get_cert_store              /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_ClientKey                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              GENERAL_NAME_free                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_NAME_get_entry                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_b64_dec_len                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              SSL_free                            /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_read                            /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_STORE_load_locations           /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              OBJ_find_sigid_algs                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_new                             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              OPENSSL_init_ssl                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_new                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_new                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_saslprep                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              X509_NAME_ENTRY_get_data            /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_get_data                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_set_data                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_load_verify_locations       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_freeaddrinfo_all                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              SSL_set_bio                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              scram_H                             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              pg_encoding_mblen                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-exec.o)
              pg_getaddrinfo_all                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              BIO_meth_get_create                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_create                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_NAME_get_index_by_NID          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_get_current_compression         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ASN1_STRING_get0_data               /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_connect                         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              X509_get_signature_nid              /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_set_noblock                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              SSL_shutdown                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_encoding_to_char                 /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              pg_strerror_r                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              pg_sprintf                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              pg_encoding_dsplen                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-misc.o)
              ASN1_STRING_length                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_getnameinfo_all                  /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              pg_tolower                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-exec.o)
              ERR_reason_error_string             /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_check_private_key               /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_clear_options                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_get_encoding_from_locale         /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              BIO_meth_free                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_strong_random                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              SSL_CTX_ctrl                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_CTX_free                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pqGetpwuid                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              SSL_get_current_cipher              /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_inet_net_ntop                    /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              scram_HMAC_init                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-auth-scram.o)
              pg_fprintf                          /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              SSL_get_version                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_encoding_max_length              /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-protocol3.o)
              BIO_meth_get_destroy                /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_set_ex_data                     /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              SSL_set_verify                      /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_destroy                /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_get_gets                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_gets                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_ctrl                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_get_ctrl                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_get_puts                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_puts                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              BIO_meth_set_read                   /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              pg_strcasecmp                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-connect.o)
              X509_get_subject_name               /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ENGINE_finish                       /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ENGINE_by_id                        /opt/ooce/pgsql-13/lib/amd64/libpq.a(fe-secure-openssl.o)
              ld: fatal: symbol referencing errors. No output written to /home/dap/pq-sys/target/debug/deps/pq_sys-f0d6d12067370f8b
              collect2: error: ld returned 1 exit status
error: aborting due to previous error

error: could not compile `pq-sys`

To learn more, run the command again with --verbose.
warning: build failed, waiting for other jobs to finish...
error: linking with `gcc` failed: exit status: 1
  |
  = note: "gcc" "-m64" "-std=c99" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.1lbrw4hjoerhcr1b.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.1lw26qj3w8kx8bjo.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.2i8dm7yabsgp97ic.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.33jwgrksepgrft02.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.3ggemh000opntogx.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.3krcfp5tvhw4uwbd.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.3wrxq3z1n95fw0p3.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.44jjxgz0svj2aio8.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.4s1508tad0ps5af1.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.4zr9ez7dxh57tgj2.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.mtnthew27qt29za.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.nbl8h01k5s71ifq.rcgu.o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8.2kpisq4kjbsu6tdn.rcgu.o" "-Wl,-z" "-Wl,ignore" "-L" "/home/dap/pq-sys/target/debug/deps" "-L" "/opt/ooce/pgsql-13/lib/amd64" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-Wl,-Bstatic" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libtest-535dee8e4f3c883b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libterm-be39c4177d57d29f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgetopts-47c81854ba90f851.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunicode_width-b8d4734d4eda0e2a.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_std-67f67341054cb595.rlib" "/home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib" "-Wl,--start-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd-50bed294e78c95c2.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libpanic_unwind-068f7a96e5aa9c3e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libminiz_oxide-79be3b483094292f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libadler-24b6fed8d1040d6b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libobject-cc9fa5d3172400da.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libaddr2line-5b1c430684c7f47b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgimli-2d01878f44f162f6.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd_detect-0a4773aa77217f2e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_demangle-b4f6a6a789a585b0.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libhashbrown-eda90539ad5b3632.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_alloc-168e3556278a7b7d.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunwind-23043d223378f39c.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcfg_if-389ca2ce1f2d8f07.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liblibc-04d9200ebe50adaf.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liballoc-30bf7d00f2682790.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_core-9bad5249b1907a01.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcore-5d546fb1aabd96b7.rlib" "-Wl,--end-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcompiler_builtins-5d2b8a810efe2040.rlib" "-Wl,-Bdynamic" "-lsocket" "-lposix4" "-lpthread" "-lresolv" "-lnsl" "-lumem" "-lgcc_s" "-lm" "-lrt" "-lpthread" "-lc" "-lssp" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-o" "/home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8" "-nodefaultlibs"
  = note: Undefined                 first referenced
           symbol                        in file
          ERR_clear_error                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_set_default_passwd_cb       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          OPENSSL_sk_pop_free                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_set_options                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_use_PrivateKey                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_get_peer_certificate            /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_set_flags                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CIPHER_get_bits                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CIPHER_get_name                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_link_canary_is_frontend          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          SSL_use_PrivateKey_file             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_ServerKey                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          scram_HMAC_final                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          BIO_int_ctrl                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_vsnprintf                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(pqexpbuffer.o)
          OPENSSL_sk_num                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_write                           /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_set_default_passwd_cb_userdata /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_write                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_get_new_index                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_get_error                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ENGINE_init                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ENGINE_free                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_clear_flags                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_s_socket                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_b64_encode                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          pg_b64_decode                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          X509_get_ext_d2i                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ENGINE_load_private_key             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_free                           /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_set_options                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ERR_get_error                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ERR_put_error                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          EVP_get_digestbyname                /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_build_secret                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          OPENSSL_sk_value                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_snprintf                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_callback_ctrl          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_HMAC_update                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          BIO_meth_get_callback_ctrl          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_digest                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_STORE_set_flags                /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_SaltedPassword                /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          TLS_method                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          OBJ_nid2sn                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_use_certificate_chain_file  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_char_to_encoding                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-exec.o)
          pg_strip_crlf                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          EVP_sha256                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          getpeereid                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          BIO_new                             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_pending                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_b64_enc_len                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          pg_md5_encrypt                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth.o)
          SSL_CTX_get_cert_store              /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_ClientKey                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          GENERAL_NAME_free                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_NAME_get_entry                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_b64_dec_len                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          SSL_free                            /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_read                            /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_STORE_load_locations           /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          OBJ_find_sigid_algs                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_new                             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          OPENSSL_init_ssl                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_new                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_new                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_saslprep                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          X509_NAME_ENTRY_get_data            /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_get_data                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_set_data                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_load_verify_locations       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_freeaddrinfo_all                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          SSL_set_bio                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          scram_H                             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          pg_encoding_mblen                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-exec.o)
          pg_getaddrinfo_all                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          BIO_meth_get_create                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_create                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_NAME_get_index_by_NID          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_get_current_compression         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ASN1_STRING_get0_data               /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_connect                         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_get_signature_nid              /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_set_noblock                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          SSL_shutdown                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_encoding_to_char                 /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          pg_strerror_r                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure.o)
          pg_sprintf                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          pg_encoding_dsplen                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-misc.o)
          ASN1_STRING_length                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_getnameinfo_all                  /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          pg_tolower                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-exec.o)
          ERR_reason_error_string             /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_check_private_key               /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_clear_options                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_get_encoding_from_locale         /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          BIO_meth_free                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_strong_random                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          SSL_CTX_ctrl                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_CTX_free                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pqGetpwuid                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          SSL_get_current_cipher              /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_inet_net_ntop                    /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-connect.o)
          scram_HMAC_init                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-auth-scram.o)
          pg_fprintf                          /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_get_version                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_encoding_max_length              /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-protocol3.o)
          BIO_meth_get_destroy                /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_set_ex_data                     /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          SSL_set_verify                      /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_destroy                /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_get_gets                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_gets                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_ctrl                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_get_ctrl                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_get_puts                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_puts                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          BIO_meth_set_read                   /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          pg_strcasecmp                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          X509_get_subject_name               /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ENGINE_finish                       /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ENGINE_by_id                        /home/dap/pq-sys/target/debug/deps/libpq_sys-d25fbe98107ae389.rlib(fe-secure-openssl.o)
          ld: fatal: symbol referencing errors. No output written to /home/dap/pq-sys/target/debug/deps/smoke-b3d1fb683ab2f0c8
          collect2: error: ld returned 1 exit status
          

error: aborting due to previous error

error: build failed

These symbols appear to be in a combination of OpenSSL and libpqcommon. In the static build case, pq-sys only picks up libpq, not libpqcommon:
https://github.com/sgrif/pq-sys/blob/master/build.rs#L51-L54

        // Link unconditionally statically
        if env::var_os("PQ_LIB_STATIC").is_some() {
            return LinkingOptions::from_name_and_type("pq", LinkType::Static);
        }

It was around this time I decided to see if there were known issues here in pq-sys. Sure enough, there are two PRs that seem relevant:

• SSL issues: issue Static linking needs to include openssl somehow. sgrif/pq-sys#25, PR Add feature for including openssl-sys sgrif/pq-sys#29
• Failure to link to pg-common when static linking: issue Static linking breaks with Postgres 12 sgrif/pq-sys#27, PR Check PostgreSQL major version and add linking flags if needed sgrif/pq-sys#28. This is specific to PostgreSQL 12 and later, according to this thread on the PostgreSQL mailing list (authored by @golddranks, who also weighed in on these issues and submitted 27 and 28).

I have a local clone of pq-sys where I've pulled in both of these changes, but now it's failing with:

PQ_LIB_STATIC=yes cargo test --features=openssl-static
  Downloaded cc v1.0.70
  Downloaded openssl-sys v0.9.66
  Downloaded libc v0.2.101
  Downloaded 3 crates (641.8 KB) in 0.33s
   Compiling pkg-config v0.3.19
   Compiling autocfg v1.0.1
   Compiling cc v1.0.70
   Compiling libc v0.2.101
   Compiling pq-sys v0.4.6 (/home/dap/pq-sys)
   Compiling openssl-sys v0.9.66
error: linking with `gcc` failed: exit status: 1
  |
  = note: "gcc" "-m64" "-std=c99" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.11faowje89f4n8no.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.165dsdqwz27k5g70.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.191vxhoocbef8xa1.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.1cgcyopr51b3l4l7.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.1fnzjs97yhbmsiwq.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.1fxhthv3c8ek5nu5.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.22aia9wtgup9i3r1.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.2a80pyym92yws6nz.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.2legzw97ba6g0srs.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.4g1p1sq0vogdnxo3.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.4jii2s5vw8dkw7a1.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.4nw9rucalty2smn0.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.57yj00f671gzzqj8.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.5d4wyqvw5s6ot9pe.rcgu.o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df.5dr7jtn5uekrfb4g.rcgu.o" "-Wl,-z" "-Wl,ignore" "-L" "/home/dap/pq-sys/target/debug/deps" "-L" "/opt/ooce/pgsql-13/lib/amd64" "-L" "/usr/lib/64" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-Wl,-Bstatic" "-Wl,--whole-archive" "-lpq" "-Wl,--no-whole-archive" "-Wl,--whole-archive" "-lpgcommon" "-Wl,--no-whole-archive" "-Wl,--whole-archive" "-lpgport" "-Wl,--no-whole-archive" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libtest-535dee8e4f3c883b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libterm-be39c4177d57d29f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgetopts-47c81854ba90f851.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunicode_width-b8d4734d4eda0e2a.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_std-67f67341054cb595.rlib" "/home/dap/pq-sys/target/debug/deps/libopenssl_sys-0948b11797b6d460.rlib" "/home/dap/pq-sys/target/debug/deps/liblibc-537f8772b02fb557.rlib" "-Wl,--start-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd-50bed294e78c95c2.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libpanic_unwind-068f7a96e5aa9c3e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libminiz_oxide-79be3b483094292f.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libadler-24b6fed8d1040d6b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libobject-cc9fa5d3172400da.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libaddr2line-5b1c430684c7f47b.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libgimli-2d01878f44f162f6.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libstd_detect-0a4773aa77217f2e.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_demangle-b4f6a6a789a585b0.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libhashbrown-eda90539ad5b3632.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_alloc-168e3556278a7b7d.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libunwind-23043d223378f39c.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcfg_if-389ca2ce1f2d8f07.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liblibc-04d9200ebe50adaf.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/liballoc-30bf7d00f2682790.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/librustc_std_workspace_core-9bad5249b1907a01.rlib" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcore-5d546fb1aabd96b7.rlib" "-Wl,--end-group" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib/libcompiler_builtins-5d2b8a810efe2040.rlib" "-Wl,-Bdynamic" "-lssl" "-lcrypto" "-lsocket" "-lposix4" "-lpthread" "-lresolv" "-lnsl" "-lumem" "-lgcc_s" "-lm" "-lrt" "-lpthread" "-lc" "-lssp" "-L" "/home/dap/.rustup/toolchains/stable-x86_64-unknown-illumos/lib/rustlib/x86_64-unknown-illumos/lib" "-o" "/home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df" "-nodefaultlibs"
  = note: ld: fatal: symbol 'pqsignal' is multiply-defined:
             (file /opt/ooce/pgsql-13/lib/amd64/libpq.a(legacy-pqsignal.o) type=FUNC; file /opt/ooce/pgsql-13/lib/amd64/libpgport.a(pqsignal.o) type=FUNC);
          ld: fatal: file processing errors. No output written to /home/dap/pq-sys/target/debug/deps/pq_sys-211adc8611e412df
          collect2: error: ld returned 1 exit status
          

error: aborting due to previous error

error: could not compile `pq-sys`

To learn more, run the command again with --verbose.
warning: build failed, waiting for other jobs to finish...
error: build failed

The symbol "pqsignal" appears to be in both static objects and the linker's unhappy. I have not dug further yet.

Solutions and workarounds that work

Regardless of anything else, you need to have libpq installed and pg_config must be on your PATH. On OmniOS and Helios, this is provided by "pkg:/ooce/library/postgresql-13" and you'll want /opt/ooce/bin on your PATH.

Then you can do any of these things:

• Build with RUSTFLAGS="-Clink-args=-R$(pg_config --libdir)" in the environment. This assumes that pq-sys will use pg_config to find PostgreSQL. If you've got pkg-config set up, or you've overridden PQ_LIB_DIR or something like that, this will definitely do the wrong thing.
• Build as normal. To run anything, set LD_LIBRARY_PATH=/opt/ooce/pgsql-13/lib/amd64. It's generally not recommended to set LD_LIBRARY_PATH as a matter of course, partly because it affects everything. Needing to do this should be considered a bug.
• Use elfedit to update the RPATH of the produced binaries. This breaks the usual cargo run and cargo test workflow -- you'd have to always build first, then use elfedit, then cargo run or cargo test.

Explicitly setting RUSTFLAGS in the environment seems like the best option to me, so my plan will be to put that in the README.

The only other avenues would be:

• Try using cargo:rust-link-arg in pq-sys, either by forking it or putting it behind a conditional check.
• Debug the static linking issues.

[luqmana]

re: the static linking error about pqsignal, I asked on their IRC channel (#postgresql on LiberaChat). The suggestion I got was to try passing -fcommon (*) since this behaviour apparently changed with GCC 10 (default went from -fno-common to -fcommon). But I wasn't actually able to get it to work, I wonder if there are different semantics for -fcommon on illumos?

(*) not even sure this would be a good option as I believe even though both functions are named the same they have different semantics.

[davepacheco]

Thinking about this more: I do think advising people to set RUSTFLAGS is less fraught than using LD_LIBRARY_PATH, but it's still pretty sucky because there's no point where we'd be able to remove this workaround until Rust provides a solution for *-sys crates to set RPATH (rust-lang/cargo#5077). So I might spend some time looking at updating pq-sys to use the unstable linker arg option.

[davepacheco]

This change to pq-sys appears to do the right thing:

diff --git a/build.rs b/build.rs
index aeb0e7e..c8e5dcb 100644
--- a/build.rs
+++ b/build.rs
@@ -83,6 +83,7 @@ fn main() {
 
     if let Ok(lib_dir) = env::var("PQ_LIB_DIR") {
         println!("cargo:rustc-link-search=native={}", lib_dir);
+        println!("cargo:rustc-link-arg=-Wl,-R{}", lib_dir);
     } else if configured_by_pkg_config() {
         return // pkg_config does everything for us, including output for cargo
     } else if configured_by_vcpkg() {
@@ -90,6 +91,7 @@ fn main() {
     } else if let Some(path) = pg_config_output("--libdir") {
         let path = replace_homebrew_path_on_mac(path);
         println!("cargo:rustc-link-search=native={}", path);
+        println!("cargo:rustc-link-arg=-Wl,-R{}", path);
     }
     println!("cargo:rustc-link-lib={}", LinkingOptions::from_env());
 }

If I run this with nightly, I get the right behavior:

$ cargo +nightly test
    Finished test [unoptimized + debuginfo] target(s) in 0.01s
     Running unittests (target/debug/deps/pq_sys-354a9fbeed88ac43)

running 8 tests
test bindgen_test_layout__PQprintOpt ... ok
test bindgen_test_layout__PQconninfoOption ... ok
test bindgen_test_layout___sFILE ... ok
test bindgen_test_layout___sbuf ... ok
test bindgen_test_layout__bindgen_ty_8 ... ok
test bindgen_test_layout__bindgen_ty_8__bindgen_ty_1 ... ok
test bindgen_test_layout_pgNotify ... ok
test bindgen_test_layout_pgresAttDesc ... ok

test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running tests/smoke.rs (target/debug/deps/smoke-622ca0eb1c9982b1)

running 1 test
test test_ssl_init ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests pq_sys

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

If I run it with stable cargo (1.54), I get a warning related to trying to use an unstable feature, and it appears to ignore it, resulting in the broken test binary and failed test:

$ cargo test
   Compiling pq-sys v0.4.6 (/home/dap/pq-sys-dynamic)
warning: cargo:rustc-link-arg requires -Zextra-link-arg flag
    Finished test [unoptimized + debuginfo] target(s) in 2.26s
     Running unittests (target/debug/deps/pq_sys-f0d6d12067370f8b)

running 8 tests
test bindgen_test_layout__PQconninfoOption ... ok
test bindgen_test_layout___sFILE ... ok
test bindgen_test_layout___sbuf ... ok
test bindgen_test_layout__PQprintOpt ... ok
test bindgen_test_layout__bindgen_ty_8 ... ok
test bindgen_test_layout_pgNotify ... ok
test bindgen_test_layout__bindgen_ty_8__bindgen_ty_1 ... ok
test bindgen_test_layout_pgresAttDesc ... ok

test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running tests/smoke.rs (target/debug/deps/smoke-b3d1fb683ab2f0c8)
ld.so.1: smoke-b3d1fb683ab2f0c8: fatal: libpq.so.5: open failed: No such file or directory
error: test failed, to rerun pass '--test smoke'

Caused by:
  process didn't exit successfully: `/home/dap/pq-sys-dynamic/target/debug/deps/smoke-b3d1fb683ab2f0c8` (signal: 9, SIGKILL: kill)

That's as expected, and no worse than what happens today without the change.

I wanted to make sure this wouldn't totally break people using stable cargo that's so old that it doesn't even know about this flag. So I changed the key to something else:

$ git diff
diff --git a/build.rs b/build.rs
index c8e5dcb..1327c3c 100644
--- a/build.rs
+++ b/build.rs
@@ -83,7 +83,7 @@ fn main() {
 
     if let Ok(lib_dir) = env::var("PQ_LIB_DIR") {
         println!("cargo:rustc-link-search=native={}", lib_dir);
-        println!("cargo:rustc-link-arg=-Wl,-R{}", lib_dir);
+        println!("cargo:rustc-link-argNOPE=-Wl,-R{}", lib_dir);
     } else if configured_by_pkg_config() {
         return // pkg_config does everything for us, including output for cargo
     } else if configured_by_vcpkg() {
@@ -91,7 +91,7 @@ fn main() {
     } else if let Some(path) = pg_config_output("--libdir") {
         let path = replace_homebrew_path_on_mac(path);
         println!("cargo:rustc-link-search=native={}", path);
-        println!("cargo:rustc-link-arg=-Wl,-R{}", path);
+        println!("cargo:rustc-link-argNOPE=-Wl,-R{}", path);
     }
     println!("cargo:rustc-link-lib={}", LinkingOptions::from_env());
 }

On both stable and nightly, this produces no warnings, a broken binary, and a failed test:

$ cargo test
   Compiling pq-sys v0.4.6 (/home/dap/pq-sys-dynamic)
    Finished test [unoptimized + debuginfo] target(s) in 1.54s
     Running unittests (target/debug/deps/pq_sys-f0d6d12067370f8b)

running 8 tests
test bindgen_test_layout__PQconninfoOption ... ok
test bindgen_test_layout__PQprintOpt ... ok
test bindgen_test_layout___sFILE ... ok
test bindgen_test_layout___sbuf ... ok
test bindgen_test_layout_pgNotify ... ok
test bindgen_test_layout__bindgen_ty_8__bindgen_ty_1 ... ok
test bindgen_test_layout_pgresAttDesc ... ok
test bindgen_test_layout__bindgen_ty_8 ... ok

test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running tests/smoke.rs (target/debug/deps/smoke-b3d1fb683ab2f0c8)
ld.so.1: smoke-b3d1fb683ab2f0c8: fatal: libpq.so.5: open failed: No such file or directory
error: test failed, to rerun pass '--test smoke'

Caused by:
  process didn't exit successfully: `/home/dap/pq-sys-dynamic/target/debug/deps/smoke-b3d1fb683ab2f0c8` (signal: 9, SIGKILL: kill)

$ cargo +nightly test
   Compiling pq-sys v0.4.6 (/home/dap/pq-sys-dynamic)
    Finished test [unoptimized + debuginfo] target(s) in 2.45s
     Running unittests (target/debug/deps/pq_sys-354a9fbeed88ac43)

running 8 tests
test bindgen_test_layout__PQconninfoOption ... ok
test bindgen_test_layout__PQprintOpt ... ok
test bindgen_test_layout___sFILE ... ok
test bindgen_test_layout__bindgen_ty_8 ... ok
test bindgen_test_layout___sbuf ... ok
test bindgen_test_layout_pgNotify ... ok
test bindgen_test_layout__bindgen_ty_8__bindgen_ty_1 ... ok
test bindgen_test_layout_pgresAttDesc ... ok

test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running tests/smoke.rs (target/debug/deps/smoke-622ca0eb1c9982b1)
ld.so.1: smoke-622ca0eb1c9982b1: fatal: libpq.so.5: open failed: No such file or directory
error: test failed, to rerun pass '--test smoke'

Caused by:
  process didn't exit successfully: `/home/dap/pq-sys-dynamic/target/debug/deps/smoke-622ca0eb1c9982b1` (signal: 9, SIGKILL: kill)

I think this is the best we can hope for. I was a little surprised it worked this way, but the Cargo docs do say that any unrecognized key is treated as metadata. So this will create confusing, useless metadata, but I believe the only consumers are people trying to consume a particular metadata key, so I don't think this is a problem.

@luqmana What do you think of this? If this seems reasonable, I think I'll create a PR upstream. In parallel I may create our own fork and update Omicron to use that with a "patch" section in Cargo.toml, just because upstream doesn't seem to be actively maintained. We can remove the "patch" and fork if/when upstream does land it.

[luqmana]

I think that's a pretty targeted and reasonable approach 👍🏾

One thing is, should we use -Wl,-R or -Wl,-rpath? Just checking on mac for example, man ld doesn't list -R just -rpath

[davepacheco]

This is a good question. I'm inclined to test this on Mac, illumos, and Linux, then add an appropriate cfg! directive to limit it to the systems where we believe it works. (I can test both syntaxes.) I'm reluctant to spend a lot of time trying to figure out a portable approach, but that's easy enough and ensures it fixes us, doesn't break anybody, and might fix other folks as well. What do you think?

[luqmana]

Yea, that sounds good. Just wanted to make sure we don't introduce a "unknown linker flag" sorta error on major platform it already works on.

[davepacheco]

I've filed an issue upstream on pq-sys: sgrif/pq-sys#36
I've submitted a PR to fix it: sgrif/pq-sys#37

Since upstream does not seem actively maintained, I've created our own fork here:
https://github.com/oxidecomputer/pq-sys
It has a README explaining the situation:
https://github.com/oxidecomputer/pq-sys/blob/oxide/README.oxide.md

I will put together a PR for Omicron that points us at that fork.

[davepacheco]

Well, I think that approach is a bust. See #220 for details.

I think at this point the remaining options are:

• stick with the RUSTFLAGS workaround indefinitely
• figure how to get static linking to libpq working
• maybe the same approach would work if we add a build.rs to omicron that emits the instruction instead of putting it into pq-sys? But where would we get the path, without replicating the same logic as pq-sys's build.rs has? Maybe we could have that emit metadata that tells us where it found it.

[davepacheco]

I tried the approach where we add a build.rs to omicron that consumes metadata from the pq-sys build and emits the linker instruction rather than putting the linker instruction into pq-sys. Here's the change to pq-sys (from its "master") to emit the metadata:

diff --git a/build.rs b/build.rs
index aeb0e7e..f891d99 100644
--- a/build.rs
+++ b/build.rs
@@ -6,6 +6,7 @@ extern crate vcpkg;
 
 use std::process::Command;
 use std::env;
+use std::path::Path;
 use std::path::PathBuf;
 use std::fmt::{self, Display};
 
@@ -83,29 +84,37 @@ fn main() {
 
     if let Ok(lib_dir) = env::var("PQ_LIB_DIR") {
         println!("cargo:rustc-link-search=native={}", lib_dir);
-    } else if configured_by_pkg_config() {
+        emit_metadata(std::slice::from_ref(&Path::new(&lib_dir)));
+    } else if configure_using_pkg_config() {
         return // pkg_config does everything for us, including output for cargo
-    } else if configured_by_vcpkg() {
+    } else if configure_using_vcpkg() {
         return // vcpkg does everything for us, including output for cargo
     } else if let Some(path) = pg_config_output("--libdir") {
         let path = replace_homebrew_path_on_mac(path);
         println!("cargo:rustc-link-search=native={}", path);
+        emit_metadata(std::slice::from_ref(&Path::new(&path)));
     }
     println!("cargo:rustc-link-lib={}", LinkingOptions::from_env());
 }
 
 #[cfg(feature = "pkg-config")]
-fn configured_by_pkg_config() -> bool {
-    pkg_config::probe_library("libpq").is_ok()
+fn configure_using_pkg_config() -> bool {
+    let pkg = pkg_config::probe_library("libpq");
+    if let Ok(pkg) = pkg {
+        emit_metadata(&pkg.link_paths.map(|pathbuf| &pathbuf));
+        true
+    } else {
+        false
+    }
 }
 
 #[cfg(not(feature = "pkg-config"))]
-fn configured_by_pkg_config() -> bool {
+fn configure_using_pkg_config() -> bool {
     false
 }
 
 #[cfg(target_env = "msvc")]
-fn configured_by_vcpkg() -> bool {
+fn configure_using_vcpkg() -> bool {
     vcpkg::probe_package("libpq").map(|_| {
 
         // found libpq which depends on openssl
@@ -123,10 +132,14 @@ fn configured_by_vcpkg() -> bool {
 }
 
 #[cfg(not(target_env = "msvc"))]
-fn configured_by_vcpkg() -> bool {
+fn configure_using_vcpkg() -> bool {
     false
 }
 
+fn emit_metadata(link_paths: &[&Path]) {
+    println!("cargo:libdir={}", link_paths.iter().map(|p| p.to_string_lossy()).collect::<Vec<_>>().join(":"));
+}
+
 fn pg_config_path() -> PathBuf {
     if let Ok(target) = env::var("TARGET") {
         let pg_config_for_target = &format!("PG_CONFIG_{}", target.to_ascii_uppercase().replace("-", "_"));

However, I found that it did not appear in the omicron build script's environment. And then I found this quote in the docs:

Note that metadata is only passed to immediate dependents, not transitive dependents.

But omicron is not an immediate dependent of pq-sys; diesel is.