ports-mgmt/sccache-overlay: Use Unix sockets and fix Poudriere builds

Poudriere builder jails seem to share one network namespace which
is problematic when building packages in parallel since sccache
binds to localhost:4226. This then leads to jails reusing the
sccache instance from other jails which is problematic and can
cause failures like

1/ missing files due to the filesystem separation:

  sccache: caused by: No such file or directory (os error 2)

2/ the sccache instance suddenly going away when a parallel build
finishes and Poudriere shuts down the jail and kills all of its
processes:

  sccache: error: failed to execute compile
  sccache: caused by: Failed to send data to or receive data from server
  sccache: caused by: Failed to read response header
  sccache: caused by: failed to fill whole buffer

To properly namespace sccache instances we can use Unix sockets.
There is a request [1] for adding support for Unix sockets
upstream but no code has been written for that AFAICT. This
commit hacks in support for them which should be just about ok
for our purposes.

[1] https://github.com/mozilla/sccache/issues/933

Tested by:	jrm

5 files changed, 116 insertions, 4 deletions

diff --git a/ports-mgmt/sccache-overlay/Makefile b/ports-mgmt/sccache-overlay/Makefile
index 3aa94da0d083..791f931c6de0 100644
--- a/ports-mgmt/sccache-overlay/Makefile
+++ b/ports-mgmt/sccache-overlay/Makefile
@@ -1,7 +1,7 @@
 PORTNAME=	sccache
 DISTVERSIONPREFIX=	v
 DISTVERSION=	0.2.15
-PORTREVISION=	3
+PORTREVISION=	4
 CATEGORIES=	ports-mgmt
 PKGNAMESUFFIX=	-overlay
 
diff --git a/ports-mgmt/sccache-overlay/files/Mk/bsd.overlay.mk b/ports-mgmt/sccache-overlay/files/Mk/bsd.overlay.mk
index da959aae4e3a..90f590b701fc 100644
--- a/ports-mgmt/sccache-overlay/files/Mk/bsd.overlay.mk
+++ b/ports-mgmt/sccache-overlay/files/Mk/bsd.overlay.mk
@@ -1,8 +1,8 @@
 .if !defined(_SCCACHE_OVERLAY_INCLUDED) && !defined(NO_SCCACHE)
 _SCCACHE_OVERLAY_INCLUDED=	yes
-_SCCACHE_SIZE_DEFAULT=		16G
 
-SCCACHE_SIZE?=	${_SCCACHE_SIZE_DEFAULT}
+SCCACHE_SIZE?=		16G
+SCCACHE_UNIX_LISTEN?=	/tmp/sccache-overlay.socket
 
 _SCCACHE_LIBS=	${LOCALBASE}/share/sccache/overlay/lib
 .for odir in ${OVERLAYS}
@@ -26,7 +26,8 @@ IGNORE=		SCCACHE_DIR not set
 _SCCACHE_ENV=	RUSTC_WRAPPER="${SCCACHE_BIN}" \
 		SCCACHE_CACHE_SIZE="${SCCACHE_SIZE}" \
 		SCCACHE_DIR="${SCCACHE_DIR}" \
-		SCCACHE_MAX_FRAME_LENGTH=104857600
+		SCCACHE_MAX_FRAME_LENGTH=104857600 \
+		SCCACHE_UNIX_LISTEN="${SCCACHE_UNIX_LISTEN}"
 CONFIGURE_ENV+=	${_SCCACHE_ENV}
 MAKE_ENV+=	${_SCCACHE_ENV}
 
diff --git a/ports-mgmt/sccache-overlay/files/patch-src_client.rs b/ports-mgmt/sccache-overlay/files/patch-src_client.rs
new file mode 100644
index 000000000000..84ed56b0e4eb
--- /dev/null
+++ b/ports-mgmt/sccache-overlay/files/patch-src_client.rs
@@ -0,0 +1,38 @@
+--- src/client.rs.orig	2021-01-12 22:25:51 UTC
++++ src/client.rs
+@@ -17,20 +17,21 @@ use crate::protocol::{Request, Response};
+ use crate::util;
+ use byteorder::{BigEndian, ByteOrder};
+ use retry::{delay::Fixed, retry};
++use std::env;
+ use std::io::{self, BufReader, BufWriter, Read};
+-use std::net::TcpStream;
++use std::os::unix::net::UnixStream;
+ 
+ /// A connection to an sccache server.
+ pub struct ServerConnection {
+     /// A reader for the socket connected to the server.
+-    reader: BufReader<TcpStream>,
++    reader: BufReader<UnixStream>,
+     /// A writer for the socket connected to the server.
+-    writer: BufWriter<TcpStream>,
++    writer: BufWriter<UnixStream>,
+ }
+ 
+ impl ServerConnection {
+     /// Create a new connection using `stream`.
+-    pub fn new(stream: TcpStream) -> io::Result<ServerConnection> {
++    pub fn new(stream: UnixStream) -> io::Result<ServerConnection> {
+         let writer = stream.try_clone()?;
+         Ok(ServerConnection {
+             reader: BufReader::new(stream),
+@@ -65,7 +66,8 @@ impl ServerConnection {
+ /// Establish a TCP connection to an sccache server listening on `port`.
+ pub fn connect_to_server(port: u16) -> io::Result<ServerConnection> {
+     trace!("connect_to_server({})", port);
+-    let stream = TcpStream::connect(("127.0.0.1", port))?;
++    let unix_socket_path = env::var_os("SCCACHE_UNIX_LISTEN").unwrap();
++    let stream = UnixStream::connect(unix_socket_path)?;
+     ServerConnection::new(stream)
+ }
+ 
diff --git a/ports-mgmt/sccache-overlay/files/patch-src_server.rs b/ports-mgmt/sccache-overlay/files/patch-src_server.rs
new file mode 100644
index 000000000000..70e9f5fcd2a2
--- /dev/null
+++ b/ports-mgmt/sccache-overlay/files/patch-src_server.rs
@@ -0,0 +1,66 @@
+--- src/server.rs.orig	2021-01-12 22:25:51 UTC
++++ src/server.rs
+@@ -63,6 +63,7 @@ use tokio_io::{AsyncRead, AsyncWrite};
+ use tokio_serde_bincode::{ReadBincode, WriteBincode};
+ use tokio_tcp::TcpListener;
+ use tokio_timer::{Delay, Timeout};
++use tokio_uds::UnixListener;
+ use tower::Service;
+ 
+ use crate::errors::*;
+@@ -430,8 +431,9 @@ pub fn start_server(config: &Config, port: u16) -> Res
+ }
+ 
+ pub struct SccacheServer<C: CommandCreatorSync> {
++    unix_socket_path: String,
+     runtime: Runtime,
+-    listener: TcpListener,
++    listener: UnixListener,
+     rx: mpsc::Receiver<ServerMessage>,
+     timeout: Duration,
+     service: SccacheService<C>,
+@@ -447,8 +449,8 @@ impl<C: CommandCreatorSync> SccacheServer<C> {
+         dist_client: DistClientContainer,
+         storage: Arc<dyn Storage>,
+     ) -> Result<SccacheServer<C>> {
+-        let addr = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port);
+-        let listener = TcpListener::bind(&SocketAddr::V4(addr))?;
++        let unix_socket_path = env::var("SCCACHE_UNIX_LISTEN")?;
++        let listener = UnixListener::bind(&unix_socket_path)?;
+ 
+         // Prepare the service which we'll use to service all incoming TCP
+         // connections.
+@@ -457,6 +459,7 @@ impl<C: CommandCreatorSync> SccacheServer<C> {
+         let service = SccacheService::new(dist_client, storage, &client, pool, tx, info);
+ 
+         Ok(SccacheServer {
++            unix_socket_path,
+             runtime,
+             listener,
+             rx,
+@@ -493,7 +496,7 @@ impl<C: CommandCreatorSync> SccacheServer<C> {
+     /// Returns the port that this server is bound to
+     #[allow(dead_code)]
+     pub fn port(&self) -> u16 {
+-        self.listener.local_addr().unwrap().port()
++        0
+     }
+ 
+     /// Runs this server to completion.
+@@ -510,6 +513,7 @@ impl<C: CommandCreatorSync> SccacheServer<C> {
+ 
+     fn _run<'a>(self, shutdown: Box<dyn Future<Item = (), Error = ()> + 'a>) -> io::Result<()> {
+         let SccacheServer {
++            unix_socket_path,
+             mut runtime,
+             listener,
+             rx,
+@@ -593,6 +597,8 @@ impl<C: CommandCreatorSync> SccacheServer<C> {
+                     io::Error::new(io::ErrorKind::Other, e)
+                 }
+             })?;
++
++        std::fs::remove_file(unix_socket_path);
+ 
+         info!("ok, fully shutting down now");
+ 
diff --git a/ports-mgmt/sccache-overlay/files/pkg-message.in b/ports-mgmt/sccache-overlay/files/pkg-message.in
index 800247b91ec1..d84fccb4eceb 100644
--- a/ports-mgmt/sccache-overlay/files/pkg-message.in
+++ b/ports-mgmt/sccache-overlay/files/pkg-message.in
@@ -9,6 +9,13 @@ Add this to /etc/make.conf
 SCCACHE_DIR=	${HOME}/.sccache
 OVERLAYS+=	%%DATADIR%%/overlay
 
+During the build a sccache server is started, bound to
+${SCCACHE_UNIX_LISTEN} (default /tmp/sccache-overlay.socket). It
+automatically shuts down after 600 seconds if nothing is using
+it. To cleanly shut it down earlier use
+
+$ make sccache-stop
+
 Poudriere setup
 ===============