diff --git a/src/Propellor/Property/SiteSpecific/JoeySites.hs b/src/Propellor/Property/SiteSpecific/JoeySites.hs
index 15dbd5e..fe961ac 100644
--- a/src/Propellor/Property/SiteSpecific/JoeySites.hs
+++ b/src/Propellor/Property/SiteSpecific/JoeySites.hs
@@ -70,7 +70,10 @@ oldUseNetServer hosts = propertyList ("olduse.net server")
 	datadir = "/var/spool/oldusenet"
 
 oldUseNetShellBox :: Property
-oldUseNetShellBox = oldUseNetInstalled "oldusenet"
+oldUseNetShellBox = propertyList "olduse.net shellbox"
+	[ oldUseNetInstalled "oldusenet"
+	, Service.running "oldusenet"
+	]
 
 oldUseNetInstalled :: Apt.Package -> Property
 oldUseNetInstalled pkg = check (not <$> Apt.isInstalled pkg) $