Merge branch 'joeyconfig'

Makefile

@@ -1,26 +1,12 @@
 CABAL?=cabal
 
-DEBDEPS=gnupg ghc cabal-install libghc-missingh-dev libghc-ansi-terminal-dev libghc-ifelse-dev libghc-unix-compat-dev libghc-hslogger-dev libghc-network-dev libghc-quickcheck2-dev libghc-mtl-dev libghc-monadcatchio-transformers-dev
+# this target is provided (and is first) to keep old versions of the
-
+# propellor cron job working, and will eventually be removed
-# this target is provided to keep old versions of the propellor cron job
+run: build
-# working, and will eventually be removed
-run: deps build
 	./propellor
 
 dev: build tags
 
-build: dist/setup-config
-	@if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi
-	@ln -sf dist/build/propellor-config/propellor-config propellor
-
-deps:
-	@if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS) || (apt-get update && apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS)); fi || true
-	@if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install libghc-async-dev || (cabal update; cabal install async); fi || true
-
-dist/setup-config: propellor.cabal
-	@if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
-	@$(CABAL) configure
-
 install: propellor.1
 	install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor
 	install -s dist/build/propellor/propellor $(DESTDIR)/usr/bin/propellor
@@ -56,3 +42,15 @@ hackage:
 	@cabal upload dist/*.tar.gz
 
 .PHONY: tags
+
+# The rules below are only used when bootstrapping new propellor
+# installations and building packages; propellor contains equivilant
+# haskell code that it uses to re-build itself.
+
+build: dist/setup-config
+	@if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi
+	@ln -sf dist/build/propellor-config/propellor-config propellor
+
+dist/setup-config: propellor.cabal
+	@if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
+	@$(CABAL) configure

config-joey.hs

@@ -43,7 +43,6 @@ hosts =                --                  (o)  `
 	, clam
 	, orca
 	, kite
-	, diatom
 	, elephant
 	, beaver
 	] ++ monsters
@@ -85,8 +84,9 @@ clam = standardSystem "clam.kitenet.net" Unstable "amd64"
 	& Ssh.randomHostKeys
 	& Apt.unattendedUpgrades
 	& Network.ipv6to4
-	& Tor.named "kite1" Tor.isRelay'
+	& Tor.isRelay
-	& Postfix.satellite
+	& Tor.named "kite1"
+	& Tor.bandwidthRate (Tor.PerMonth "400 GB")
 
 	& Docker.configured
 	& Docker.garbageCollected `period` Daily
@@ -223,16 +223,6 @@ kite = standardSystemUnhardened "kite.kitenet.net" Testing "amd64"
 	& branchableSecondary
 	& Dns.secondaryFor ["animx"] hosts "animx.eu.org"
 
-diatom :: Host
-diatom = standardSystem "diatom.kitenet.net" (Stable "wheezy") "amd64"
-	[ "dying" ]
-	& ipv4 "107.170.31.195"
-	& Ssh.hostKeys hostContext
-		[ (SshDsa, "ssh-dss AAAAB3NzaC1kc3MAAACBAO9tnPUT4p+9z7K6/OYuiBNHaij4Nzv5YVBih1vMl+ALz0gYAj8RWJzXmqp5buFAyfgOoLw+H9s1bBS01Sy3i07Dm6cx1fWG4RXL/E/3w1tavX99GD2bBxDBu890ebA5Tp+eFRJkS9+JwSvFiF6CP7NbVjifCagoUO56Ig048RwDAAAAFQDPY2xM3q6KwsVQliel23nrd0rV2QAAAIEAga3hj1hL00rYPNnAUzT8GAaSP62S4W68lusErH+KPbsMwFBFY/Ib1FVf8k6Zn6dZLh/HH/RtJi0JwdzPI1IFW+lwVbKfwBvhQ1lw9cH2rs1UIVgi7Wxdgfy8gEWxf+QIqn62wG+Ulf/HkWGvTrRpoJqlYRNS/gnOWj9Z/4s99koAAACBAM/uJIo2I0nK15wXiTYs/NYUZA7wcErugFn70TRbSgduIFH6U/CQa3rgHJw9DCPCQJLq7pwCnFH7too/qaK+czDk04PsgqV0+Jc7957gU5miPg50d60eJMctHV4eQ1FpwmGGfXxRBR9k2ZvikWYatYir3L6/x1ir7M0bA9IzNU45")
-		, (SshRsa, "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA2QAJEuvbTmaN9ex9i9bjPhMGj+PHUYq2keIiaIImJ+8mo+yKSaGUxebG4tpuDPx6KZjdycyJt74IXfn1voGUrfzwaEY9NkqOP3v6OWTC3QeUGqDCeJ2ipslbEd9Ep9XBp+/ldDQm60D0XsIZdmDeN6MrHSbKF4fXv1bqpUoUILk=")
-		]
-	& alias "ns2.kitenet.net"
-
 elephant :: Host
 elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
 	[ "Storage, big data, and backups, omnomnom!"
@@ -265,7 +255,6 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
 	
 	& alias "znc.kitenet.net"
 	& JoeySites.ircBouncer
-	-- I'd rather this were on diatom, but it needs unstable.
 	& alias "kgb.kitenet.net"
 	& JoeySites.kgbServer
 	

debian/changelog

@@ -1,3 +1,14 @@
+propellor (2.2.0) UNRELEASED; urgency=medium
+
+  * When running shimmed (eg in a docker container),
+    improve process name visible in ps.
+  * Add shebang to cron.daily etc files. 
+  * Some changes to tor configuration, minor API change.
+  * Propellor now builds itself, and gets its build dependencies installed
+    when deploying to a new host, without needing the Makefile.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 16 Feb 2015 19:00:48 -0400
+
 propellor (2.1.0) unstable; urgency=medium
 
   * Additional tor properties, including support for making relays, 

propellor.cabal

@@ -1,5 +1,5 @@
 Name: propellor
-Version: 2.1.0
+Version: 2.2.0
 Cabal-Version: >= 1.6
 License: BSD3
 Maintainer: Joey Hess <id@joeyh.name>
@@ -128,6 +128,7 @@ Library
     Propellor.Types.Result
     Propellor.Types.CmdLine
   Other-Modules:
+    Propellor.Bootstrap
     Propellor.Git
     Propellor.Gpg
     Propellor.Spin

src/Propellor/Bootstrap.hs

@@ -0,0 +1,97 @@
+module Propellor.Bootstrap (
+	bootstrapPropellorCommand,
+	installGitCommand,
+	buildPropellor,
+) where
+
+import Propellor
+import Utility.SafeCommand
+
+import System.Posix.Files
+import Data.List
+
+type ShellCommand = String
+
+-- Shell command line to build propellor, used when bootstrapping on a new
+-- host. Should be run inside the propellor source tree, and will install
+-- all necessary build dependencies.
+bootstrapPropellorCommand :: ShellCommand
+bootstrapPropellorCommand = "if ! test -x ./propellor; then " ++ go ++ "; fi"
+  where
+	go = intercalate " && "
+		[ depsCommand
+		, buildCommand
+		]
+
+buildCommand :: ShellCommand
+buildCommand = intercalate " && "
+	[ "cabal configure"
+	, "cabal build"
+	, "ln -sf dist/build/propellor-config/propellor-config propellor"
+	]
+
+depsCommand :: ShellCommand
+depsCommand = 
+	"(" ++ aptinstall debdeps ++ " || (apt-get update && " ++ aptinstall debdeps ++ ")) && "
+	++ "(" ++ aptinstall ["libghc-async-dev"] ++ " || (" ++ cabalinstall ["async"] ++  ")) || "
+	++ "(" ++ cabalinstall ["--only-dependencies"] ++ ")"
+  where
+	aptinstall ps = "apt-get --no-upgrade --no-install-recommends -y install " ++ unwords ps
+
+	cabalinstall ps = "cabal update; cabal install " ++ unwords ps
+
+	-- This is the same build deps listed in debian/control.
+	debdeps =
+		[ "gnupg"
+		, "ghc"
+		, "cabal-install"
+		-- async is not available in debian stable
+		-- , "libghc-async-dev"
+		, "libghc-missingh-dev"
+		, "libghc-hslogger-dev"
+		, "libghc-unix-compat-dev"
+		, "libghc-ansi-terminal-dev"
+		, "libghc-ifelse-dev"
+		, "libghc-network-dev"
+		, "libghc-quickcheck2-dev"
+		, "libghc-mtl-dev"
+		, "libghc-monadcatchio-transformers-dev"
+		]
+
+
+installGitCommand :: ShellCommand
+installGitCommand = "if ! git --version >/dev/null; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git; fi"
+
+buildPropellor :: IO ()
+buildPropellor = unlessM (actionMessage "Propellor build" build) $
+	errorMessage "Propellor build failed!"
+
+-- Build propellor using cabal, and symlink propellor to where cabal
+-- leaves the built binary.
+--
+-- For speed, only runs cabal configure when it's not been run before.
+-- If the build fails cabal may need to have configure re-run.
+build :: IO Bool
+build = catchBoolIO $ do
+	make "dist/setup-config" ["propellor.cabal"] $
+		cabal ["configure"]
+	unlessM (cabal ["build"]) $ do
+		void $ cabal ["configure"]
+		unlessM (cabal ["build"]) $
+			error "cabal build failed"
+	nukeFile "propellor"
+	createSymbolicLink "dist/build/propellor-config/propellor-config" "propellor"
+	return True
+
+make :: FilePath -> [FilePath] -> IO Bool -> IO ()
+make dest srcs builder = do
+	dt <- getmtime dest
+	st <- mapM getmtime srcs
+	when (dt == Nothing || any (> dt) st) $
+		unlessM builder $
+			error $ "failed to make " ++ dest
+  where
+	getmtime = catchMaybeIO . getModificationTime
+
+cabal :: [String] -> IO Bool
+cabal = boolSystem "cabal" . map Param

src/Propellor/CmdLine.hs

@@ -12,6 +12,7 @@ import qualified Network.BSD
 import Propellor
 import Propellor.Gpg
 import Propellor.Git
+import Propellor.Bootstrap
 import Propellor.Spin
 import Propellor.Types.CmdLine
 import qualified Propellor.Property.Docker as Docker
@@ -31,6 +32,7 @@ usage h = hPutStrLn h $ unlines
 	, "  propellor --edit field context"
 	, "  propellor --list-fields"
 	, "  propellor --merge"
+	, "  propellor --build"
 	]
 
 usageError :: [String] -> IO a
@@ -128,19 +130,16 @@ unknownhost h hosts = errorMessage $ unlines
 	]
 
 buildFirst :: CmdLine -> IO () -> IO ()
-buildFirst cmdline next = ifM (doesFileExist "Makefile")
+buildFirst cmdline next = do
-	( do
 	oldtime <- getmtime
-		ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
+	buildPropellor
-			( do
 	newtime <- getmtime
 	if newtime == oldtime
 		then next
-					else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
+		else void $ boolSystem "./propellor"
-			, errorMessage "Propellor build failed!" 
+			[ Param "--continue"
-			)
+			, Param (show cmdline)
-	, next
+			]
-	)
   where
 	getmtime = catchMaybeIO $ getModificationTime "propellor"
 
@@ -155,10 +154,12 @@ updateFirst cmdline next = ifM hasOrigin (updateFirst' cmdline next, next)
 
 updateFirst' :: CmdLine -> IO () -> IO ()
 updateFirst' cmdline next = ifM fetchOrigin
-	( ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
+	( do
-		( void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
+		buildPropellor
-			, errorMessage "Propellor build failed!" 
+		void $ boolSystem "./propellor"
-		)
+			[ Param "--continue"
+			, Param (show cmdline)
+			]
 	, next
 	)
 

src/Propellor/Property/Cron.hs

@@ -30,7 +30,10 @@ data Times
 job :: Desc -> Times -> UserName -> FilePath -> String -> Property NoInfo
 job desc times user cddir command = combineProperties ("cronned " ++ desc)
 	[ cronjobfile `File.hasContent`
-		[ "# Generated by propellor"
+		[ case times of
+			Times _ -> ""
+			_ -> "#!/bin/sh\nset -e"
+		, "# Generated by propellor"
 		, ""
 		, "SHELL=/bin/sh"
 		, "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"

src/Propellor/Property/SiteSpecific/JoeySites.hs

@@ -125,17 +125,6 @@ mumbleServer hosts = combineProperties hn $ props
   where
 	hn = "mumble.debian.net"
 
-obnamLowMem :: Property NoInfo
-obnamLowMem = combineProperties "obnam tuned for low memory use"
-	[ Obnam.latestVersion
-	, "/etc/obnam.conf" `File.containsLines`
-		[ "[config]"
-		, "# Suggested by liw to keep Obnam memory consumption down (at some speed cost)."
-		, "upload-queue-size = 96"
-		, "lru-size = 96"
-		]
-	]
-
 -- git.kitenet.net and git.joeyh.name
 gitServer :: [Host] -> Property HasInfo
 gitServer hosts = propertyList "git.kitenet.net setup" $ props
@@ -282,7 +271,8 @@ gitAnnexDistributor = combineProperties "git-annex distributor, including rsync
 	& "/etc/default/rsync" `File.containsLine` "RSYNC_ENABLE=true"
 		`onChange` Service.running "rsync"
 	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild"
-	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks"
+	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite"
+	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/windows"
 	-- git-annex distribution signing key
 	& Gpg.keyImported (Gpg.GpgKeyId "89C809CB") "joey"
   where

src/Propellor/Property/Tor.hs

@@ -5,9 +5,11 @@ import qualified Propellor.Property.File as File
 import qualified Propellor.Property.Apt as Apt
 import qualified Propellor.Property.Service as Service
 import Utility.FileMode
+import Utility.DataUnits
 
 import System.Posix.Files
 import Data.Char
+import Data.List
 
 type HiddenServiceName = String
 
@@ -17,60 +19,36 @@ type NodeName = String
 --
 -- Uses port 443
 isBridge :: Property NoInfo
-isBridge = isBridge' []
+isBridge = configured
-
+	[ ("BridgeRelay", "1")
-isBridge' :: [String] -> Property NoInfo
+	, ("Exitpolicy", "reject *:*")
-isBridge' extraconfig = server config
+	, ("ORPort", "443")
+	]
 	`describe` "tor bridge"
-  where
+	`requires` server
-	config = 
-		[ "BridgeRelay 1"
-		, "Exitpolicy reject *:*"
-		, "ORPort 443"
-		] ++ extraconfig
 
 -- | Sets up a tor relay.
 --
 -- Uses port 443
 isRelay :: Property NoInfo
-isRelay = isRelay' []
+isRelay = configured
-
+	[ ("BridgeRelay", "0")
-isRelay' :: [String] -> Property NoInfo
+	, ("Exitpolicy", "reject *:*")
-isRelay' extraconfig = server config
+	, ("ORPort", "443")
+	]
 	`describe` "tor relay"
-  where
+	`requires` server
-	config = 
-		[ "BridgeRelay 0"
-		, "Exitpolicy reject *:*"
-		, "ORPort 443"
-		] ++ extraconfig
 
--- | Converts a property like isBridge' or isRelay' to be a named
+-- | Makes the tor node be named, with a known private key.
--- node, with a known private key.
 --
 -- This can be moved to a different IP without needing to wait to
 -- accumulate trust.
---
+named :: NodeName -> Property HasInfo
--- The base property can be used to start out and then upgraded to 
+named n = configured [("Nickname", n')]
--- a named property later.
+	`describe` ("tor node named " ++ n')
-named :: NodeName -> ([String] -> Property NoInfo) -> Property HasInfo
-named n basep = p `describe` (getDesc p ++ " " ++ n)
-  where
-	p = basep ["Nickname " ++ saneNickname n]
 	`requires` torPrivKey (Context ("tor " ++ n))
-
--- | A tor server (bridge, relay, or exit)
--- Don't use if you just want to run tor for personal use.
-server :: [String] -> Property NoInfo
-server extraconfig = setup
-	`requires` Apt.installed ["tor", "ntp"]
-	`describe` "tor server"
   where
-	setup = mainConfig `File.hasContent` config
+	n' = saneNickname n
-		`onChange` restarted
-	config = 
-		[ "SocksPort 0"
-		] ++ extraconfig
 
 torPrivKey :: Context -> Property HasInfo
 torPrivKey context = f `File.hasPrivContent` context
@@ -80,15 +58,58 @@ torPrivKey context = f `File.hasPrivContent` context
   where
 	f = "/var/lib/tor/keys/secret_id_key"
 
+-- | A tor server (bridge, relay, or exit)
+-- Don't use if you just want to run tor for personal use.
+server :: Property NoInfo
+server = configured [("SocksPort", "0")]
+	`requires` Apt.installed ["tor", "ntp"]
+	`describe` "tor server"
+
+-- | Specifies configuration settings. Any lines in the config file
+-- that set other values for the specified settings will be removed,
+-- while other settings are left as-is. Tor is restarted when
+-- configuration is changed.
+configured :: [(String, String)] -> Property NoInfo
+configured settings = File.fileProperty "tor configured" go mainConfig
+	`onChange` restarted
+  where
+	ks = map fst settings
+	go ls = sort $ map toconfig $
+		filter (\(k, _) -> k `notElem` ks) (map fromconfig ls)
+		++ settings
+	toconfig (k, v) = k ++ " " ++ v
+	fromconfig = separate (== ' ')
+
+data BwLimit
+	= PerSecond String
+	| PerDay String
+	| PerMonth String
+
+-- | Limit incoming and outgoing traffic to the specified
+-- amount each.
+--
+-- For example, PerSecond "30 kibibytes" is the minimum limit
+-- for a useful relay.
+bandwidthRate :: BwLimit -> Property NoInfo
+bandwidthRate (PerSecond s) = bandwidthRate' s 1
+bandwidthRate (PerDay s) = bandwidthRate' s (24*60*60)
+bandwidthRate (PerMonth s) = bandwidthRate' s (31*24*60*60)
+
+bandwidthRate' :: String -> Integer -> Property NoInfo
+bandwidthRate' s divby = case readSize dataUnits s of
+	Just sz -> let v = show (sz `div` divby) ++ " bytes"
+		in configured [("BandwidthRate", v)]
+			`describe` ("tor BandwidthRate " ++ v)
+	Nothing -> property ("unable to parse " ++ s) noChange
+
 hiddenServiceAvailable :: HiddenServiceName -> Int -> Property NoInfo
 hiddenServiceAvailable hn port = hiddenServiceHostName prop
   where
-	prop = mainConfig `File.containsLines`
+	prop = configured
-		[ unwords ["HiddenServiceDir", varLib </> hn]
+		[ ("HiddenServiceDir", varLib </> hn)
-		, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
+		, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
 		]
 		`describe` "hidden service available"
-		`onChange` Service.reloaded "tor"
 	hiddenServiceHostName p =  adjustPropertySatisfy p $ \satisfy -> do
 		r <- satisfy
 		h <- liftIO $ readFile (varLib </> hn </> "hostname")
@@ -96,12 +117,11 @@ hiddenServiceAvailable hn port = hiddenServiceHostName prop
 		return r
 
 hiddenService :: HiddenServiceName -> Int -> Property NoInfo
-hiddenService hn port = mainConfig `File.containsLines`
+hiddenService hn port = configured
-	[ unwords ["HiddenServiceDir", varLib </> hn]
+	[ ("HiddenServiceDir", varLib </> hn)
-	, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
+	, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
 	]
 	`describe` unwords ["hidden service available:", hn, show port]
-	`onChange` restarted
 
 hiddenServiceData :: IsContext c => HiddenServiceName -> c -> Property HasInfo
 hiddenServiceData hn context = combineProperties desc

src/Propellor/Shim.hs

@@ -33,6 +33,9 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
 	let linker = (dest ++) $ 
 		fromMaybe (error "cannot find ld-linux linker") $
 			headMaybe $ filter ("ld-linux" `isInfixOf`) libs'
+	let linkersym = takeDirectory linker </> takeFileName propellorbin
+	createSymbolicLink linkersym (takeFileName linker)
+
 	let gconvdir = (dest ++) $ takeDirectory $
 		fromMaybe (error "cannot find gconv directory") $
 			headMaybe $ filter ("/gconv/" `isInfixOf`) glibclibs
@@ -42,7 +45,7 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
 		[ shebang
 		, "GCONV_PATH=" ++ shellEscape gconvdir
 		, "export GCONV_PATH"
-		, "exec " ++ unwords (map shellEscape $ linker : linkerparams) ++ 
+		, "exec " ++ unwords (map shellEscape $ linkersym : linkerparams) ++ 
 			" " ++ shellEscape (fromMaybe propellorbin propellorbinpath) ++ " \"$@\""
 		]
 	modifyFileMode shim (addModes executeModes)

src/Propellor/Spin.hs

@@ -24,6 +24,7 @@ import Propellor.PrivData.Paths
 import Propellor.Git
 import Propellor.Ssh
 import Propellor.Gpg
+import Propellor.Bootstrap
 import Propellor.Types.CmdLine
 import qualified Propellor.Shim as Shim
 import Utility.FileMode
@@ -69,7 +70,7 @@ spin target relay hst = do
 	probecmd = intercalate " ; "
 		[ "if [ ! -d " ++ localdir ++ "/.git ]"
 		, "then (" ++ intercalate " && "
-			[ "if ! git --version || ! make --version; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git make; fi"
+			[ installGitCommand
 			, "echo " ++ toMarked statusMarker (show NeedGitClone)
 			] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled)
 		, "else " ++ updatecmd
@@ -78,7 +79,7 @@ spin target relay hst = do
 	
 	updatecmd = intercalate " && "
 		[ "cd " ++ localdir
-		, "if ! test -x ./propellor; then make deps build; fi"
+		, bootstrapPropellorCommand
 		, if viarelay
 			then "./propellor --continue " ++
 				shellEscape (show (Relay target))

src/wrapper.hs

@@ -15,6 +15,7 @@
 module Main where
 
 import Propellor.Message
+import Propellor.Bootstrap
 import Utility.UserInfo
 import Utility.Monad
 import Utility.Process
@@ -91,13 +92,10 @@ wrapper args propellordir propellorbin = do
 					warnoutofdate propellordir True
 	buildruncfg = do
 		changeWorkingDirectory propellordir
-		ifM (boolSystem "make" [Param "build"])
+		buildPropellor
-			( do
 		putStrLn ""
 		putStrLn ""
 		chain
-			, error "Propellor build failed."
-			)
 	chain = do
 		(_, _, _, pid) <- createProcess (proc propellorbin args) 
 		exitWith =<< waitForProcess pid