Merge branch 'joeyconfig'

This commit is contained in:
Joey Hess 2015-02-28 13:08:05 -04:00
commit 970ffbd0d6
12 changed files with 234 additions and 122 deletions

View File

@ -1,26 +1,12 @@
CABAL?=cabal
DEBDEPS=gnupg ghc cabal-install libghc-missingh-dev libghc-ansi-terminal-dev libghc-ifelse-dev libghc-unix-compat-dev libghc-hslogger-dev libghc-network-dev libghc-quickcheck2-dev libghc-mtl-dev libghc-monadcatchio-transformers-dev
# this target is provided to keep old versions of the propellor cron job
# working, and will eventually be removed
run: deps build
# this target is provided (and is first) to keep old versions of the
# propellor cron job working, and will eventually be removed
run: build
./propellor
dev: build tags
build: dist/setup-config
@if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi
@ln -sf dist/build/propellor-config/propellor-config propellor
deps:
@if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS) || (apt-get update && apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS)); fi || true
@if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install libghc-async-dev || (cabal update; cabal install async); fi || true
dist/setup-config: propellor.cabal
@if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
@$(CABAL) configure
install: propellor.1
install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor
install -s dist/build/propellor/propellor $(DESTDIR)/usr/bin/propellor
@ -56,3 +42,15 @@ hackage:
@cabal upload dist/*.tar.gz
.PHONY: tags
# The rules below are only used when bootstrapping new propellor
# installations and building packages; propellor contains equivilant
# haskell code that it uses to re-build itself.
build: dist/setup-config
@if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi
@ln -sf dist/build/propellor-config/propellor-config propellor
dist/setup-config: propellor.cabal
@if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
@$(CABAL) configure

View File

@ -43,7 +43,6 @@ hosts = -- (o) `
, clam
, orca
, kite
, diatom
, elephant
, beaver
] ++ monsters
@ -85,8 +84,9 @@ clam = standardSystem "clam.kitenet.net" Unstable "amd64"
& Ssh.randomHostKeys
& Apt.unattendedUpgrades
& Network.ipv6to4
& Tor.named "kite1" Tor.isRelay'
& Postfix.satellite
& Tor.isRelay
& Tor.named "kite1"
& Tor.bandwidthRate (Tor.PerMonth "400 GB")
& Docker.configured
& Docker.garbageCollected `period` Daily
@ -223,16 +223,6 @@ kite = standardSystemUnhardened "kite.kitenet.net" Testing "amd64"
& branchableSecondary
& Dns.secondaryFor ["animx"] hosts "animx.eu.org"
diatom :: Host
diatom = standardSystem "diatom.kitenet.net" (Stable "wheezy") "amd64"
[ "dying" ]
& ipv4 "107.170.31.195"
& Ssh.hostKeys hostContext
[ (SshDsa, "ssh-dss AAAAB3NzaC1kc3MAAACBAO9tnPUT4p+9z7K6/OYuiBNHaij4Nzv5YVBih1vMl+ALz0gYAj8RWJzXmqp5buFAyfgOoLw+H9s1bBS01Sy3i07Dm6cx1fWG4RXL/E/3w1tavX99GD2bBxDBu890ebA5Tp+eFRJkS9+JwSvFiF6CP7NbVjifCagoUO56Ig048RwDAAAAFQDPY2xM3q6KwsVQliel23nrd0rV2QAAAIEAga3hj1hL00rYPNnAUzT8GAaSP62S4W68lusErH+KPbsMwFBFY/Ib1FVf8k6Zn6dZLh/HH/RtJi0JwdzPI1IFW+lwVbKfwBvhQ1lw9cH2rs1UIVgi7Wxdgfy8gEWxf+QIqn62wG+Ulf/HkWGvTrRpoJqlYRNS/gnOWj9Z/4s99koAAACBAM/uJIo2I0nK15wXiTYs/NYUZA7wcErugFn70TRbSgduIFH6U/CQa3rgHJw9DCPCQJLq7pwCnFH7too/qaK+czDk04PsgqV0+Jc7957gU5miPg50d60eJMctHV4eQ1FpwmGGfXxRBR9k2ZvikWYatYir3L6/x1ir7M0bA9IzNU45")
, (SshRsa, "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA2QAJEuvbTmaN9ex9i9bjPhMGj+PHUYq2keIiaIImJ+8mo+yKSaGUxebG4tpuDPx6KZjdycyJt74IXfn1voGUrfzwaEY9NkqOP3v6OWTC3QeUGqDCeJ2ipslbEd9Ep9XBp+/ldDQm60D0XsIZdmDeN6MrHSbKF4fXv1bqpUoUILk=")
]
& alias "ns2.kitenet.net"
elephant :: Host
elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
[ "Storage, big data, and backups, omnomnom!"
@ -265,7 +255,6 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
& alias "znc.kitenet.net"
& JoeySites.ircBouncer
-- I'd rather this were on diatom, but it needs unstable.
& alias "kgb.kitenet.net"
& JoeySites.kgbServer

11
debian/changelog vendored
View File

@ -1,3 +1,14 @@
propellor (2.2.0) UNRELEASED; urgency=medium
* When running shimmed (eg in a docker container),
improve process name visible in ps.
* Add shebang to cron.daily etc files.
* Some changes to tor configuration, minor API change.
* Propellor now builds itself, and gets its build dependencies installed
when deploying to a new host, without needing the Makefile.
-- Joey Hess <id@joeyh.name> Mon, 16 Feb 2015 19:00:48 -0400
propellor (2.1.0) unstable; urgency=medium
* Additional tor properties, including support for making relays,

View File

@ -1,5 +1,5 @@
Name: propellor
Version: 2.1.0
Version: 2.2.0
Cabal-Version: >= 1.6
License: BSD3
Maintainer: Joey Hess <id@joeyh.name>
@ -128,6 +128,7 @@ Library
Propellor.Types.Result
Propellor.Types.CmdLine
Other-Modules:
Propellor.Bootstrap
Propellor.Git
Propellor.Gpg
Propellor.Spin

View File

@ -0,0 +1,97 @@
module Propellor.Bootstrap (
bootstrapPropellorCommand,
installGitCommand,
buildPropellor,
) where
import Propellor
import Utility.SafeCommand
import System.Posix.Files
import Data.List
type ShellCommand = String
-- Shell command line to build propellor, used when bootstrapping on a new
-- host. Should be run inside the propellor source tree, and will install
-- all necessary build dependencies.
bootstrapPropellorCommand :: ShellCommand
bootstrapPropellorCommand = "if ! test -x ./propellor; then " ++ go ++ "; fi"
where
go = intercalate " && "
[ depsCommand
, buildCommand
]
buildCommand :: ShellCommand
buildCommand = intercalate " && "
[ "cabal configure"
, "cabal build"
, "ln -sf dist/build/propellor-config/propellor-config propellor"
]
depsCommand :: ShellCommand
depsCommand =
"(" ++ aptinstall debdeps ++ " || (apt-get update && " ++ aptinstall debdeps ++ ")) && "
++ "(" ++ aptinstall ["libghc-async-dev"] ++ " || (" ++ cabalinstall ["async"] ++ ")) || "
++ "(" ++ cabalinstall ["--only-dependencies"] ++ ")"
where
aptinstall ps = "apt-get --no-upgrade --no-install-recommends -y install " ++ unwords ps
cabalinstall ps = "cabal update; cabal install " ++ unwords ps
-- This is the same build deps listed in debian/control.
debdeps =
[ "gnupg"
, "ghc"
, "cabal-install"
-- async is not available in debian stable
-- , "libghc-async-dev"
, "libghc-missingh-dev"
, "libghc-hslogger-dev"
, "libghc-unix-compat-dev"
, "libghc-ansi-terminal-dev"
, "libghc-ifelse-dev"
, "libghc-network-dev"
, "libghc-quickcheck2-dev"
, "libghc-mtl-dev"
, "libghc-monadcatchio-transformers-dev"
]
installGitCommand :: ShellCommand
installGitCommand = "if ! git --version >/dev/null; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git; fi"
buildPropellor :: IO ()
buildPropellor = unlessM (actionMessage "Propellor build" build) $
errorMessage "Propellor build failed!"
-- Build propellor using cabal, and symlink propellor to where cabal
-- leaves the built binary.
--
-- For speed, only runs cabal configure when it's not been run before.
-- If the build fails cabal may need to have configure re-run.
build :: IO Bool
build = catchBoolIO $ do
make "dist/setup-config" ["propellor.cabal"] $
cabal ["configure"]
unlessM (cabal ["build"]) $ do
void $ cabal ["configure"]
unlessM (cabal ["build"]) $
error "cabal build failed"
nukeFile "propellor"
createSymbolicLink "dist/build/propellor-config/propellor-config" "propellor"
return True
make :: FilePath -> [FilePath] -> IO Bool -> IO ()
make dest srcs builder = do
dt <- getmtime dest
st <- mapM getmtime srcs
when (dt == Nothing || any (> dt) st) $
unlessM builder $
error $ "failed to make " ++ dest
where
getmtime = catchMaybeIO . getModificationTime
cabal :: [String] -> IO Bool
cabal = boolSystem "cabal" . map Param

View File

@ -12,6 +12,7 @@ import qualified Network.BSD
import Propellor
import Propellor.Gpg
import Propellor.Git
import Propellor.Bootstrap
import Propellor.Spin
import Propellor.Types.CmdLine
import qualified Propellor.Property.Docker as Docker
@ -31,6 +32,7 @@ usage h = hPutStrLn h $ unlines
, " propellor --edit field context"
, " propellor --list-fields"
, " propellor --merge"
, " propellor --build"
]
usageError :: [String] -> IO a
@ -128,19 +130,16 @@ unknownhost h hosts = errorMessage $ unlines
]
buildFirst :: CmdLine -> IO () -> IO ()
buildFirst cmdline next = ifM (doesFileExist "Makefile")
( do
oldtime <- getmtime
ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
( do
newtime <- getmtime
if newtime == oldtime
then next
else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
, errorMessage "Propellor build failed!"
)
, next
)
buildFirst cmdline next = do
oldtime <- getmtime
buildPropellor
newtime <- getmtime
if newtime == oldtime
then next
else void $ boolSystem "./propellor"
[ Param "--continue"
, Param (show cmdline)
]
where
getmtime = catchMaybeIO $ getModificationTime "propellor"
@ -155,10 +154,12 @@ updateFirst cmdline next = ifM hasOrigin (updateFirst' cmdline next, next)
updateFirst' :: CmdLine -> IO () -> IO ()
updateFirst' cmdline next = ifM fetchOrigin
( ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
( void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
, errorMessage "Propellor build failed!"
)
( do
buildPropellor
void $ boolSystem "./propellor"
[ Param "--continue"
, Param (show cmdline)
]
, next
)

View File

@ -30,7 +30,10 @@ data Times
job :: Desc -> Times -> UserName -> FilePath -> String -> Property NoInfo
job desc times user cddir command = combineProperties ("cronned " ++ desc)
[ cronjobfile `File.hasContent`
[ "# Generated by propellor"
[ case times of
Times _ -> ""
_ -> "#!/bin/sh\nset -e"
, "# Generated by propellor"
, ""
, "SHELL=/bin/sh"
, "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"

View File

@ -125,17 +125,6 @@ mumbleServer hosts = combineProperties hn $ props
where
hn = "mumble.debian.net"
obnamLowMem :: Property NoInfo
obnamLowMem = combineProperties "obnam tuned for low memory use"
[ Obnam.latestVersion
, "/etc/obnam.conf" `File.containsLines`
[ "[config]"
, "# Suggested by liw to keep Obnam memory consumption down (at some speed cost)."
, "upload-queue-size = 96"
, "lru-size = 96"
]
]
-- git.kitenet.net and git.joeyh.name
gitServer :: [Host] -> Property HasInfo
gitServer hosts = propertyList "git.kitenet.net setup" $ props
@ -282,7 +271,8 @@ gitAnnexDistributor = combineProperties "git-annex distributor, including rsync
& "/etc/default/rsync" `File.containsLine` "RSYNC_ENABLE=true"
`onChange` Service.running "rsync"
& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild"
& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks"
& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite"
& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/windows"
-- git-annex distribution signing key
& Gpg.keyImported (Gpg.GpgKeyId "89C809CB") "joey"
where

View File

@ -5,9 +5,11 @@ import qualified Propellor.Property.File as File
import qualified Propellor.Property.Apt as Apt
import qualified Propellor.Property.Service as Service
import Utility.FileMode
import Utility.DataUnits
import System.Posix.Files
import Data.Char
import Data.List
type HiddenServiceName = String
@ -17,60 +19,36 @@ type NodeName = String
--
-- Uses port 443
isBridge :: Property NoInfo
isBridge = isBridge' []
isBridge' :: [String] -> Property NoInfo
isBridge' extraconfig = server config
isBridge = configured
[ ("BridgeRelay", "1")
, ("Exitpolicy", "reject *:*")
, ("ORPort", "443")
]
`describe` "tor bridge"
where
config =
[ "BridgeRelay 1"
, "Exitpolicy reject *:*"
, "ORPort 443"
] ++ extraconfig
`requires` server
-- | Sets up a tor relay.
--
-- Uses port 443
isRelay :: Property NoInfo
isRelay = isRelay' []
isRelay' :: [String] -> Property NoInfo
isRelay' extraconfig = server config
isRelay = configured
[ ("BridgeRelay", "0")
, ("Exitpolicy", "reject *:*")
, ("ORPort", "443")
]
`describe` "tor relay"
where
config =
[ "BridgeRelay 0"
, "Exitpolicy reject *:*"
, "ORPort 443"
] ++ extraconfig
`requires` server
-- | Converts a property like isBridge' or isRelay' to be a named
-- node, with a known private key.
-- | Makes the tor node be named, with a known private key.
--
-- This can be moved to a different IP without needing to wait to
-- accumulate trust.
--
-- The base property can be used to start out and then upgraded to
-- a named property later.
named :: NodeName -> ([String] -> Property NoInfo) -> Property HasInfo
named n basep = p `describe` (getDesc p ++ " " ++ n)
named :: NodeName -> Property HasInfo
named n = configured [("Nickname", n')]
`describe` ("tor node named " ++ n')
`requires` torPrivKey (Context ("tor " ++ n))
where
p = basep ["Nickname " ++ saneNickname n]
`requires` torPrivKey (Context ("tor " ++ n))
-- | A tor server (bridge, relay, or exit)
-- Don't use if you just want to run tor for personal use.
server :: [String] -> Property NoInfo
server extraconfig = setup
`requires` Apt.installed ["tor", "ntp"]
`describe` "tor server"
where
setup = mainConfig `File.hasContent` config
`onChange` restarted
config =
[ "SocksPort 0"
] ++ extraconfig
n' = saneNickname n
torPrivKey :: Context -> Property HasInfo
torPrivKey context = f `File.hasPrivContent` context
@ -80,15 +58,58 @@ torPrivKey context = f `File.hasPrivContent` context
where
f = "/var/lib/tor/keys/secret_id_key"
-- | A tor server (bridge, relay, or exit)
-- Don't use if you just want to run tor for personal use.
server :: Property NoInfo
server = configured [("SocksPort", "0")]
`requires` Apt.installed ["tor", "ntp"]
`describe` "tor server"
-- | Specifies configuration settings. Any lines in the config file
-- that set other values for the specified settings will be removed,
-- while other settings are left as-is. Tor is restarted when
-- configuration is changed.
configured :: [(String, String)] -> Property NoInfo
configured settings = File.fileProperty "tor configured" go mainConfig
`onChange` restarted
where
ks = map fst settings
go ls = sort $ map toconfig $
filter (\(k, _) -> k `notElem` ks) (map fromconfig ls)
++ settings
toconfig (k, v) = k ++ " " ++ v
fromconfig = separate (== ' ')
data BwLimit
= PerSecond String
| PerDay String
| PerMonth String
-- | Limit incoming and outgoing traffic to the specified
-- amount each.
--
-- For example, PerSecond "30 kibibytes" is the minimum limit
-- for a useful relay.
bandwidthRate :: BwLimit -> Property NoInfo
bandwidthRate (PerSecond s) = bandwidthRate' s 1
bandwidthRate (PerDay s) = bandwidthRate' s (24*60*60)
bandwidthRate (PerMonth s) = bandwidthRate' s (31*24*60*60)
bandwidthRate' :: String -> Integer -> Property NoInfo
bandwidthRate' s divby = case readSize dataUnits s of
Just sz -> let v = show (sz `div` divby) ++ " bytes"
in configured [("BandwidthRate", v)]
`describe` ("tor BandwidthRate " ++ v)
Nothing -> property ("unable to parse " ++ s) noChange
hiddenServiceAvailable :: HiddenServiceName -> Int -> Property NoInfo
hiddenServiceAvailable hn port = hiddenServiceHostName prop
where
prop = mainConfig `File.containsLines`
[ unwords ["HiddenServiceDir", varLib </> hn]
, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
prop = configured
[ ("HiddenServiceDir", varLib </> hn)
, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
]
`describe` "hidden service available"
`onChange` Service.reloaded "tor"
hiddenServiceHostName p = adjustPropertySatisfy p $ \satisfy -> do
r <- satisfy
h <- liftIO $ readFile (varLib </> hn </> "hostname")
@ -96,12 +117,11 @@ hiddenServiceAvailable hn port = hiddenServiceHostName prop
return r
hiddenService :: HiddenServiceName -> Int -> Property NoInfo
hiddenService hn port = mainConfig `File.containsLines`
[ unwords ["HiddenServiceDir", varLib </> hn]
, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
hiddenService hn port = configured
[ ("HiddenServiceDir", varLib </> hn)
, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
]
`describe` unwords ["hidden service available:", hn, show port]
`onChange` restarted
hiddenServiceData :: IsContext c => HiddenServiceName -> c -> Property HasInfo
hiddenServiceData hn context = combineProperties desc

View File

@ -33,6 +33,9 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
let linker = (dest ++) $
fromMaybe (error "cannot find ld-linux linker") $
headMaybe $ filter ("ld-linux" `isInfixOf`) libs'
let linkersym = takeDirectory linker </> takeFileName propellorbin
createSymbolicLink linkersym (takeFileName linker)
let gconvdir = (dest ++) $ takeDirectory $
fromMaybe (error "cannot find gconv directory") $
headMaybe $ filter ("/gconv/" `isInfixOf`) glibclibs
@ -42,7 +45,7 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
[ shebang
, "GCONV_PATH=" ++ shellEscape gconvdir
, "export GCONV_PATH"
, "exec " ++ unwords (map shellEscape $ linker : linkerparams) ++
, "exec " ++ unwords (map shellEscape $ linkersym : linkerparams) ++
" " ++ shellEscape (fromMaybe propellorbin propellorbinpath) ++ " \"$@\""
]
modifyFileMode shim (addModes executeModes)

View File

@ -24,6 +24,7 @@ import Propellor.PrivData.Paths
import Propellor.Git
import Propellor.Ssh
import Propellor.Gpg
import Propellor.Bootstrap
import Propellor.Types.CmdLine
import qualified Propellor.Shim as Shim
import Utility.FileMode
@ -69,7 +70,7 @@ spin target relay hst = do
probecmd = intercalate " ; "
[ "if [ ! -d " ++ localdir ++ "/.git ]"
, "then (" ++ intercalate " && "
[ "if ! git --version || ! make --version; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git make; fi"
[ installGitCommand
, "echo " ++ toMarked statusMarker (show NeedGitClone)
] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled)
, "else " ++ updatecmd
@ -78,7 +79,7 @@ spin target relay hst = do
updatecmd = intercalate " && "
[ "cd " ++ localdir
, "if ! test -x ./propellor; then make deps build; fi"
, bootstrapPropellorCommand
, if viarelay
then "./propellor --continue " ++
shellEscape (show (Relay target))

View File

@ -15,6 +15,7 @@
module Main where
import Propellor.Message
import Propellor.Bootstrap
import Utility.UserInfo
import Utility.Monad
import Utility.Process
@ -91,13 +92,10 @@ wrapper args propellordir propellorbin = do
warnoutofdate propellordir True
buildruncfg = do
changeWorkingDirectory propellordir
ifM (boolSystem "make" [Param "build"])
( do
putStrLn ""
putStrLn ""
chain
, error "Propellor build failed."
)
buildPropellor
putStrLn ""
putStrLn ""
chain
chain = do
(_, _, _, pid) <- createProcess (proc propellorbin args)
exitWith =<< waitForProcess pid