From 67d989c7c5ba8d18340eeddafbbdadc8b3ae91ab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 01:18:51 -0400 Subject: [PATCH 01/12] propellor spin --- config-joey.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-joey.hs b/config-joey.hs index d6f174d..c5309ad 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -80,7 +80,7 @@ clam = standardSystem "clam.kitenet.net" Unstable "amd64" ! Ssh.listenPort 80 ! Ssh.listenPort 443 - ! Debootstrap.built "/tmp/chroot" (System (Debian Unstable) "amd64") [] + & Debootstrap.built "/tmp/chroot" (System (Debian Unstable) "amd64") [] orca :: Host orca = standardSystem "orca.kitenet.net" Unstable "amd64" From cc8bbcf95b9b0cdcd2db17ee6049263498caa79b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 12:50:01 -0400 Subject: [PATCH 02/12] propellor spin From b8b746a7f1bdbf179136959a85138fde60c43588 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 14:06:55 -0400 Subject: [PATCH 03/12] starting work on a Chroot module factored out info up-propigation code rom Docker --- config-joey.hs | 8 +++- propellor.cabal | 2 + src/Propellor.hs | 2 + src/Propellor/Host.hs | 64 ++++++++++++++++++++++++++++++++ src/Propellor/Property.hs | 34 +---------------- src/Propellor/Property/Chroot.hs | 57 ++++++++++++++++++++++++++++ src/Propellor/Property/Docker.hs | 23 +++++------- src/Propellor/Types.hs | 21 ++++++++++- 8 files changed, 162 insertions(+), 49 deletions(-) create mode 100644 src/Propellor/Host.hs create mode 100644 src/Propellor/Property/Chroot.hs diff --git a/config-joey.hs b/config-joey.hs index c5309ad..a11e1d8 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -24,7 +24,7 @@ import qualified Propellor.Property.Postfix as Postfix import qualified Propellor.Property.Grub as Grub import qualified Propellor.Property.Obnam as Obnam import qualified Propellor.Property.Gpg as Gpg -import qualified Propellor.Property.Debootstrap as Debootstrap +import qualified Propellor.Property.Chroot as Chroot import qualified Propellor.Property.HostingProvider.DigitalOcean as DigitalOcean import qualified Propellor.Property.HostingProvider.CloudAtCost as CloudAtCost import qualified Propellor.Property.HostingProvider.Linode as Linode @@ -80,8 +80,12 @@ clam = standardSystem "clam.kitenet.net" Unstable "amd64" ! Ssh.listenPort 80 ! Ssh.listenPort 443 - & Debootstrap.built "/tmp/chroot" (System (Debian Unstable) "amd64") [] + & Chroot.provisioned testChroot +testChroot :: Chroot.Chroot +testChroot = Chroot.chroot "/tmp/chroot" (System (Debian Unstable) "amd64") + & File.hasContent "/foo" ["hello"] + orca :: Host orca = standardSystem "orca.kitenet.net" Unstable "amd64" [ "Main git-annex build box." ] diff --git a/propellor.cabal b/propellor.cabal index 38e3da2..7f2b237 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -74,6 +74,7 @@ Library Propellor.Property.Apt Propellor.Property.Cmd Propellor.Property.Hostname + Propellor.Property.Chroot Propellor.Property.Cron Propellor.Property.Debootstrap Propellor.Property.Dns @@ -102,6 +103,7 @@ Library Propellor.Property.SiteSpecific.GitHome Propellor.Property.SiteSpecific.JoeySites Propellor.Property.SiteSpecific.GitAnnexBuilder + Propellor.Host Propellor.CmdLine Propellor.Info Propellor.Message diff --git a/src/Propellor.hs b/src/Propellor.hs index c0ef14f..6e31e27 100644 --- a/src/Propellor.hs +++ b/src/Propellor.hs @@ -33,6 +33,7 @@ module Propellor ( module Propellor.Types , module Propellor.Property , module Propellor.Property.Cmd + , module Propellor.Host , module Propellor.Info , module Propellor.PrivData , module Propellor.Engine @@ -51,6 +52,7 @@ import Propellor.PrivData import Propellor.Message import Propellor.Exception import Propellor.Info +import Propellor.Host import Utility.PartialPrelude as X import Utility.Process as X diff --git a/src/Propellor/Host.hs b/src/Propellor/Host.hs new file mode 100644 index 0000000..14d56e2 --- /dev/null +++ b/src/Propellor/Host.hs @@ -0,0 +1,64 @@ +{-# LANGUAGE PackageImports #-} + +module Propellor.Host where + +import Data.Monoid +import qualified Data.Set as S + +import Propellor.Types +import Propellor.Info +import Propellor.Property +import Propellor.PrivData + +-- | Starts accumulating the properties of a Host. +-- +-- > host "example.com" +-- > & someproperty +-- > ! oldproperty +-- > & otherproperty +host :: HostName -> Host +host hn = Host hn [] mempty + +-- | Something that can accumulate properties. +class Hostlike h where + -- | Adds a property. + -- + -- Can add Properties and RevertableProperties + (&) :: IsProp p => h -> p -> h + + -- | Like (&), but adds the property as the + -- first property of the host. Normally, property + -- order should not matter, but this is useful + -- when it does. + (&^) :: IsProp p => h -> p -> h + + getHost :: h -> Host + +instance Hostlike Host where + (Host hn ps is) & p = Host hn (ps ++ [toProp p]) (is <> getInfo p) + (Host hn ps is) &^ p = Host hn ([toProp p] ++ ps) (getInfo p <> is) + getHost h = h + +-- | Adds a property in reverted form. +(!) :: Hostlike h => h -> RevertableProperty -> h +h ! p = h & revert p + +infixl 1 &^ +infixl 1 & +infixl 1 ! + +-- | When eg, docking a container, some of the Info about the container +-- should propigate out to the Host it's on. This includes DNS info, +-- so that eg, aliases of the container are reflected in the dns for the +-- host where it runs. +-- +-- This adjusts the Property that docks a container, to include such info +-- from the container. +propigateInfo :: Hostlike hl => hl -> Property -> (Info -> Info) -> Property +propigateInfo hl p f = combineProperties (propertyDesc p) $ + p' : dnsprops ++ privprops + where + p' = p { propertyInfo = f (propertyInfo p) } + i = hostInfo (getHost hl) + dnsprops = map addDNS (S.toList $ _dns i) + privprops = map addPrivDataField (S.toList $ _privDataFields i) diff --git a/src/Propellor/Property.hs b/src/Propellor/Property.hs index bf69ff6..1d750a7 100644 --- a/src/Propellor/Property.hs +++ b/src/Propellor/Property.hs @@ -3,6 +3,7 @@ module Propellor.Property where import System.Directory +import System.FilePath import Control.Monad import Data.Monoid import Control.Monad.IfElse @@ -12,7 +13,6 @@ import Propellor.Types import Propellor.Info import Propellor.Engine import Utility.Monad -import System.FilePath -- Constructs a Property. property :: Desc -> Propellor Result -> Property @@ -135,38 +135,6 @@ revert (RevertableProperty p1 p2) = RevertableProperty p2 p1 unrevertable :: RevertableProperty -> Property unrevertable (RevertableProperty p1 _p2) = p1 --- | Starts accumulating the properties of a Host. --- --- > host "example.com" --- > & someproperty --- > ! oldproperty --- > & otherproperty -host :: HostName -> Host -host hn = Host hn [] mempty - -class Hostlike h where - -- | Adds a property to a Host - -- - -- Can add Properties and RevertableProperties - (&) :: IsProp p => h -> p -> h - -- | Like (&), but adds the property as the - -- first property of the host. Normally, property - -- order should not matter, but this is useful - -- when it does. - (&^) :: IsProp p => h -> p -> h - -instance Hostlike Host where - (Host hn ps is) & p = Host hn (ps ++ [toProp p]) (is <> getInfo p) - (Host hn ps is) &^ p = Host hn ([toProp p] ++ ps) (getInfo p <> is) - --- | Adds a property to the Host in reverted form. -(!) :: Hostlike h => h -> RevertableProperty -> h -h ! p = h & revert p - -infixl 1 &^ -infixl 1 & -infixl 1 ! - -- Changes the action that is performed to satisfy a property. adjustProperty :: Property -> (Propellor Result -> Propellor Result) -> Property adjustProperty p f = p { propertySatisfy = f (propertySatisfy p) } diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs new file mode 100644 index 0000000..e504693 --- /dev/null +++ b/src/Propellor/Property/Chroot.hs @@ -0,0 +1,57 @@ +module Propellor.Property.Chroot ( + Chroot, + chroot, + provisioned, +) where + +import Propellor +import qualified Propellor.Property.Debootstrap as Debootstrap + +import qualified Data.Map as M + +data Chroot = Chroot FilePath System Host + +instance Hostlike Chroot where + (Chroot l s h) & p = Chroot l s (h & p) + (Chroot l s h) &^ p = Chroot l s (h &^ p) + getHost (Chroot _ _ h) = h + +-- | Defines a Chroot at the given location, containing the specified +-- System. Properties can be added to configure the Chroot. +-- +-- > chroot "/srv/chroot/ghc-dev" (System (Debian Unstable) "amd64" +-- > & Apt.installed ["build-essential", "ghc", "haskell-platform"] +-- > & ... +chroot :: FilePath -> System -> Chroot +chroot location system = Chroot location system (Host location [] mempty) + +-- | Ensures that the chroot exists and is provisioned according to its +-- properties. +-- +-- Reverting this property removes the chroot. Note that it does not ensure +-- that any processes that might be running inside the chroot are stopped. +provisioned :: Chroot -> RevertableProperty +provisioned c@(Chroot loc system _) = RevertableProperty + (propigateChrootInfo c (go "exists" setup)) + (go "removed" teardown) + where + go desc a = property ("chroot " ++ loc ++ " " ++ desc) $ do + ensureProperties [a] + + setup = provisionChroot c `requires` built + + built = case system of + (System (Debian _) _) -> debootstrap + (System (Ubuntu _) _) -> debootstrap + + debootstrap = unrevertable (Debootstrap.built loc system []) + + teardown = undefined + +propigateChrootInfo :: Chroot -> Property -> Property +propigateChrootInfo c@(Chroot loc _ h) p = propigateInfo c p (<> chrootinfo) + where + chrootinfo = mempty $ mempty { _chroots = M.singleton loc h } + +provisionChroot :: Chroot -> Property +provisionChroot = undefined diff --git a/src/Propellor/Property/Docker.hs b/src/Propellor/Property/Docker.hs index 676d323..92cc124 100644 --- a/src/Propellor/Property/Docker.hs +++ b/src/Propellor/Property/Docker.hs @@ -52,7 +52,6 @@ import System.Posix.Process import Prelude hiding (init) import Data.List hiding (init) import Data.List.Utils -import qualified Data.Set as S import qualified Data.Map as M installed :: Property @@ -78,8 +77,10 @@ data Container = Container Image Host instance Hostlike Container where (Container i h) & p = Container i (h & p) (Container i h) &^ p = Container i (h &^ p) + getHost (Container _ h) = h --- | Builds a Container with a given name, image, and properties. +-- | Defines a Container with a given name, image, and properties. +-- Properties can be added to configure the Container. -- -- > container "web-server" "debian" -- > & publish "80:80" @@ -100,11 +101,9 @@ container cn image = Container image (Host cn [] info) -- -- Reverting this property ensures that the container is stopped and -- removed. -docked - :: Container - -> RevertableProperty +docked :: Container -> RevertableProperty docked ctr@(Container _ h) = RevertableProperty - (propigateInfo ctr (go "docked" setup)) + (propigateContainerInfo ctr (go "docked" setup)) (go "undocked" teardown) where cn = hostName h @@ -131,14 +130,12 @@ docked ctr@(Container _ h) = RevertableProperty ] ] -propigateInfo :: Container -> Property -> Property -propigateInfo (Container _ h@(Host hn _ containerinfo)) p = - combineProperties (propertyDesc p) $ p' : dnsprops ++ privprops +propigateContainerInfo :: Container -> Property -> Property +propigateContainerInfo ctr@(Container _ h) p = + propigateInfo ctr p (<> dockerinfo) where - p' = p { propertyInfo = propertyInfo p <> dockerinfo } - dockerinfo = dockerInfo $ mempty { _dockerContainers = M.singleton hn h } - dnsprops = map addDNS (S.toList $ _dns containerinfo) - privprops = map addPrivDataField (S.toList $ _privDataFields containerinfo) + dockerinfo = dockerInfo $ + mempty { _dockerContainers = M.singleton (hostName h) h } mkContainerInfo :: ContainerId -> Container -> ContainerInfo mkContainerInfo cid@(ContainerId hn _cn) (Container img h) = diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index 90c08e6..4e0a8de 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -25,6 +25,7 @@ module Propellor.Types , fromVal , DockerInfo(..) , DockerRunParam(..) + , ChrootInfo(..) , module Propellor.Types.OS , module Propellor.Types.Dns ) where @@ -166,11 +167,12 @@ data Info = Info , _dns :: S.Set Dns.Record , _namedconf :: Dns.NamedConfMap , _dockerinfo :: DockerInfo + , _chrootinfo :: ChrootInfo } deriving (Eq, Show) instance Monoid Info where - mempty = Info mempty mempty mempty mempty mempty mempty mempty + mempty = Info mempty mempty mempty mempty mempty mempty mempty mempty mappend old new = Info { _os = _os old <> _os new , _privDataFields = _privDataFields old <> _privDataFields new @@ -179,6 +181,7 @@ instance Monoid Info where , _dns = _dns old <> _dns new , _namedconf = _namedconf old <> _namedconf new , _dockerinfo = _dockerinfo old <> _dockerinfo new + , _chrootinfo = _chrootinfo old <> _chrootinfo new } data Val a = Val a | NoVal @@ -217,3 +220,19 @@ newtype DockerRunParam = DockerRunParam (HostName -> String) instance Show DockerRunParam where show (DockerRunParam a) = a "" + +data ChrootInfo = ChrootInfo + { _chroots :: M.Map FilePath Host + } + deriving (Show) + +instance Monoid ChrootInfo where + mempty = ChrootInfo mempty + mappend old new = ChrootInfo + { _chroots = M.union (_chroots old) (_chroots new) + } + +instance Eq ChrootInfo where + x == y = and + [ M.keys (_chroots x) == M.keys (_chroots y) + ] From 411612f8cbb128dee0b46285faa5ed78986bce8d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 14:15:34 -0400 Subject: [PATCH 04/12] propellor spin From def44311327640b2bdb7b5f1c4f9cc4dca761327 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 14:16:26 -0400 Subject: [PATCH 05/12] remove unused Eq instance for Info Its implementation was .. shady at best, as Host has no Eq --- src/Propellor/Types.hs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index 4e0a8de..16ddcc7 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -169,7 +169,7 @@ data Info = Info , _dockerinfo :: DockerInfo , _chrootinfo :: ChrootInfo } - deriving (Eq, Show) + deriving (Show) instance Monoid Info where mempty = Info mempty mempty mempty mempty mempty mempty mempty mempty @@ -210,12 +210,6 @@ instance Monoid DockerInfo where , _dockerContainers = M.union (_dockerContainers old) (_dockerContainers new) } -instance Eq DockerInfo where - x == y = and - [ let simpl v = map (\(DockerRunParam a) -> a "") (_dockerRunParams v) - in simpl x == simpl y - ] - newtype DockerRunParam = DockerRunParam (HostName -> String) instance Show DockerRunParam where @@ -231,8 +225,3 @@ instance Monoid ChrootInfo where mappend old new = ChrootInfo { _chroots = M.union (_chroots old) (_chroots new) } - -instance Eq ChrootInfo where - x == y = and - [ M.keys (_chroots x) == M.keys (_chroots y) - ] From 4d155864fadb5571d788ed645c842ad853f55d71 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 14:16:53 -0400 Subject: [PATCH 06/12] propellor spin From a4f04fcb02d76d9903c5bbc65827565bad6c2d8c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 15:15:28 -0400 Subject: [PATCH 07/12] propellor spin --- propellor.cabal | 2 +- src/Propellor/CmdLine.hs | 6 +- src/Propellor/Engine.hs | 18 +++++ src/Propellor/Property/Chroot.hs | 76 ++++++++++++++++++++- src/Propellor/Property/Docker.hs | 14 +--- src/Propellor/{Property/Docker => }/Shim.hs | 7 +- src/Propellor/Types.hs | 1 + 7 files changed, 103 insertions(+), 21 deletions(-) rename src/Propellor/{Property/Docker => }/Shim.hs (89%) diff --git a/propellor.cabal b/propellor.cabal index 7f2b237..23708fd 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -121,7 +121,7 @@ Library Propellor.Ssh Propellor.PrivData.Paths Propellor.Protocol - Propellor.Property.Docker.Shim + Propellor.Shim Utility.Applicative Utility.Data Utility.Directory diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 061c970..466b60f 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -15,7 +15,8 @@ import Propellor.Git import Propellor.Ssh import Propellor.Server import qualified Propellor.Property.Docker as Docker -import qualified Propellor.Property.Docker.Shim as DockerShim +import qualified Propellor.Property.Chroot as Chroot +import qualified Propellor.Shim as Shim import Utility.SafeCommand usage :: Handle -> IO () @@ -72,7 +73,7 @@ processCmdLine = go =<< getArgs -- | Runs propellor on hosts, as controlled by command-line options. defaultMain :: [Host] -> IO () defaultMain hostlist = do - DockerShim.cleanEnv + Shim.cleanEnv checkDebugMode cmdline <- processCmdLine debug ["command line: ", show cmdline] @@ -85,6 +86,7 @@ defaultMain hostlist = do go _ ListFields = listPrivDataFields hostlist go _ (AddKey keyid) = addKey keyid go _ (DockerChain hn cid) = Docker.chain hostlist hn cid + go _ (ChrootChain hn loc) = Chroot.chain hostlist hn loc go _ (DockerInit hn) = Docker.init hn go _ (GitPush fin fout) = gitPushHelper fin fout go _ (Update _) = forceConsole >> fetchFirst (onlyprocess update) diff --git a/src/Propellor/Engine.hs b/src/Propellor/Engine.hs index 3fa9ffc..0cbf624 100644 --- a/src/Propellor/Engine.hs +++ b/src/Propellor/Engine.hs @@ -11,12 +11,15 @@ import "mtl" Control.Monad.Reader import Control.Exception (bracket) import System.PosixCompat import System.Posix.IO +import Data.Maybe import Propellor.Types import Propellor.Message import Propellor.Exception import Propellor.Info import Utility.Exception +import Utility.PartialPrelude +import Utility.Monad runPropellor :: Host -> Propellor a -> IO a runPropellor host a = runReaderT (runWithHost a) host @@ -62,3 +65,18 @@ onlyProcess lockfile a = bracket lock unlock (const a) return l unlock = closeFd alreadyrunning = error "Propellor is already running on this host!" + +-- | Reads and displays each line from the Handle, except for the last line +-- which is a Result. +processChainOutput :: Handle -> IO Result +processChainOutput h = go Nothing + where + go lastline = do + v <- catchMaybeIO (hGetLine h) + case v of + Nothing -> pure $ fromMaybe FailedChange $ + readish =<< lastline + Just s -> do + maybe noop putStrLn lastline + hFlush stdout + go (Just s) diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index e504693..38e09b5 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -2,12 +2,17 @@ module Propellor.Property.Chroot ( Chroot, chroot, provisioned, + chain, ) where import Propellor import qualified Propellor.Property.Debootstrap as Debootstrap +import qualified Propellor.Shim as Shim +import Utility.SafeCommand import qualified Data.Map as M +import Data.List.Utils +import System.Posix.Directory data Chroot = Chroot FilePath System Host @@ -35,8 +40,7 @@ provisioned c@(Chroot loc system _) = RevertableProperty (propigateChrootInfo c (go "exists" setup)) (go "removed" teardown) where - go desc a = property ("chroot " ++ loc ++ " " ++ desc) $ do - ensureProperties [a] + go desc a = property (chrootDesc c desc) $ ensureProperties [a] setup = provisionChroot c `requires` built @@ -53,5 +57,71 @@ propigateChrootInfo c@(Chroot loc _ h) p = propigateInfo c p (<> chrootinfo) where chrootinfo = mempty $ mempty { _chroots = M.singleton loc h } +-- | Propellor is run inside the chroot to provision it. +-- +-- Strange and wonderful tricks let the host's /usr/local/propellor +-- be used inside the chroot, without needing to install anything. provisionChroot :: Chroot -> Property -provisionChroot = undefined +provisionChroot c@(Chroot loc _ _) = property (chrootDesc c "provisioned") $ do + let d = localdir shimdir c + let me = localdir "propellor" + shim <- liftIO $ ifM (doesDirectoryExist d) + ( pure (Shim.file me d) + , Shim.setup me d + ) + ifM (liftIO $ bindmount shim) + ( chainprovision shim + , return FailedChange + ) + where + bindmount shim = ifM (doesFileExist (loc ++ shim)) + ( return True + , do + let mntpnt = loc ++ localdir + createDirectoryIfMissing True mntpnt + boolSystem "mount" + [ Param "--bind" + , File localdir, File mntpnt + ] + ) + + chainprovision shim = do + parenthost <- asks hostName + let p = inChrootProcess c + [ shim + , "--continue" + , show $ toChain parenthost c + ] + liftIO $ withHandle StdoutHandle createProcessSuccess p + processChainOutput + +toChain :: HostName -> Chroot -> CmdLine +toChain parenthost (Chroot loc _ _) = ChrootChain parenthost loc + +chain :: [Host] -> HostName -> FilePath -> IO () +chain hostlist hn loc = case findHostNoAlias hostlist hn of + Nothing -> errorMessage ("cannot find host " ++ hn) + Just parenthost -> case M.lookup loc (_chroots $ _chrootinfo $ hostInfo parenthost) of + Nothing -> errorMessage ("cannot find chroot " ++ loc ++ " on host " ++ hn) + Just h -> go h + where + go h = do + changeWorkingDirectory localdir + onlyProcess (provisioningLock loc) $ do + r <- runPropellor h $ ensureProperties $ hostProperties h + putStrLn $ "\n" ++ show r + +inChrootProcess :: Chroot -> [String] -> CreateProcess +inChrootProcess (Chroot loc _ _) cmd = proc "chroot" (loc:cmd) + +provisioningLock :: FilePath -> FilePath +provisioningLock containerloc = "chroot" mungeloc containerloc ++ ".lock" + +shimdir :: Chroot -> FilePath +shimdir (Chroot loc _ _) = "chroot" mungeloc loc ++ ".shim" + +mungeloc :: FilePath -> String +mungeloc = replace "/" "_" + +chrootDesc :: Chroot -> String -> String +chrootDesc (Chroot loc _ _) desc = "chroot " ++ loc ++ " " ++ desc diff --git a/src/Propellor/Property/Docker.hs b/src/Propellor/Property/Docker.hs index 92cc124..5cf60ff 100644 --- a/src/Propellor/Property/Docker.hs +++ b/src/Propellor/Property/Docker.hs @@ -41,7 +41,7 @@ module Propellor.Property.Docker ( import Propellor hiding (init) import qualified Propellor.Property.File as File import qualified Propellor.Property.Apt as Apt -import qualified Propellor.Property.Docker.Shim as Shim +import qualified Propellor.Shim as Shim import Utility.SafeCommand import Utility.Path import Utility.ThreadScheduler @@ -432,20 +432,10 @@ provisionContainer cid = containerDesc cid $ property "provisioned" $ liftIO $ d [ if isConsole msgh then "-it" else "-i" ] (shim : params) r <- withHandle StdoutHandle createProcessSuccess p $ - processoutput Nothing + processChainOutput when (r /= FailedChange) $ setProvisionedFlag cid return r - where - processoutput lastline h = do - v <- catchMaybeIO (hGetLine h) - case v of - Nothing -> pure $ fromMaybe FailedChange $ - readish =<< lastline - Just s -> do - maybe noop putStrLn lastline - hFlush stdout - processoutput (Just s) h toChain :: ContainerId -> CmdLine toChain cid = DockerChain (containerHostName cid) (fromContainerId cid) diff --git a/src/Propellor/Property/Docker/Shim.hs b/src/Propellor/Shim.hs similarity index 89% rename from src/Propellor/Property/Docker/Shim.hs rename to src/Propellor/Shim.hs index c2f35d0..5b5aa68 100644 --- a/src/Propellor/Property/Docker/Shim.hs +++ b/src/Propellor/Shim.hs @@ -1,9 +1,10 @@ --- | Support for running propellor, as built outside a docker container, --- inside the container. +-- | Support for running propellor, as built outside a container, +-- inside the container, without needing to install anything into the +-- container. -- -- Note: This is currently Debian specific, due to glibcLibs. -module Propellor.Property.Docker.Shim (setup, cleanEnv, file) where +module Propellor.Shim (setup, cleanEnv, file) where import Propellor import Utility.LinuxMkLibs diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index 16ddcc7..56eafc6 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -155,6 +155,7 @@ data CmdLine | Update HostName | DockerInit HostName | DockerChain HostName String + | ChrootChain HostName FilePath | GitPush Fd Fd deriving (Read, Show, Eq) From 2293b46bf714ca5675515496c041952310b7bae2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 16:04:48 -0400 Subject: [PATCH 08/12] fix chroot info propigation --- debian/changelog | 1 + src/Propellor/Property/Chroot.hs | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 155d512..e36e151 100644 --- a/debian/changelog +++ b/debian/changelog @@ -19,6 +19,7 @@ propellor (1.0.0) UNRELEASED; urgency=medium in the main host list, and are instead passed to Docker.docked. (API change) * Added support for using debootstrap from propellor. + * Propellor can now be used to provision chroots. -- Joey Hess Mon, 10 Nov 2014 11:15:27 -0400 diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index 38e09b5..9ce9dda 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -53,9 +53,11 @@ provisioned c@(Chroot loc system _) = RevertableProperty teardown = undefined propigateChrootInfo :: Chroot -> Property -> Property -propigateChrootInfo c@(Chroot loc _ h) p = propigateInfo c p (<> chrootinfo) - where - chrootinfo = mempty $ mempty { _chroots = M.singleton loc h } +propigateChrootInfo c p = propigateInfo c p (<> chrootInfo c) + +chrootInfo :: Chroot -> Info +chrootInfo (Chroot loc _ h) = + mempty { _chrootinfo = mempty { _chroots = M.singleton loc h } } -- | Propellor is run inside the chroot to provision it. -- From 00cab6901feec708b69d4f0ac1a4d3d0ec7caa45 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 16:05:06 -0400 Subject: [PATCH 09/12] propellor spin From 3669bd61d0e15682ce25b9a82788b8d69f87d123 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 16:07:57 -0400 Subject: [PATCH 10/12] propellor spin --- src/Propellor/Engine.hs | 2 +- src/Propellor/Property/Chroot.hs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Propellor/Engine.hs b/src/Propellor/Engine.hs index 0cbf624..969769c 100644 --- a/src/Propellor/Engine.hs +++ b/src/Propellor/Engine.hs @@ -77,6 +77,6 @@ processChainOutput h = go Nothing Nothing -> pure $ fromMaybe FailedChange $ readish =<< lastline Just s -> do - maybe noop putStrLn lastline + maybe noop (\l -> unless (null l) (putStrLn l)) lastline hFlush stdout go (Just s) diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index 9ce9dda..ba7bf96 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -109,6 +109,7 @@ chain hostlist hn loc = case findHostNoAlias hostlist hn of where go h = do changeWorkingDirectory localdir + forceConsole onlyProcess (provisioningLock loc) $ do r <- runPropellor h $ ensureProperties $ hostProperties h putStrLn $ "\n" ++ show r From f9cc7c149ead60d1178fa3b480282f6089cf79bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 17:18:26 -0400 Subject: [PATCH 11/12] incomplete systemd container support --- debian/changelog | 2 + propellor.cabal | 1 + src/Propellor/Property.hs | 4 - src/Propellor/Property/Chroot.hs | 6 +- src/Propellor/Property/Debootstrap.hs | 2 +- src/Propellor/Property/Systemd.hs | 103 ++++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 src/Propellor/Property/Systemd.hs diff --git a/debian/changelog b/debian/changelog index e36e151..d6dc615 100644 --- a/debian/changelog +++ b/debian/changelog @@ -20,6 +20,8 @@ propellor (1.0.0) UNRELEASED; urgency=medium Docker.docked. (API change) * Added support for using debootstrap from propellor. * Propellor can now be used to provision chroots. + * systemd-nspawn containers can now be managed by propellor, very similar + to its handling of docker containers. -- Joey Hess Mon, 10 Nov 2014 11:15:27 -0400 diff --git a/propellor.cabal b/propellor.cabal index 23708fd..f45900c 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -95,6 +95,7 @@ Library Propellor.Property.Service Propellor.Property.Ssh Propellor.Property.Sudo + Propellor.Property.Systemd Propellor.Property.Tor Propellor.Property.User Propellor.Property.HostingProvider.CloudAtCost diff --git a/src/Propellor/Property.hs b/src/Propellor/Property.hs index 1d750a7..6ace5e4 100644 --- a/src/Propellor/Property.hs +++ b/src/Propellor/Property.hs @@ -131,10 +131,6 @@ boolProperty desc a = property desc $ ifM (liftIO a) revert :: RevertableProperty -> RevertableProperty revert (RevertableProperty p1 p2) = RevertableProperty p2 p1 --- | Turns a revertable property into a regular property. -unrevertable :: RevertableProperty -> Property -unrevertable (RevertableProperty p1 _p2) = p1 - -- Changes the action that is performed to satisfy a property. adjustProperty :: Property -> (Propellor Result -> Propellor Result) -> Property adjustProperty p f = p { propertySatisfy = f (propertySatisfy p) } diff --git a/src/Propellor/Property/Chroot.hs b/src/Propellor/Property/Chroot.hs index ba7bf96..798330b 100644 --- a/src/Propellor/Property/Chroot.hs +++ b/src/Propellor/Property/Chroot.hs @@ -1,5 +1,5 @@ module Propellor.Property.Chroot ( - Chroot, + Chroot(..), chroot, provisioned, chain, @@ -24,7 +24,7 @@ instance Hostlike Chroot where -- | Defines a Chroot at the given location, containing the specified -- System. Properties can be added to configure the Chroot. -- --- > chroot "/srv/chroot/ghc-dev" (System (Debian Unstable) "amd64" +-- > chroot "/srv/chroot/ghc-dev" (System (Debian Unstable) "amd64") -- > & Apt.installed ["build-essential", "ghc", "haskell-platform"] -- > & ... chroot :: FilePath -> System -> Chroot @@ -48,7 +48,7 @@ provisioned c@(Chroot loc system _) = RevertableProperty (System (Debian _) _) -> debootstrap (System (Ubuntu _) _) -> debootstrap - debootstrap = unrevertable (Debootstrap.built loc system []) + debootstrap = toProp (Debootstrap.built loc system []) teardown = undefined diff --git a/src/Propellor/Property/Debootstrap.hs b/src/Propellor/Property/Debootstrap.hs index 4e7bc74..5f521c3 100644 --- a/src/Propellor/Property/Debootstrap.hs +++ b/src/Propellor/Property/Debootstrap.hs @@ -33,7 +33,7 @@ built target system@(System _ arch) extraparams = RevertableProperty setup teardown where setup = check (unpopulated target <||> ispartial) setupprop - `requires` unrevertable installed + `requires` toProp installed teardown = check (not <$> unpopulated target) teardownprop diff --git a/src/Propellor/Property/Systemd.hs b/src/Propellor/Property/Systemd.hs new file mode 100644 index 0000000..be08a84 --- /dev/null +++ b/src/Propellor/Property/Systemd.hs @@ -0,0 +1,103 @@ +module Propellor.Property.Systemd ( + installed, + persistentJournal, + container, + nspawned, +) where + +import Propellor +import qualified Propellor.Property.Chroot as Chroot +import qualified Propellor.Property.Apt as Apt +import Utility.SafeCommand + +import Data.List.Utils + +type MachineName = String + +type NspawnParam = CommandParam + +data Container = Container MachineName System [CommandParam] Host + +instance Hostlike Container where + (Container n s ps h) & p = Container n s ps (h & p) + (Container n s ps h) &^ p = Container n s ps (h &^ p) + getHost (Container _ _ _ h) = h + +-- dbus is only a Recommends of systemd, but is needed for communication +-- from the systemd inside a container to the one outside, so make sure it +-- gets installed. +installed :: Property +installed = Apt.installed ["systemd", "dbus"] + +-- | Sets up persistent storage of the journal. +persistentJournal :: Property +persistentJournal = check (not <$> doesDirectoryExist dir) $ + combineProperties "persistent systetemd journal" + [ cmdProperty "install" ["-d", "-g", "systemd-journal", dir] + , cmdProperty "setfacl" ["-R", "-nm", "g:adm:rx,d:g:adm:rx", dir] + ] + `requires` Apt.installed ["acl"] + where + dir = "/var/log/journal" + +-- | Defines a container with a given machine name, containing the specified +-- System. Properties can be added to configure the Container. +-- +-- > container "webserver" (System (Debian Unstable) "amd64") [] +container :: MachineName -> System -> [NspawnParam] -> Container +container name system ps = Container name system ps (Host name [] mempty) + +-- | Runs a container using systemd-nspawn. +-- +-- A systemd unit is set up for the container, so it will automatically +-- be started on boot. +-- +-- Systemd is automatically installed inside the container, and will +-- communicate with the host's systemd. This allows systemctl to be used to +-- examine the status of services running inside the container. +-- +-- When the host system has persistentJournal enabled, journactl can be +-- used to examine logs forwarded from the container. +-- +-- Reverting this property stops the container, removes the systemd unit, +-- and deletes the chroot and all its contents. +nspawned :: Container -> RevertableProperty +nspawned c@(Container name system _ h) = RevertableProperty setup teardown + where + -- TODO after container is running, use nsenter to enter it + -- and run propellor to finish provisioning. + setup = toProp (nspawnService c) + `requires` toProp chrootprovisioned + + teardown = toProp (revert (chrootprovisioned)) + `requires` toProp (revert (nspawnService c)) + + -- When provisioning the chroot, pass a version of the Host + -- that only has the Property of systemd being installed. + -- This is to avoid starting any daemons in the chroot, + -- which would not run in the container's namespace. + chrootprovisioned = Chroot.provisioned $ + Chroot.Chroot (containerDir name) system $ + h { hostProperties = [installed] } + +nspawnService :: Container -> RevertableProperty +nspawnService (Container name _ ps _) = RevertableProperty setup teardown + where + service = nspawnServiceName name + servicefile = "/etc/systemd/system/multi-user.target.wants" service + + setup = check (not <$> doesFileExist servicefile) $ + combineProperties ("container running " ++ service) + [ cmdProperty "systemctl" ["enable", service] + , cmdProperty "systemctl" ["start", service] + ] + + -- TODO adjust execStart line to reflect ps + + teardown = undefined + +nspawnServiceName :: MachineName -> String +nspawnServiceName name = "systemd-nspawn@" ++ name ++ ".service" + +containerDir :: MachineName -> FilePath +containerDir name = "/var/lib/container" ++ replace "/" "_" name From ba862ae8877c21eb63f7fba08ba5fc934a4c391c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 20 Nov 2014 19:21:17 -0400 Subject: [PATCH 12/12] adding blog feeds --- doc/feeds.mdwn | 4 ++++ doc/index.mdwn | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 doc/feeds.mdwn diff --git a/doc/feeds.mdwn b/doc/feeds.mdwn new file mode 100644 index 0000000..7e35993 --- /dev/null +++ b/doc/feeds.mdwn @@ -0,0 +1,4 @@ +Aggregating propellor blog posts etc.. + +* [[!aggregate expirecount=25 name="joey" feedurl="http://joeyh.name/blog/propellor/" url="http://joeyh.name/blog/propellor/index.rss"]] + diff --git a/doc/index.mdwn b/doc/index.mdwn index f5fd880..d670006 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -31,3 +31,7 @@ You are encouraged to send patches and improve it. See [[contributing]]. ## news [[!inline pages="news/* and !*/Discussion" show="4" archive=yes]] + +## feeds + +[[!inline pages="feeds/* and !*/Discussion" show="4" archive=yes]]