From 39b9a61591bf28b1c9c5dc18a6f668c8becb9f6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:42:51 -0400 Subject: [PATCH 01/45] propellor spin From 4a0cac113cf999a58a60f7db7a11d5b0ad623699 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:53:42 -0400 Subject: [PATCH 02/45] fix color display when running propellor inside docker --- src/Propellor/CmdLine.hs | 6 ++++-- src/Propellor/Message.hs | 12 +++++++++++- src/Propellor/Property/Docker.hs | 6 +++--- src/Propellor/Types.hs | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index e7da0a8..a79a582 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -55,7 +55,8 @@ processCmdLine = go =<< getArgs go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage "--continue serialization failure" - go ("--chain":h:[]) = return $ Chain h + go ("--chain":h:[]) = return $ Chain h False + go ("--chain":h:b:[]) = return $ Chain h (Prelude.read b) go ("--docker":h:[]) = return $ Docker h go ("--gitpush":fin:fout:_) = return $ GitPush (Prelude.read fin) (Prelude.read fout) go (h:[]) @@ -86,7 +87,8 @@ defaultMain hostlist = do go _ (Edit field context) = editPrivData field context go _ ListFields = listPrivDataFields hostlist go _ (AddKey keyid) = addKey keyid - go _ (Chain hn) = withhost hn $ \h -> do + go _ (Chain hn isconsole) = withhost hn $ \h -> do + when isconsole forceConsole r <- runPropellor h $ ensureProperties $ hostProperties h putStrLn $ "\n" ++ show r go _ (Docker hn) = Docker.chain hn diff --git a/src/Propellor/Message.hs b/src/Propellor/Message.hs index e184a59..639171c 100644 --- a/src/Propellor/Message.hs +++ b/src/Propellor/Message.hs @@ -6,20 +6,30 @@ import System.Console.ANSI import System.IO import System.Log.Logger import "mtl" Control.Monad.Reader +import Data.Maybe +import Control.Applicative import Propellor.Types import Utility.Monad +import Utility.Env data MessageHandle = ConsoleMessageHandle | TextMessageHandle mkMessageHandle :: IO MessageHandle -mkMessageHandle = ifM (hIsTerminalDevice stdout) +mkMessageHandle = ifM (hIsTerminalDevice stdout <||> (isJust <$> getEnv "PROPELLOR_CONSOLE")) ( return ConsoleMessageHandle , return TextMessageHandle ) +forceConsole :: IO () +forceConsole = void $ setEnv "PROPELLOR_CONSOLE" "1" True + +isConsole :: MessageHandle -> Bool +isConsole ConsoleMessageHandle = True +isConsole _ = False + whenConsole :: MessageHandle -> IO () -> IO () whenConsole ConsoleMessageHandle a = a whenConsole _ _ = return () diff --git a/src/Propellor/Property/Docker.hs b/src/Propellor/Property/Docker.hs index 5a7a084..d005592 100644 --- a/src/Propellor/Property/Docker.hs +++ b/src/Propellor/Property/Docker.hs @@ -416,7 +416,7 @@ chain s = case toContainerId s of -- to avoid ever provisioning twice at the same time. whenM (checkProvisionedFlag cid) $ do let shim = Shim.file (localdir "propellor") (localdir shimdir cid) - unlessM (boolSystem shim [Param "--continue", Param $ show $ Chain $ containerHostName cid]) $ + unlessM (boolSystem shim [Param "--continue", Param $ show $ Chain (containerHostName cid) False]) $ warningMessage "Boot provision failed!" void $ async $ job reapzombies void $ async $ job $ simpleSh $ namedPipe cid @@ -440,13 +440,13 @@ chain s = case toContainerId s of provisionContainer :: ContainerId -> Property provisionContainer cid = containerDesc cid $ property "provisioned" $ liftIO $ do let shim = Shim.file (localdir "propellor") (localdir shimdir cid) + msgh <- mkMessageHandle + let params = ["--continue", show $ Chain (containerHostName cid) (isConsole msgh)] r <- simpleShClientRetry 60 (namedPipe cid) shim params (go Nothing) when (r /= FailedChange) $ setProvisionedFlag cid return r where - params = ["--continue", show $ Chain $ containerHostName cid] - go lastline (v:rest) = case v of StdoutLine s -> do maybe noop putStrLn lastline diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index 72ccd22..f70eee6 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -144,7 +144,7 @@ data CmdLine | ListFields | AddKey String | Continue CmdLine - | Chain HostName + | Chain HostName Bool | Boot HostName | Docker HostName | GitPush Fd Fd From fff8d91a04549a224e86a9fbaddbcc2bbe33636b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:54:13 -0400 Subject: [PATCH 03/45] propellor spin From 75a824c7883e53c75dafa7bc5144ee44e51d3cd2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:57:04 -0400 Subject: [PATCH 04/45] fix display of progress etc lines before protocol line --- src/Propellor/Protocol.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Propellor/Protocol.hs b/src/Propellor/Protocol.hs index 99afb31..198b3f3 100644 --- a/src/Propellor/Protocol.hs +++ b/src/Propellor/Protocol.hs @@ -1,7 +1,7 @@ -- | This is a simple line-based protocol used for communication between -- a local and remote propellor. It's sent over a ssh channel, and lines of -- the protocol can be interspersed with other, non-protocol lines --- that should be ignored. +-- that should be passed through to be displayed. module Propellor.Protocol where @@ -48,7 +48,9 @@ getMarked h marker = go =<< catchMaybeIO (hGetLine h) where go Nothing = return Nothing go (Just l) = case fromMarked marker l of - Nothing -> getMarked h marker + Nothing -> do + hPutStrLn stderr l + getMarked h marker Just v -> return (Just v) req :: Stage -> Marker -> (String -> IO ()) -> IO () From 72c6fb90cb126fa5a414fbd502ee4c5615c444f8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:57:24 -0400 Subject: [PATCH 05/45] propellor spin From 557458d03ccd591223c574a48df09fca9e311f05 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:59:26 -0400 Subject: [PATCH 06/45] wording --- src/Propellor/CmdLine.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index a79a582..9797d03 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -285,7 +285,7 @@ spin hn hst = do -- Initial git clone, used for bootstrapping. sendGitClone :: HostName -> IO () -sendGitClone hn = void $ actionMessage ("Pushing git repository to " ++ hn) $ do +sendGitClone hn = void $ actionMessage ("Cloning git repository to " ++ hn) $ do branch <- getCurrentBranch cacheparams <- sshCachingParams hn withTmpFile "propellor.git" $ \tmp _ -> allM id From 74e833241c49a240e6045c1bc4a08c1c52f2dc60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 17:59:40 -0400 Subject: [PATCH 07/45] propellor spin From d952c844072fd2c1a3a63dcecc7bfb305162d222 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:01:25 -0400 Subject: [PATCH 08/45] propellor spin From e262a49a80ccc1f33fee62d1d3e1d54f22c7e6f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:07:26 -0400 Subject: [PATCH 09/45] enable terminal for --boot --- src/Propellor/CmdLine.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 9797d03..d863ed3 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -196,12 +196,14 @@ getCurrentGitSha1 branchref = readProcess "git" ["show-ref", "--hash", branchref -- updated, it's run. spin :: HostName -> Host -> IO () spin hn hst = do - void $ gitCommit [Param "--allow-empty", Param "-a", Param "-m", Param "propellor spin"] + void $ actionMessage "git commit (signed)" $ + gitCommit [Param "--allow-empty", Param "-a", Param "-m", Param "propellor spin"] -- Push to central origin repo first, if possible. -- The remote propellor will pull from there, which avoids -- us needing to send stuff directly to the remote host. whenM hasOrigin $ - void $ boolSystem "git" [Param "push"] + void $ actionMessage "pushing to central git repository" $ + boolSystem "git" [Param "push"] cacheparams <- toCommand <$> sshCachingParams hn comm cacheparams =<< hostprivdata @@ -212,7 +214,7 @@ spin hn hst = do comm cacheparams privdata = withBothHandles createProcessSuccess - (proc "ssh" $ cacheparams ++ [user, bootstrapcmd]) + (proc "ssh" $ cacheparams ++ ["-t", user, bootstrapcmd]) (comm' cacheparams privdata) comm' cacheparams privdata (toh, fromh) = loop where From 01bcf447e3bf0d86a523a1788f3c5925a1fb8d69 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:11:19 -0400 Subject: [PATCH 10/45] propellor spin From 18903ad30c9a94b03c445eda2e15d6acb885e794 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:13:42 -0400 Subject: [PATCH 11/45] display improvements --- src/Propellor/CmdLine.hs | 10 ++++++---- src/Propellor/Protocol.hs | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index d863ed3..6716e36 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -100,7 +100,9 @@ defaultMain hostlist = do ( onlyProcess $ withhost hn mainProperties , go True (Spin hn) ) - go False (Boot _) = onlyProcess boot + go False (Boot _) = do + forceConsole + onlyProcess boot withhost :: HostName -> (Host -> IO ()) -> IO () withhost hn a = maybe (unknownhost hn hostlist) a (findHost hostlist hn) @@ -196,18 +198,18 @@ getCurrentGitSha1 branchref = readProcess "git" ["show-ref", "--hash", branchref -- updated, it's run. spin :: HostName -> Host -> IO () spin hn hst = do - void $ actionMessage "git commit (signed)" $ + void $ actionMessage "Git commit (signed)" $ gitCommit [Param "--allow-empty", Param "-a", Param "-m", Param "propellor spin"] -- Push to central origin repo first, if possible. -- The remote propellor will pull from there, which avoids -- us needing to send stuff directly to the remote host. whenM hasOrigin $ - void $ actionMessage "pushing to central git repository" $ + void $ actionMessage "Push to central git repository" $ boolSystem "git" [Param "push"] cacheparams <- toCommand <$> sshCachingParams hn comm cacheparams =<< hostprivdata - unlessM (boolSystem "ssh" (map Param (cacheparams ++ ["-t", user, runcmd]))) $ + unlessM (boolSystem "ssh" (map Param (cacheparams ++ [user, runcmd]))) $ error $ "remote propellor failed (running: " ++ runcmd ++")" where hostprivdata = show . filterPrivData hst <$> decryptPrivData diff --git a/src/Propellor/Protocol.hs b/src/Propellor/Protocol.hs index 198b3f3..7249e2b 100644 --- a/src/Propellor/Protocol.hs +++ b/src/Propellor/Protocol.hs @@ -49,7 +49,8 @@ getMarked h marker = go =<< catchMaybeIO (hGetLine h) go Nothing = return Nothing go (Just l) = case fromMarked marker l of Nothing -> do - hPutStrLn stderr l + unless (null l) $ + hPutStrLn stderr l getMarked h marker Just v -> return (Just v) From f8917a505e80ed58b0f3a0cab20d3941229882a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:13:52 -0400 Subject: [PATCH 12/45] propellor spin From 8f5166748a0a722946817e3626b383cf9fa346c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:15:04 -0400 Subject: [PATCH 13/45] propellor spin --- src/Propellor/CmdLine.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 6716e36..bbbcf18 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -209,14 +209,14 @@ spin hn hst = do cacheparams <- toCommand <$> sshCachingParams hn comm cacheparams =<< hostprivdata - unlessM (boolSystem "ssh" (map Param (cacheparams ++ [user, runcmd]))) $ + unlessM (boolSystem "ssh" (map Param (cacheparams ++ ["-t", user, runcmd]))) $ error $ "remote propellor failed (running: " ++ runcmd ++")" where hostprivdata = show . filterPrivData hst <$> decryptPrivData comm cacheparams privdata = withBothHandles createProcessSuccess - (proc "ssh" $ cacheparams ++ ["-t", user, bootstrapcmd]) + (proc "ssh" $ cacheparams ++ [ user, bootstrapcmd]) (comm' cacheparams privdata) comm' cacheparams privdata (toh, fromh) = loop where From 24280d80172f2d999b3373e190c502b93631709b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:16:19 -0400 Subject: [PATCH 14/45] propellor spin From 4bbee8eac6c49fb5f6722910afb353aadcd1ef03 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:18:32 -0400 Subject: [PATCH 15/45] minimize output during build --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9185099..43d7d05 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ run: deps build dev: build tags build: dist/setup-config - if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi - ln -sf dist/build/propellor-config/propellor-config propellor + @if ! $(CABAL) build; then $(CABAL) configure; $(CABAL) build; fi + @ln -sf dist/build/propellor-config/propellor-config propellor deps: @if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS) || (apt-get update && apt-get --no-upgrade --no-install-recommends -y install $(DEBDEPS)); fi || true From b7bbde3d72e81fb04d5c3747f14c0b3a656334c0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:26:15 -0400 Subject: [PATCH 16/45] avoid extra git pull when --spin calles first --boot and then --run --- src/Propellor/CmdLine.hs | 4 +++- src/Propellor/Types.hs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index bbbcf18..21ad5db 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -96,6 +96,8 @@ defaultMain hostlist = do go True cmdline@(Spin _) = buildFirst cmdline $ go False cmdline go True cmdline = updateFirst cmdline $ go False cmdline go False (Spin hn) = withhost hn $ spin hn + go False cmdline@(SimpleRun hn) = buildFirst cmdline $ + go False (Run hn) go False (Run hn) = ifM ((==) 0 <$> getRealUserID) ( onlyProcess $ withhost hn mainProperties , go True (Spin hn) @@ -278,7 +280,7 @@ spin hn hst = do ] runcmd = mkcmd - [ "cd " ++ localdir ++ " && ./propellor --run " ++ hn ] + [ "cd " ++ localdir ++ " && ./propellor --continue " ++ shellEscape (show (SimpleRun hn)) ] showremote s = putStrLn s diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index f70eee6..e0a7113 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -138,6 +138,7 @@ instance ActionResult Result where data CmdLine = Run HostName | Spin HostName + | SimpleRun HostName | Set PrivDataField Context | Dump PrivDataField Context | Edit PrivDataField Context From d7afee993e3cf9f1f1bf1e82c6858bded43c995d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:26:53 -0400 Subject: [PATCH 17/45] propellor spin From 6200173cdf86abdf8f4ce7374a3c04b2d1c7231a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:29:42 -0400 Subject: [PATCH 18/45] propellor spin From 45592b442b02c41993c9c62eb7f06bcb1267c117 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:39:10 -0400 Subject: [PATCH 19/45] factor out git repo module --- propellor.cabal | 1 + src/Propellor/CmdLine.hs | 37 +----------------------------------- src/Propellor/Git.hs | 41 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 src/Propellor/Git.hs diff --git a/propellor.cabal b/propellor.cabal index 0a01ada..a9aa4d4 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -113,6 +113,7 @@ Library Other-Modules: Propellor.Types.Info Propellor.CmdLine + Propellor.Git Propellor.Gpg Propellor.SimpleSh Propellor.PrivData.Paths diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 21ad5db..c5dcd34 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -19,6 +19,7 @@ import Propellor import Propellor.Protocol import Propellor.PrivData.Paths import Propellor.Gpg +import Propellor.Git import qualified Propellor.Property.Docker as Docker import qualified Propellor.Property.Docker.Shim as DockerShim import Utility.FileMode @@ -143,10 +144,6 @@ buildFirst cmdline next = do where getmtime = catchMaybeIO $ getModificationTime "propellor" -getCurrentBranch :: IO String -getCurrentBranch = takeWhile (/= '\n') - <$> readProcess "git" ["symbolic-ref", "--short", "HEAD"] - updateFirst :: CmdLine -> IO () -> IO () updateFirst cmdline next = ifM hasOrigin (updateFirst' cmdline next, next) @@ -192,9 +189,6 @@ updateFirst' cmdline next = do , errorMessage "Propellor build failed!" ) -getCurrentGitSha1 :: String -> IO String -getCurrentGitSha1 branchref = readProcess "git" ["show-ref", "--hash", branchref] - -- spin handles deploying propellor to a remote host, if it's not already -- installed there, or updating it if it is. Once the remote propellor is -- updated, it's run. @@ -351,35 +345,6 @@ gitPush hin hout = void $ fromstdin `concurrently` tostdout hFlush toh connect fromh toh -hasOrigin :: IO Bool -hasOrigin = do - rs <- lines <$> readProcess "git" ["remote"] - return $ "origin" `elem` rs - -setRepoUrl :: String -> IO () -setRepoUrl "" = return () -setRepoUrl url = do - subcmd <- ifM hasOrigin (pure "set-url", pure "add") - void $ boolSystem "git" [Param "remote", Param subcmd, Param "origin", Param url] - -- same as --set-upstream-to, except origin branch - -- may not have been pulled yet - branch <- getCurrentBranch - let branchval s = "branch." ++ branch ++ "." ++ s - void $ boolSystem "git" [Param "config", Param (branchval "remote"), Param "origin"] - void $ boolSystem "git" [Param "config", Param (branchval "merge"), Param $ "refs/heads/"++branch] - -getRepoUrl :: IO (Maybe String) -getRepoUrl = getM get urls - where - urls = ["remote.deploy.url", "remote.origin.url"] - get u = do - v <- catchMaybeIO $ - takeWhile (/= '\n') - <$> readProcess "git" ["config", u] - return $ case v of - Just url | not (null url) -> Just url - _ -> Nothing - checkDebugMode :: IO () checkDebugMode = go =<< getEnv "PROPELLOR_DEBUG" where diff --git a/src/Propellor/Git.hs b/src/Propellor/Git.hs new file mode 100644 index 0000000..0de82f8 --- /dev/null +++ b/src/Propellor/Git.hs @@ -0,0 +1,41 @@ +module Propellor.Git where + +import Propellor +import Utility.SafeCommand + +getCurrentBranch :: IO String +getCurrentBranch = takeWhile (/= '\n') + <$> readProcess "git" ["symbolic-ref", "--short", "HEAD"] + +getCurrentGitSha1 :: String -> IO String +getCurrentGitSha1 branchref = readProcess "git" ["show-ref", "--hash", branchref] + +setRepoUrl :: String -> IO () +setRepoUrl "" = return () +setRepoUrl url = do + subcmd <- ifM hasOrigin (pure "set-url", pure "add") + void $ boolSystem "git" [Param "remote", Param subcmd, Param "origin", Param url] + -- same as --set-upstream-to, except origin branch + -- may not have been pulled yet + branch <- getCurrentBranch + let branchval s = "branch." ++ branch ++ "." ++ s + void $ boolSystem "git" [Param "config", Param (branchval "remote"), Param "origin"] + void $ boolSystem "git" [Param "config", Param (branchval "merge"), Param $ "refs/heads/"++branch] + +getRepoUrl :: IO (Maybe String) +getRepoUrl = getM get urls + where + urls = ["remote.deploy.url", "remote.origin.url"] + get u = do + v <- catchMaybeIO $ + takeWhile (/= '\n') + <$> readProcess "git" ["config", u] + return $ case v of + Just url | not (null url) -> Just url + _ -> Nothing + +hasOrigin :: IO Bool +hasOrigin = do + rs <- lines <$> readProcess "git" ["remote"] + return $ "origin" `elem` rs + From 47bcd4e1306b808311f39f398b0e539700e5efc6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:42:36 -0400 Subject: [PATCH 20/45] factor out ssh module --- propellor.cabal | 1 + src/Propellor/CmdLine.hs | 38 +---------------------------------- src/Propellor/Ssh.hs | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 src/Propellor/Ssh.hs diff --git a/propellor.cabal b/propellor.cabal index a9aa4d4..2a8e3a0 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -116,6 +116,7 @@ Library Propellor.Git Propellor.Gpg Propellor.SimpleSh + Propellor.Ssh Propellor.PrivData.Paths Propellor.Protocol Propellor.Property.Docker.Shim diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index c5dcd34..3e24dd3 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -10,7 +10,6 @@ import System.Log.Handler.Simple import System.PosixCompat import Control.Exception (bracket) import System.Posix.IO -import Data.Time.Clock.POSIX import Control.Concurrent.Async import qualified Data.ByteString as B import System.Process (std_in, std_out) @@ -20,11 +19,11 @@ import Propellor.Protocol import Propellor.PrivData.Paths import Propellor.Gpg import Propellor.Git +import Propellor.Ssh import qualified Propellor.Property.Docker as Docker import qualified Propellor.Property.Docker.Shim as DockerShim import Utility.FileMode import Utility.SafeCommand -import Utility.UserInfo usage :: IO a usage = do @@ -355,38 +354,3 @@ checkDebugMode = go =<< getEnv "PROPELLOR_DEBUG" updateGlobalLogger rootLoggerName $ setLevel DEBUG . setHandlers [f] go _ = noop - --- Parameters can be passed to both ssh and scp, to enable a ssh connection --- caching socket. --- --- If the socket already exists, check if its mtime is older than 10 --- minutes, and if so stop that ssh process, in order to not try to --- use an old stale connection. (atime would be nicer, but there's --- a good chance a laptop uses noatime) -sshCachingParams :: HostName -> IO [CommandParam] -sshCachingParams hn = do - home <- myHomeDir - let cachedir = home ".ssh" "propellor" - createDirectoryIfMissing False cachedir - let socketfile = cachedir hn ++ ".sock" - let ps = - [ Param "-o", Param ("ControlPath=" ++ socketfile) - , Params "-o ControlMaster=auto -o ControlPersist=yes" - ] - - maybe noop (expireold ps socketfile) - =<< catchMaybeIO (getFileStatus socketfile) - - return ps - - where - expireold ps f s = do - now <- truncate <$> getPOSIXTime :: IO Integer - if modificationTime s > fromIntegral now - tenminutes - then touchFile f - else do - void $ boolSystem "ssh" $ - [ Params "-O stop" ] ++ ps ++ - [ Param "localhost" ] - nukeFile f - tenminutes = 600 diff --git a/src/Propellor/Ssh.hs b/src/Propellor/Ssh.hs new file mode 100644 index 0000000..969517a --- /dev/null +++ b/src/Propellor/Ssh.hs @@ -0,0 +1,43 @@ +module Propellor.Ssh where + +import Propellor +import Utility.SafeCommand +import Utility.UserInfo + +import System.PosixCompat +import Data.Time.Clock.POSIX + +-- Parameters can be passed to both ssh and scp, to enable a ssh connection +-- caching socket. +-- +-- If the socket already exists, check if its mtime is older than 10 +-- minutes, and if so stop that ssh process, in order to not try to +-- use an old stale connection. (atime would be nicer, but there's +-- a good chance a laptop uses noatime) +sshCachingParams :: HostName -> IO [CommandParam] +sshCachingParams hn = do + home <- myHomeDir + let cachedir = home ".ssh" "propellor" + createDirectoryIfMissing False cachedir + let socketfile = cachedir hn ++ ".sock" + let ps = + [ Param "-o", Param ("ControlPath=" ++ socketfile) + , Params "-o ControlMaster=auto -o ControlPersist=yes" + ] + + maybe noop (expireold ps socketfile) + =<< catchMaybeIO (getFileStatus socketfile) + + return ps + + where + expireold ps f s = do + now <- truncate <$> getPOSIXTime :: IO Integer + if modificationTime s > fromIntegral now - tenminutes + then touchFile f + else do + void $ boolSystem "ssh" $ + [ Params "-O stop" ] ++ ps ++ + [ Param "localhost" ] + nukeFile f + tenminutes = 600 From 66466a953d9094a7165c8f26225e20aab30369a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:44:24 -0400 Subject: [PATCH 21/45] reorg --- src/Propellor/CmdLine.hs | 15 --------------- src/Propellor/Message.hs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 3e24dd3..8c67f37 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -3,10 +3,6 @@ module Propellor.CmdLine where import System.Environment (getArgs) import Data.List import System.Exit -import System.Log.Logger -import System.Log.Formatter -import System.Log.Handler (setFormatter, LogHandler) -import System.Log.Handler.Simple import System.PosixCompat import Control.Exception (bracket) import System.Posix.IO @@ -343,14 +339,3 @@ gitPush hin hout = void $ fromstdin `concurrently` tostdout B.hPut toh b hFlush toh connect fromh toh - -checkDebugMode :: IO () -checkDebugMode = go =<< getEnv "PROPELLOR_DEBUG" - where - go (Just "1") = do - f <- setFormatter - <$> streamHandler stderr DEBUG - <*> pure (simpleLogFormatter "[$time] $msg") - updateGlobalLogger rootLoggerName $ - setLevel DEBUG . setHandlers [f] - go _ = noop diff --git a/src/Propellor/Message.hs b/src/Propellor/Message.hs index 639171c..a1e510a 100644 --- a/src/Propellor/Message.hs +++ b/src/Propellor/Message.hs @@ -5,6 +5,9 @@ module Propellor.Message where import System.Console.ANSI import System.IO import System.Log.Logger +import System.Log.Formatter +import System.Log.Handler (setFormatter, LogHandler) +import System.Log.Handler.Simple import "mtl" Control.Monad.Reader import Data.Maybe import Control.Applicative @@ -98,3 +101,14 @@ colorLine h intensity color msg = do -- | Causes a debug message to be displayed when PROPELLOR_DEBUG=1 debug :: [String] -> IO () debug = debugM "propellor" . unwords + +checkDebugMode :: IO () +checkDebugMode = go =<< getEnv "PROPELLOR_DEBUG" + where + go (Just "1") = do + f <- setFormatter + <$> streamHandler stderr DEBUG + <*> pure (simpleLogFormatter "[$time] $msg") + updateGlobalLogger rootLoggerName $ + setLevel DEBUG . setHandlers [f] + go _ = noop From 94369ef76baed1ef188d6d3030dee83f08690284 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:45:07 -0400 Subject: [PATCH 22/45] propellor spin From 80a20d6df36e789a452474fb6d1329583ebe95b2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:46:10 -0400 Subject: [PATCH 23/45] cleanup --- src/Propellor/CmdLine.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 8c67f37..ebe790e 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -245,7 +245,7 @@ spin hn hst = do hClose toh -- Display remaining output. void $ tryIO $ forever $ - showremote =<< hGetLine fromh + putStrLn =<< hGetLine fromh hClose fromh dispatch Nothing = return () @@ -271,8 +271,6 @@ spin hn hst = do runcmd = mkcmd [ "cd " ++ localdir ++ " && ./propellor --continue " ++ shellEscape (show (SimpleRun hn)) ] - showremote s = putStrLn s - sendprivdata toh privdata = void $ actionMessage ("Sending privdata (" ++ show (length privdata) ++ " bytes) to " ++ hn) $ do sendMarked toh privDataMarker privdata From 34cbe738b7e1fc38fa26eeb0cc9f203f5f429fae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:47:44 -0400 Subject: [PATCH 24/45] don't need to parse --gitpush; just use a Continue --- src/Propellor/CmdLine.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index ebe790e..8220cbe 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -54,7 +54,6 @@ processCmdLine = go =<< getArgs go ("--chain":h:[]) = return $ Chain h False go ("--chain":h:b:[]) = return $ Chain h (Prelude.read b) go ("--docker":h:[]) = return $ Docker h - go ("--gitpush":fin:fout:_) = return $ GitPush (Prelude.read fin) (Prelude.read fout) go (h:[]) | "--" `isPrefixOf` h = usage | otherwise = return $ Run h @@ -310,7 +309,7 @@ boot = do hout <- dup stdOutput hClose stdin hClose stdout - unlessM (boolSystem "git" [Param "pull", Param "--progress", Param "--upload-pack", Param $ "./propellor --gitpush " ++ show hin ++ " " ++ show hout, Param "."]) $ + unlessM (boolSystem "git" [Param "pull", Param "--progress", Param "--upload-pack", Param $ "./propellor --continue " ++ show (GitPush hin hout), Param "."]) $ errorMessage "git pull from client failed" -- Shim for git push over the propellor ssh channel. From 2bbb2aa6e2be7152f9532fb45a098b816a92217b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:49:32 -0400 Subject: [PATCH 25/45] remove --docker; use Continue And --chain was already not used. --- src/Propellor/CmdLine.hs | 3 --- src/Propellor/Property/Docker.hs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 8220cbe..df4d44a 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -51,9 +51,6 @@ processCmdLine = go =<< getArgs go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage "--continue serialization failure" - go ("--chain":h:[]) = return $ Chain h False - go ("--chain":h:b:[]) = return $ Chain h (Prelude.read b) - go ("--docker":h:[]) = return $ Docker h go (h:[]) | "--" `isPrefixOf` h = usage | otherwise = return $ Run h diff --git a/src/Propellor/Property/Docker.hs b/src/Propellor/Property/Docker.hs index d005592..491955d 100644 --- a/src/Propellor/Property/Docker.hs +++ b/src/Propellor/Property/Docker.hs @@ -385,7 +385,7 @@ runningContainer cid@(ContainerId hn cn) image runps = containerDesc cid $ prope liftIO $ writeFile (identFile cid) (show ident) ensureProperty $ boolProperty "run" $ runContainer img (runps ++ ["-i", "-d", "-t"]) - [shim, "--docker", fromContainerId cid] + [shim, "--continue", show (Docker (fromContainerId cid))] -- | Called when propellor is running inside a docker container. -- The string should be the container's ContainerId. From 0b58bc17100eab7f2f41e86a2d3e19796fc3e903 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:52:51 -0400 Subject: [PATCH 26/45] propellor spin From 7299cd73206c47b27a0a80ae9810ca4506942b11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:56:15 -0400 Subject: [PATCH 27/45] expand --- doc/centralized_git_repository.mdwn | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/centralized_git_repository.mdwn b/doc/centralized_git_repository.mdwn index f47aa92..46cf89e 100644 --- a/doc/centralized_git_repository.mdwn +++ b/doc/centralized_git_repository.mdwn @@ -4,7 +4,13 @@ directly to the host. This makes it easy to get started with propellor. A central git repository allows hosts to run propellor from cron and pick up any updates you may have pushed. This is useful when managing several -hosts with propellor. +hosts with propellor. + +The central repository does not need to be trusted; it can be hosted +anywhere, and propellor will only accept verified gpg signed git commits +from it. See [[security]] for details, but this means you can put it +on github without github being able to 0wn your propellor driven hosts, for +example. You can add a central git repository to your existing propellor setup easily: From 95fda710cb2c7637ab4b7cc437dfa4e1d1cef831 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 18:58:47 -0400 Subject: [PATCH 28/45] update --- doc/security.mdwn | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/security.mdwn b/doc/security.mdwn index bcbc28e..0bc4c6e 100644 --- a/doc/security.mdwn +++ b/doc/security.mdwn @@ -6,13 +6,13 @@ The only trusted machine is the laptop where you run `propellor --spin` to connect to a remote host. And that one only because you have a ssh key or login password to the host. -Since the hosts propellor deploys are not trusted by the central git -repository, they have to use git:// or http:// to pull from the central -git repository, rather than ssh://. +Since the hosts propellor deploys do not trust the central git repository, +and it doesn't trust them, it's normal to use git:// or http:// to pull +from the central git repository, rather than ssh://. -So, to avoid a MITM attack, propellor checks that any commit it fetches -from origin is gpg signed by a trusted gpg key, and refuses to deploy it -otherwise. +Since propellor doesn't trust the central git repository, it checks +that any commit it fetches from it is gpg signed by a trusted gpg key, +and refuses to deploy it otherwise. That is only done when privdata/keyring.gpg exists. To set it up: From 907ecfb464516cf30c2e54e63b17e4c79306f46e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:00:34 -0400 Subject: [PATCH 29/45] update --- doc/security.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/security.mdwn b/doc/security.mdwn index 0bc4c6e..12ae18d 100644 --- a/doc/security.mdwn +++ b/doc/security.mdwn @@ -21,8 +21,8 @@ That is only done when privdata/keyring.gpg exists. To set it up: In order to be secure from the beginning, when `propellor --spin` is used to bootstrap propellor on a new host, it transfers the local git repositry -to the remote host over ssh. After that, the remote host knows the -gpg key, and will use it to verify git fetches. +to the remote host over ssh. After that, the host knows the gpg key, and +will use it to verify git fetches. Since the propoellor git repository is public, you can't store in cleartext private data such as passwords, ssh private keys, etc. From 65d3f085dcd86c53549a3b126e0888f09c5e1925 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:33:11 -0400 Subject: [PATCH 30/45] remove what should be dead code While old propellor's can emit Ready, they won't if they've managed to updateFirst. If updateFirst fails due to eg, inaccessiable central repo, those old propellor's are not able to receive inline git pushes anyway, so are not going to update no matter what, so no point in making --spin work in that case. --- src/Propellor/CmdLine.hs | 10 ---------- src/Propellor/Protocol.hs | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index df4d44a..66da633 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -233,16 +233,6 @@ spin hn hst = do hClose fromh sendGitClone hn comm cacheparams privdata - -- Ready is only sent by old versions of - -- propellor. They expect to get privdata, - -- and then no more protocol communication. - dispatch (Just Ready) = do - sendprivdata toh privdata - hClose toh - -- Display remaining output. - void $ tryIO $ forever $ - putStrLn =<< hGetLine fromh - hClose fromh dispatch Nothing = return () user = "root@"++hn diff --git a/src/Propellor/Protocol.hs b/src/Propellor/Protocol.hs index 7249e2b..f8b706c 100644 --- a/src/Propellor/Protocol.hs +++ b/src/Propellor/Protocol.hs @@ -9,7 +9,7 @@ import Data.List import Propellor -data Stage = Ready | NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush +data Stage = NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush deriving (Read, Show, Eq) type Marker = String From c9fed0fdaa103e091fdee4ab4ab94dd921ce174a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:36:30 -0400 Subject: [PATCH 31/45] add --update, which will one day replace --boot But no time soon, since that would break --spin to old versions of propellor Maybe after 1 year? --- src/Propellor/CmdLine.hs | 16 ++++++++-------- src/Propellor/Types.hs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 66da633..6c3920c 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -41,7 +41,8 @@ processCmdLine = go =<< getArgs where go ("--help":_) = usage go ("--spin":h:[]) = return $ Spin h - go ("--boot":h:[]) = return $ Boot h + go ("--update":h:[]) = return $ Update h + go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--run":h:[]) = return $ Run h go ("--add-key":k:[]) = return $ AddKey k go ("--set":f:c:[]) = withprivfield f c Set @@ -94,9 +95,9 @@ defaultMain hostlist = do ( onlyProcess $ withhost hn mainProperties , go True (Spin hn) ) - go False (Boot _) = do + go False (Update _) = do forceConsole - onlyProcess boot + onlyProcess update withhost :: HostName -> (Host -> IO ()) -> IO () withhost hn a = maybe (unknownhost hn hostlist) a (findHost hostlist hn) @@ -282,11 +283,10 @@ sendGitClone hn = void $ actionMessage ("Cloning git repository to " ++ hn) $ do , "rm -f " ++ remotebundle ] --- Called "boot" for historical reasons, but what this really does is --- update the privdata, repo url, and git repo over the ssh connection from the --- client that ran propellor --spin. -boot :: IO () -boot = do +-- Update the privdata, repo url, and git repo over the ssh +-- connection from the client that ran propellor --spin. +update :: IO () +update = do req NeedRepoUrl repoUrlMarker setRepoUrl makePrivDataDir req NeedPrivData privDataMarker $ diff --git a/src/Propellor/Types.hs b/src/Propellor/Types.hs index e0a7113..a1d25b4 100644 --- a/src/Propellor/Types.hs +++ b/src/Propellor/Types.hs @@ -146,7 +146,7 @@ data CmdLine | AddKey String | Continue CmdLine | Chain HostName Bool - | Boot HostName + | Update HostName | Docker HostName | GitPush Fd Fd deriving (Read, Show, Eq) From 8b6531ea43e43bd979ad9b8125fc21c6602dea38 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:43:53 -0400 Subject: [PATCH 32/45] reorg --- src/Propellor/CmdLine.hs | 25 +++++-------------------- src/Propellor/Git.hs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 6c3920c..c85906d 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -148,29 +148,14 @@ updateFirst' cmdline next = do oldsha <- getCurrentGitSha1 branchref - whenM (doesFileExist keyring) $ do - {- To verify origin branch commit's signature, have to - - convince gpg to use our keyring. While running git log. - - Which has no way to pass options to gpg. - - Argh! -} - let gpgconf = privDataDir "gpg.conf" - writeFile gpgconf $ unlines - [ " keyring " ++ keyring - , "no-auto-check-trustdb" - ] - -- gpg is picky about perms - modifyFileMode privDataDir (removeModes otherGroupModes) - s <- readProcessEnv "git" ["log", "-n", "1", "--format=%G?", originbranch] - (Just [("GNUPGHOME", privDataDir)]) - nukeFile $ privDataDir "trustdb.gpg" - nukeFile $ privDataDir "pubring.gpg" - nukeFile $ privDataDir "gpg.conf" - if s == "U\n" || s == "G\n" - then do + whenM (doesFileExist keyring) $ + ifM (verifyOriginBranch originbranch) + ( do putStrLn $ "git branch " ++ originbranch ++ " gpg signature verified; merging" hFlush stdout void $ boolSystem "git" [Param "merge", Param originbranch] - else warningMessage $ "git branch " ++ originbranch ++ " is not signed with a trusted gpg key; refusing to deploy it! (Running with previous configuration instead.)" + , warningMessage $ "git branch " ++ originbranch ++ " is not signed with a trusted gpg key; refusing to deploy it! (Running with previous configuration instead.)" + ) newsha <- getCurrentGitSha1 branchref diff --git a/src/Propellor/Git.hs b/src/Propellor/Git.hs index 0de82f8..51ed3df 100644 --- a/src/Propellor/Git.hs +++ b/src/Propellor/Git.hs @@ -1,7 +1,10 @@ module Propellor.Git where import Propellor +import Propellor.PrivData.Paths +import Propellor.Gpg import Utility.SafeCommand +import Utility.FileMode getCurrentBranch :: IO String getCurrentBranch = takeWhile (/= '\n') @@ -39,3 +42,23 @@ hasOrigin = do rs <- lines <$> readProcess "git" ["remote"] return $ "origin" `elem` rs +{- To verify origin branch commit's signature, have to convince gpg + - to use our keyring. + - While running git log. Which has no way to pass options to gpg. + - Argh! + -} +verifyOriginBranch :: String -> IO Bool +verifyOriginBranch originbranch = do + let gpgconf = privDataDir "gpg.conf" + writeFile gpgconf $ unlines + [ " keyring " ++ keyring + , "no-auto-check-trustdb" + ] + -- gpg is picky about perms + modifyFileMode privDataDir (removeModes otherGroupModes) + s <- readProcessEnv "git" ["log", "-n", "1", "--format=%G?", originbranch] + (Just [("GNUPGHOME", privDataDir)]) + nukeFile $ privDataDir "trustdb.gpg" + nukeFile $ privDataDir "pubring.gpg" + nukeFile $ privDataDir "gpg.conf" + return (s == "U\n" || s == "G\n") From ce859eaa9a329c3dd10869b1f73ee1fdf154bed5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:48:02 -0400 Subject: [PATCH 33/45] fix long line --- src/Propellor/CmdLine.hs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index c85906d..cd491c6 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -281,8 +281,16 @@ update = do hout <- dup stdOutput hClose stdin hClose stdout - unlessM (boolSystem "git" [Param "pull", Param "--progress", Param "--upload-pack", Param $ "./propellor --continue " ++ show (GitPush hin hout), Param "."]) $ + unlessM (boolSystem "git" (pullparams hin hout)) $ errorMessage "git pull from client failed" + where + pullparams hin hout = + [ Param "pull" + , Param "--progress" + , Param "--upload-pack" + , Param $ "./propellor --continue " ++ show (GitPush hin hout) + , Param "." + ] -- Shim for git push over the propellor ssh channel. -- Reads from stdin and sends it to hout; From a0d5f41a6c3bb7ff69c78e014834c8ac92acca22 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 19:49:34 -0400 Subject: [PATCH 34/45] reorger to match usage --- src/Propellor/CmdLine.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index cd491c6..7c7bc65 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -39,16 +39,16 @@ usage = do processCmdLine :: IO CmdLine processCmdLine = go =<< getArgs where - go ("--help":_) = usage - go ("--spin":h:[]) = return $ Spin h - go ("--update":h:[]) = return $ Update h - go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--run":h:[]) = return $ Run h + go ("--spin":h:[]) = return $ Spin h go ("--add-key":k:[]) = return $ AddKey k go ("--set":f:c:[]) = withprivfield f c Set go ("--dump":f:c:[]) = withprivfield f c Dump go ("--edit":f:c:[]) = withprivfield f c Edit go ("--list-fields":[]) = return ListFields + go ("--help":_) = usage + go ("--update":h:[]) = return $ Update h + go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage "--continue serialization failure" From d4a4f0193e30aee1ed37ceab7a85760510ac0d1f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:19:10 -0400 Subject: [PATCH 35/45] refactor --- src/Propellor/CmdLine.hs | 91 +++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 7c7bc65..91bf2b6 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -181,46 +181,11 @@ spin hn hst = do boolSystem "git" [Param "push"] cacheparams <- toCommand <$> sshCachingParams hn - comm cacheparams =<< hostprivdata - unlessM (boolSystem "ssh" (map Param (cacheparams ++ ["-t", user, runcmd]))) $ + comm hn hst $ withBothHandles createProcessSuccess + (proc "ssh" $ cacheparams ++ [user, bootstrapcmd]) + unlessM (boolSystem "ssh" (map Param $ cacheparams ++ ["-t", user, runcmd])) $ error $ "remote propellor failed (running: " ++ runcmd ++")" where - hostprivdata = show . filterPrivData hst <$> decryptPrivData - - comm cacheparams privdata = - withBothHandles createProcessSuccess - (proc "ssh" $ cacheparams ++ [ user, bootstrapcmd]) - (comm' cacheparams privdata) - comm' cacheparams privdata (toh, fromh) = loop - where - loop = dispatch =<< (maybe Nothing readish <$> getMarked fromh statusMarker) - dispatch (Just NeedRepoUrl) = do - sendMarked toh repoUrlMarker - =<< (fromMaybe "" <$> getRepoUrl) - loop - dispatch (Just NeedPrivData) = do - sendprivdata toh privdata - loop - dispatch (Just NeedGitPush) = do - void $ actionMessage ("Sending git update to " ++ hn) $ do - sendMarked toh gitPushMarker "" - let p = (proc "git" ["upload-pack", "."]) - { std_in = UseHandle fromh - , std_out = UseHandle toh - } - (Nothing, Nothing, Nothing, h) <- createProcess p - r <- waitForProcess h - -- no more protocol possible after git push - hClose fromh - hClose toh - return (r == ExitSuccess) - dispatch (Just NeedGitClone) = do - hClose toh - hClose fromh - sendGitClone hn - comm cacheparams privdata - dispatch Nothing = return () - user = "root@"++hn mkcmd = shellWrap . intercalate " ; " @@ -243,10 +208,52 @@ spin hn hst = do runcmd = mkcmd [ "cd " ++ localdir ++ " && ./propellor --continue " ++ shellEscape (show (SimpleRun hn)) ] - sendprivdata toh privdata = void $ - actionMessage ("Sending privdata (" ++ show (length privdata) ++ " bytes) to " ++ hn) $ do - sendMarked toh privDataMarker privdata - return True +comm :: HostName -> Host -> (((Handle, Handle) -> IO ()) -> IO ()) -> IO () +comm hn hst connect = connect go + where + go (toh, fromh) = do + let loop = go (toh, fromh) + v <- (maybe Nothing readish <$> getMarked fromh statusMarker) + case v of + (Just NeedRepoUrl) -> do + sendRepoUrl toh + loop + (Just NeedPrivData) -> do + sendPrivData hn hst toh + loop + (Just NeedGitPush) -> do + sendGitUpdate hn fromh toh + -- no more protocol possible after git push + hClose fromh + hClose toh + (Just NeedGitClone) -> do + hClose toh + hClose fromh + sendGitClone hn + comm hn hst connect + Nothing -> return () + +sendRepoUrl :: Handle -> IO () +sendRepoUrl toh = sendMarked toh repoUrlMarker =<< (fromMaybe "" <$> getRepoUrl) + +sendPrivData :: HostName -> Host -> Handle -> IO () +sendPrivData hn hst toh = do + privdata <- show . filterPrivData hst <$> decryptPrivData + void $ actionMessage ("Sending privdata (" ++ show (length privdata) ++ " bytes) to " ++ hn) $ do + sendMarked toh privDataMarker privdata + return True + +sendGitUpdate :: HostName -> Handle -> Handle -> IO () +sendGitUpdate hn fromh toh = + void $ actionMessage ("Sending git update to " ++ hn) $ do + sendMarked toh gitPushMarker "" + (Nothing, Nothing, Nothing, h) <- createProcess p + (==) ExitSuccess <$> waitForProcess h + where + p = (proc "git" ["upload-pack", "."]) + { std_in = UseHandle fromh + , std_out = UseHandle toh + } -- Initial git clone, used for bootstrapping. sendGitClone :: HostName -> IO () From be1287d5f957528f71b7798d57bfedb7f30c5ced Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:19:50 -0400 Subject: [PATCH 36/45] refactor --- src/Propellor/CmdLine.hs | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 91bf2b6..3cb6715 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -208,6 +208,30 @@ spin hn hst = do runcmd = mkcmd [ "cd " ++ localdir ++ " && ./propellor --continue " ++ shellEscape (show (SimpleRun hn)) ] +-- Update the privdata, repo url, and git repo over the ssh +-- connection from the client that ran propellor --spin. +update :: IO () +update = do + req NeedRepoUrl repoUrlMarker setRepoUrl + makePrivDataDir + req NeedPrivData privDataMarker $ + writeFileProtected privDataLocal + req NeedGitPush gitPushMarker $ \_ -> do + hin <- dup stdInput + hout <- dup stdOutput + hClose stdin + hClose stdout + unlessM (boolSystem "git" (pullparams hin hout)) $ + errorMessage "git pull from client failed" + where + pullparams hin hout = + [ Param "pull" + , Param "--progress" + , Param "--upload-pack" + , Param $ "./propellor --continue " ++ show (GitPush hin hout) + , Param "." + ] + comm :: HostName -> Host -> (((Handle, Handle) -> IO ()) -> IO ()) -> IO () comm hn hst connect = connect go where @@ -275,30 +299,6 @@ sendGitClone hn = void $ actionMessage ("Cloning git repository to " ++ hn) $ do , "rm -f " ++ remotebundle ] --- Update the privdata, repo url, and git repo over the ssh --- connection from the client that ran propellor --spin. -update :: IO () -update = do - req NeedRepoUrl repoUrlMarker setRepoUrl - makePrivDataDir - req NeedPrivData privDataMarker $ - writeFileProtected privDataLocal - req NeedGitPush gitPushMarker $ \_ -> do - hin <- dup stdInput - hout <- dup stdOutput - hClose stdin - hClose stdout - unlessM (boolSystem "git" (pullparams hin hout)) $ - errorMessage "git pull from client failed" - where - pullparams hin hout = - [ Param "pull" - , Param "--progress" - , Param "--upload-pack" - , Param $ "./propellor --continue " ++ show (GitPush hin hout) - , Param "." - ] - -- Shim for git push over the propellor ssh channel. -- Reads from stdin and sends it to hout; -- reads from hin and sends it to stdout. From cdb5583e57c5fb79f0dd8343ad4d83df49274ac6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:20:02 -0400 Subject: [PATCH 37/45] propellor spin From 8686e1a5ba2ce3e2fbac5a967d07e12a05a31396 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:26:37 -0400 Subject: [PATCH 38/45] propellor spin From 04737b8975ad831f9f98b35956055800b265edff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:27:41 -0400 Subject: [PATCH 39/45] propellor spin From 2ad3334b26648157f7c7e2c584b3eaf99b9bce89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:28:16 -0400 Subject: [PATCH 40/45] propellor spin --- src/Propellor/CmdLine.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 3cb6715..ab6d861 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -51,7 +51,7 @@ processCmdLine = go =<< getArgs go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline - Nothing -> errorMessage "--continue serialization failure" + Nothing -> errorMessage $ "--continue serialization failure (" ++ s ++ ")" go (h:[]) | "--" `isPrefixOf` h = usage | otherwise = return $ Run h From cdad4fbe15dbc42524d591463731898fa3d73e43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:29:31 -0400 Subject: [PATCH 41/45] propellor spin --- src/Propellor/CmdLine.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index ab6d861..2f56c18 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -21,9 +21,9 @@ import qualified Propellor.Property.Docker.Shim as DockerShim import Utility.FileMode import Utility.SafeCommand -usage :: IO a -usage = do - putStrLn $ unlines +usage :: Handle -> IO a +usage h = do + hPutStrLn h $ unlines [ "Usage:" , " propellor" , " propellor hostname" @@ -46,21 +46,21 @@ processCmdLine = go =<< getArgs go ("--dump":f:c:[]) = withprivfield f c Dump go ("--edit":f:c:[]) = withprivfield f c Edit go ("--list-fields":[]) = return ListFields - go ("--help":_) = usage + go ("--help":_) = usage stdout go ("--update":h:[]) = return $ Update h go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage $ "--continue serialization failure (" ++ s ++ ")" go (h:[]) - | "--" `isPrefixOf` h = usage + | "--" `isPrefixOf` h = usage stderr | otherwise = return $ Run h go [] = do s <- takeWhile (/= '\n') <$> readProcess "hostname" ["-f"] if null s then errorMessage "Cannot determine hostname! Pass it on the command line." else return $ Run s - go _ = usage + go _ = usage stderr withprivfield s c f = case readish s of Just pf -> return $ f pf (Context c) From f9f60822ff56c439cc4d08a6259ed73176f850d2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:33:25 -0400 Subject: [PATCH 42/45] propellor spin --- src/Propellor/CmdLine.hs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 2f56c18..b3f7f70 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -21,20 +21,23 @@ import qualified Propellor.Property.Docker.Shim as DockerShim import Utility.FileMode import Utility.SafeCommand -usage :: Handle -> IO a -usage h = do - hPutStrLn h $ unlines - [ "Usage:" - , " propellor" - , " propellor hostname" - , " propellor --spin hostname" - , " propellor --add-key keyid" - , " propellor --set field context" - , " propellor --dump field context" - , " propellor --edit field context" - , " propellor --list-fields" - ] - exitFailure +usage :: Handle -> IO () +usage h = hPutStrLn h $ unlines + [ "Usage:" + , " propellor" + , " propellor hostname" + , " propellor --spin hostname" + , " propellor --add-key keyid" + , " propellor --set field context" + , " propellor --dump field context" + , " propellor --edit field context" + , " propellor --list-fields" + ] + +usageError :: [String] -> IO a +usageError ps = do + usage stderr + error ("(Unexpected: " ++ show ps) processCmdLine :: IO CmdLine processCmdLine = go =<< getArgs @@ -46,21 +49,23 @@ processCmdLine = go =<< getArgs go ("--dump":f:c:[]) = withprivfield f c Dump go ("--edit":f:c:[]) = withprivfield f c Edit go ("--list-fields":[]) = return ListFields - go ("--help":_) = usage stdout + go ("--help":_) = do + usage stdout + exitFailure go ("--update":h:[]) = return $ Update h go ("--boot":h:[]) = return $ Update h -- for back-compat go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage $ "--continue serialization failure (" ++ s ++ ")" go (h:[]) - | "--" `isPrefixOf` h = usage stderr + | "--" `isPrefixOf` h = usageError [h] | otherwise = return $ Run h go [] = do s <- takeWhile (/= '\n') <$> readProcess "hostname" ["-f"] if null s then errorMessage "Cannot determine hostname! Pass it on the command line." else return $ Run s - go _ = usage stderr + go v = usageError v withprivfield s c f = case readish s of Just pf -> return $ f pf (Context c) From 6e327fa1a3de1801714cad63dcb145d49bcfb008 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:38:11 -0400 Subject: [PATCH 43/45] propellor spin --- src/Propellor/CmdLine.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index b3f7f70..6552b12 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -57,6 +57,7 @@ processCmdLine = go =<< getArgs go ("--continue":s:[]) = case readish s of Just cmdline -> return $ Continue cmdline Nothing -> errorMessage $ "--continue serialization failure (" ++ s ++ ")" + go ("--gitpush":fin:fout:_) = return $ GitPush (Prelude.read fin) (Prelude.read fout) go (h:[]) | "--" `isPrefixOf` h = usageError [h] | otherwise = return $ Run h @@ -233,7 +234,7 @@ update = do [ Param "pull" , Param "--progress" , Param "--upload-pack" - , Param $ "./propellor --continue " ++ show (GitPush hin hout) + , Param $ "./propellor --gitpush " ++ show hin ++ " " ++ show hout , Param "." ] From c2ab73ba45bc55e3490b9d89bbca51b73154d497 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:38:30 -0400 Subject: [PATCH 44/45] propellor spin From b964b4836321832ad8d3be7268fd3af9ed8f5ea8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Nov 2014 20:41:41 -0400 Subject: [PATCH 45/45] tense --- src/Propellor/CmdLine.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 6552b12..ee56301 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -287,7 +287,7 @@ sendGitUpdate hn fromh toh = -- Initial git clone, used for bootstrapping. sendGitClone :: HostName -> IO () -sendGitClone hn = void $ actionMessage ("Cloning git repository to " ++ hn) $ do +sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do branch <- getCurrentBranch cacheparams <- sshCachingParams hn withTmpFile "propellor.git" $ \tmp _ -> allM id