From 403a82b8b7b223aad6d71641b5b928a4ad984034 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 22:26:39 -0400 Subject: [PATCH 01/27] email update --- propellor.1 | 2 +- propellor.cabal | 2 +- src/Utility/Applicative.hs | 2 +- src/Utility/Data.hs | 2 +- src/Utility/Directory.hs | 2 +- src/Utility/Env.hs | 2 +- src/Utility/Exception.hs | 2 +- src/Utility/FileMode.hs | 2 +- src/Utility/FileSystemEncoding.hs | 2 +- src/Utility/LinuxMkLibs.hs | 2 +- src/Utility/Misc.hs | 2 +- src/Utility/Monad.hs | 2 +- src/Utility/Path.hs | 2 +- src/Utility/PosixFiles.hs | 2 +- src/Utility/Process.hs | 2 +- src/Utility/QuickCheck.hs | 2 +- src/Utility/SafeCommand.hs | 2 +- src/Utility/Scheduled.hs | 2 +- src/Utility/Table.hs | 2 +- src/Utility/ThreadScheduler.hs | 2 +- src/Utility/Tmp.hs | 2 +- src/Utility/UserInfo.hs | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/propellor.1 b/propellor.1 index 3ee3bf4..27547f1 100644 --- a/propellor.1 +++ b/propellor.1 @@ -12,4 +12,4 @@ and configured in haskell. The first time you run propellor, it will set up a ~/.propellor/ repository. Edit ~/.propellor/config.hs to configure it. .SH AUTHOR -Joey Hess +Joey Hess diff --git a/propellor.cabal b/propellor.cabal index 645e5fa..9fe7a26 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -2,7 +2,7 @@ Name: propellor Version: 1.0.0 Cabal-Version: >= 1.6 License: BSD3 -Maintainer: Joey Hess +Maintainer: Joey Hess Author: Joey Hess Stability: Stable Copyright: 2014 Joey Hess diff --git a/src/Utility/Applicative.hs b/src/Utility/Applicative.hs index fd8944b..fce3c04 100644 --- a/src/Utility/Applicative.hs +++ b/src/Utility/Applicative.hs @@ -1,6 +1,6 @@ {- applicative stuff - - - Copyright 2012 Joey Hess + - Copyright 2012 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Data.hs b/src/Utility/Data.hs index 2df12b3..5ecd218 100644 --- a/src/Utility/Data.hs +++ b/src/Utility/Data.hs @@ -1,6 +1,6 @@ {- utilities for simple data types - - - Copyright 2013 Joey Hess + - Copyright 2013 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Directory.hs b/src/Utility/Directory.hs index d92327c..6b50016 100644 --- a/src/Utility/Directory.hs +++ b/src/Utility/Directory.hs @@ -1,6 +1,6 @@ {- directory manipulation - - - Copyright 2011-2014 Joey Hess + - Copyright 2011-2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Env.hs b/src/Utility/Env.hs index 6763c24..dd502fd 100644 --- a/src/Utility/Env.hs +++ b/src/Utility/Env.hs @@ -1,6 +1,6 @@ {- portable environment variables - - - Copyright 2013 Joey Hess + - Copyright 2013 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Exception.hs b/src/Utility/Exception.hs index 1fecf65..c6510db 100644 --- a/src/Utility/Exception.hs +++ b/src/Utility/Exception.hs @@ -1,6 +1,6 @@ {- Simple IO exception handling (and some more) - - - Copyright 2011-2012 Joey Hess + - Copyright 2011-2012 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/FileMode.hs b/src/Utility/FileMode.hs index c2ef683..82568f6 100644 --- a/src/Utility/FileMode.hs +++ b/src/Utility/FileMode.hs @@ -1,6 +1,6 @@ {- File mode utilities. - - - Copyright 2010-2012 Joey Hess + - Copyright 2010-2012 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/FileSystemEncoding.hs b/src/Utility/FileSystemEncoding.hs index b81fdc5..4e8f2ff 100644 --- a/src/Utility/FileSystemEncoding.hs +++ b/src/Utility/FileSystemEncoding.hs @@ -1,6 +1,6 @@ {- GHC File system encoding handling. - - - Copyright 2012-2014 Joey Hess + - Copyright 2012-2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/LinuxMkLibs.hs b/src/Utility/LinuxMkLibs.hs index 1dc4e1e..d32de1a 100644 --- a/src/Utility/LinuxMkLibs.hs +++ b/src/Utility/LinuxMkLibs.hs @@ -1,6 +1,6 @@ {- Linux library copier and binary shimmer - - - Copyright 2013 Joey Hess + - Copyright 2013 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Misc.hs b/src/Utility/Misc.hs index 949f41e..e4eccac 100644 --- a/src/Utility/Misc.hs +++ b/src/Utility/Misc.hs @@ -1,6 +1,6 @@ {- misc utility functions - - - Copyright 2010-2011 Joey Hess + - Copyright 2010-2011 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Monad.hs b/src/Utility/Monad.hs index eba3c42..878e0da 100644 --- a/src/Utility/Monad.hs +++ b/src/Utility/Monad.hs @@ -1,6 +1,6 @@ {- monadic stuff - - - Copyright 2010-2012 Joey Hess + - Copyright 2010-2012 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Path.hs b/src/Utility/Path.hs index 99c9438..ea62157 100644 --- a/src/Utility/Path.hs +++ b/src/Utility/Path.hs @@ -1,6 +1,6 @@ {- path manipulation - - - Copyright 2010-2014 Joey Hess + - Copyright 2010-2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/PosixFiles.hs b/src/Utility/PosixFiles.hs index 5abbb57..5a94ead 100644 --- a/src/Utility/PosixFiles.hs +++ b/src/Utility/PosixFiles.hs @@ -2,7 +2,7 @@ - - This is like System.PosixCompat.Files, except with a fixed rename. - - - Copyright 2014 Joey Hess + - Copyright 2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Process.hs b/src/Utility/Process.hs index 4550d94..3e01054 100644 --- a/src/Utility/Process.hs +++ b/src/Utility/Process.hs @@ -1,7 +1,7 @@ {- System.Process enhancements, including additional ways of running - processes, and logging. - - - Copyright 2012 Joey Hess + - Copyright 2012 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/QuickCheck.hs b/src/Utility/QuickCheck.hs index a498ee6..54200d3 100644 --- a/src/Utility/QuickCheck.hs +++ b/src/Utility/QuickCheck.hs @@ -1,6 +1,6 @@ {- QuickCheck with additional instances - - - Copyright 2012-2014 Joey Hess + - Copyright 2012-2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/SafeCommand.hs b/src/Utility/SafeCommand.hs index 86e60db..a555620 100644 --- a/src/Utility/SafeCommand.hs +++ b/src/Utility/SafeCommand.hs @@ -1,6 +1,6 @@ {- safely running shell commands - - - Copyright 2010-2013 Joey Hess + - Copyright 2010-2013 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Scheduled.hs b/src/Utility/Scheduled.hs index 305410c..3a1a6cd 100644 --- a/src/Utility/Scheduled.hs +++ b/src/Utility/Scheduled.hs @@ -1,6 +1,6 @@ {- scheduled activities - - - Copyright 2013-2014 Joey Hess + - Copyright 2013-2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/Table.hs b/src/Utility/Table.hs index 910038e..4e862ff 100644 --- a/src/Utility/Table.hs +++ b/src/Utility/Table.hs @@ -1,6 +1,6 @@ {- text based table generation - - - Copyright 2014 Joey Hess + - Copyright 2014 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/ThreadScheduler.hs b/src/Utility/ThreadScheduler.hs index fc026d7..eb00974 100644 --- a/src/Utility/ThreadScheduler.hs +++ b/src/Utility/ThreadScheduler.hs @@ -1,6 +1,6 @@ {- thread scheduling - - - Copyright 2012, 2013 Joey Hess + - Copyright 2012, 2013 Joey Hess - Copyright 2011 Bas van Dijk & Roel van Dijk - - License: BSD-2-clause diff --git a/src/Utility/Tmp.hs b/src/Utility/Tmp.hs index 0dc9f2c..d0cae33 100644 --- a/src/Utility/Tmp.hs +++ b/src/Utility/Tmp.hs @@ -1,6 +1,6 @@ {- Temporary files and directories. - - - Copyright 2010-2013 Joey Hess + - Copyright 2010-2013 Joey Hess - - License: BSD-2-clause -} diff --git a/src/Utility/UserInfo.hs b/src/Utility/UserInfo.hs index 617c3e9..e2c248b 100644 --- a/src/Utility/UserInfo.hs +++ b/src/Utility/UserInfo.hs @@ -1,6 +1,6 @@ {- user info - - - Copyright 2012 Joey Hess + - Copyright 2012 Joey Hess - - License: BSD-2-clause -} From 1b56e73c076b156497d04240b68d347b27c97c75 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 22:27:13 -0400 Subject: [PATCH 02/27] copyright update --- debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index 1d333cc..d007114 100644 --- a/debian/copyright +++ b/debian/copyright @@ -2,7 +2,7 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Source: native package Files: * -Copyright: © 2010-2014 Joey Hess +Copyright: © 2010-2014 Joey Hess and contributors License: BSD-2-clause License: BSD-2-clause From 4c5d5f34c8c0142d82ddcd2abd5df122d2403edd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:23:07 -0400 Subject: [PATCH 03/27] propellor spin --- config-joey.hs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/config-joey.hs b/config-joey.hs index 2971c1a..3555d83 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -101,6 +101,7 @@ orca = standardSystem "orca.kitenet.net" Unstable "amd64" & Apt.unattendedUpgrades & Postfix.satellite + & Systemd.persistentJournal & Docker.configured & Docker.docked (GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h") & Docker.docked (GitAnnexBuilder.standardAutoBuilderContainer dockerImage "i386" 45 "2h") @@ -125,7 +126,8 @@ kite = standardSystemUnhardened "kite.kitenet.net" Unstable "amd64" & Apt.installed ["linux-image-amd64"] & Linode.chainPVGrub 5 & Apt.unattendedUpgrades - & Apt.installed ["systemd"] + & Systemd.installed + & Systemd.persistentJournal & Ssh.hostKeys (Context "kitenet.net") & Ssh.passwordAuthentication True -- Since ssh password authentication is allowed: @@ -241,20 +243,25 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64" , "(Encrypt all data stored here.)" ] & ipv4 "193.234.225.114" - & Grub.chainPVGrub "hd0,0" "xen/xvda1" 30 + + & Grub.chainPVGrub "hd0,0" "xen/xvda1" 30 & Postfix.satellite & Apt.unattendedUpgrades + & Systemd.installed + & Systemd.persistentJournal & Ssh.hostKeys ctx & sshPubKey "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAJkoPRhUGT8EId6m37uBdYEtq42VNwslKnc9mmO+89ody066q6seHKeFY6ImfwjcyIjM30RTzEwftuVNQnbEB0=" & Ssh.keyImported SshRsa "joey" ctx & Apt.serviceInstalledRunning "swapspace" - & alias "eubackup.kitenet.net" + + & alias "eubackup.kitenet.net" & Apt.installed ["obnam", "sshfs", "rsync"] & JoeySites.obnamRepos ["wren", "pell", "kite"] & JoeySites.githubBackup & JoeySites.rsyncNetBackup hosts & JoeySites.backupsBackedupTo hosts "usbackup.kitenet.net" "lib/backup/eubackup" - & alias "podcatcher.kitenet.net" + + & alias "podcatcher.kitenet.net" & JoeySites.podcatcher & alias "znc.kitenet.net" @@ -262,7 +269,8 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64" -- I'd rather this were on diatom, but it needs unstable. & alias "kgb.kitenet.net" & JoeySites.kgbServer - & alias "mumble.kitenet.net" + + & alias "mumble.kitenet.net" & JoeySites.mumbleServer hosts & alias "ns3.kitenet.net" From d8c632d7a567eb0fecec094acf66913ac2ff78d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:24:38 -0400 Subject: [PATCH 04/27] propellor spin From 40bfcacf4e4f019153c1679cd3955cb3bb2f047f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:26:13 -0400 Subject: [PATCH 05/27] more quiet --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43d7d05..61d3242 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,8 @@ deps: @if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install libghc-async-dev || (cabal update; cabal install async); fi || true dist/setup-config: propellor.cabal - if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi - $(CABAL) configure + @if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi + @$(CABAL) configure install: install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor From 0a415a7b41f871a514575d0ac2475374f49db70a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:30:01 -0400 Subject: [PATCH 06/27] show last line if it cannot be parsed as a serialized Result --- src/Propellor/Engine.hs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Propellor/Engine.hs b/src/Propellor/Engine.hs index 969769c..0a51736 100644 --- a/src/Propellor/Engine.hs +++ b/src/Propellor/Engine.hs @@ -11,7 +11,6 @@ import "mtl" Control.Monad.Reader import Control.Exception (bracket) import System.PosixCompat import System.Posix.IO -import Data.Maybe import Propellor.Types import Propellor.Message @@ -74,8 +73,14 @@ processChainOutput h = go Nothing go lastline = do v <- catchMaybeIO (hGetLine h) case v of - Nothing -> pure $ fromMaybe FailedChange $ - readish =<< lastline + Nothing -> case lastline of + Nothing -> pure FailedChange + Just l -> case readish l of + Just r -> pure r + Nothing -> do + putStrLn l + hFlush stdout + return FailedChange Just s -> do maybe noop (\l -> unless (null l) (putStrLn l)) lastline hFlush stdout From 0c39e3137d185fb24e26ed60c112ed72811e002e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:30:55 -0400 Subject: [PATCH 07/27] propellor spin From 6eb64d56e3718339b209853b838d04606dcd2dbb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:33:39 -0400 Subject: [PATCH 08/27] avoid setting title when not at console --- src/Propellor/Engine.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Propellor/Engine.hs b/src/Propellor/Engine.hs index 0a51736..b551ca0 100644 --- a/src/Propellor/Engine.hs +++ b/src/Propellor/Engine.hs @@ -27,7 +27,9 @@ mainProperties :: Host -> IO () mainProperties host = do r <- runPropellor host $ ensureProperties [Property "overall" (ensureProperties $ hostProperties host) mempty] - setTitle "propellor: done" + h <- mkMessageHandle + whenConsole h $ + setTitle "propellor: done" hFlush stdout case r of FailedChange -> exitWith (ExitFailure 1) From aeb35683879acad42a309ad26d82dce571286fe2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:34:45 -0400 Subject: [PATCH 09/27] propellor spin From d5cf4db6e1ae1c8bb5235ab274e787130bb24041 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 21 Nov 2014 23:36:35 -0400 Subject: [PATCH 10/27] propellor spin From f62d2fb18389947ce11021ba80b2aee52c6d03c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:22:19 -0400 Subject: [PATCH 11/27] propellor --spin can now deploy propellor to hosts that do not have git, ghc, or apt-get. This is accomplished by uploading a fairly portable precompiled tarball of propellor. --- debian/changelog | 8 ++++++++ src/Propellor/CmdLine.hs | 27 +++++++++++++++------------ src/Propellor/Git.hs | 2 +- src/Propellor/Protocol.hs | 2 +- src/Propellor/Server.hs | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/debian/changelog b/debian/changelog index 66e5cae..784dcdb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +propellor (1.0.1) UNRELEASED; urgency=medium + + * propellor --spin can now deploy propellor to hosts that do not have + git, ghc, or apt-get. This is accomplished by uploading a fairly + portable precompiled tarball of propellor. + + -- Joey Hess Sat, 22 Nov 2014 00:12:35 -0400 + propellor (1.0.0) unstable; urgency=medium * propellor --spin can now be used to update remote hosts, without diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 142efa1..ec2ca7e 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -114,16 +114,19 @@ unknownhost h hosts = errorMessage $ unlines ] buildFirst :: CmdLine -> IO () -> IO () -buildFirst cmdline next = do - oldtime <- getmtime - ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"]) - ( do - newtime <- getmtime - if newtime == oldtime - then next - else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)] - , errorMessage "Propellor build failed!" - ) +buildFirst cmdline next = ifM (doesFileExist "Makefile") + ( do + oldtime <- getmtime + ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"]) + ( do + newtime <- getmtime + if newtime == oldtime + then next + else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)] + , errorMessage "Propellor build failed!" + ) + , next + ) where getmtime = catchMaybeIO $ getModificationTime "propellor" @@ -172,11 +175,11 @@ spin hn hst = do updatecmd = mkcmd [ "if [ ! -d " ++ localdir ++ " ]" - , "then " ++ intercalate " && " + , "then (" ++ intercalate " && " [ "apt-get update" , "apt-get --no-install-recommends --no-upgrade -y install git make" , "echo " ++ toMarked statusMarker (show NeedGitClone) - ] + ] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled) , "else " ++ intercalate " && " [ "cd " ++ localdir , "if ! test -x ./propellor; then make deps build; fi" diff --git a/src/Propellor/Git.hs b/src/Propellor/Git.hs index 73de1de..e5f464c 100644 --- a/src/Propellor/Git.hs +++ b/src/Propellor/Git.hs @@ -38,7 +38,7 @@ getRepoUrl = getM get urls _ -> Nothing hasOrigin :: IO Bool -hasOrigin = do +hasOrigin = catchDefaultIO False $ do rs <- lines <$> readProcess "git" ["remote"] return $ "origin" `elem` rs diff --git a/src/Propellor/Protocol.hs b/src/Propellor/Protocol.hs index 68c2443..95a671b 100644 --- a/src/Propellor/Protocol.hs +++ b/src/Propellor/Protocol.hs @@ -13,7 +13,7 @@ import Data.List import Propellor -data Stage = NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush +data Stage = NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush | NeedPrecompiled deriving (Read, Show, Eq) type Marker = String diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 513a81f..786d121 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -16,6 +16,7 @@ import Propellor.Protocol import Propellor.PrivData.Paths import Propellor.Git import Propellor.Ssh +import qualified Propellor.Shim as Shim import Utility.FileMode import Utility.SafeCommand @@ -69,6 +70,11 @@ updateServer hn hst connect = connect go hClose fromh sendGitClone hn updateServer hn hst connect + (Just NeedPrecompiled) -> do + hClose toh + hClose fromh + sendPrecompiled hn + updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () @@ -113,6 +119,32 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do , "rm -f " ++ remotebundle ] +-- Send a tarball containing the precompiled propellor, and libraries. +-- This should be reasonably portable, as long as the remote host has the +-- same architecture as the build host. +sendPrecompiled :: HostName -> IO () +sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort " ++ hn) $ do + cacheparams <- sshCachingParams hn + withTmpDir "propellor" $ \tmpdir -> + bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do + changeWorkingDirectory tmpdir + let shimdir = "propellor" + let me = localdir "propellor" + void $ Shim.setup me shimdir + withTmpFile "propellor.tar" $ \tarball -> allM id + [ boolSystem "strip" [File me] + , boolSystem "tar" [Param "cf", File tmp, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball) + , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] + ] + where + remotetarball = "/usr/local/propellor.tar" + unpackcmd = shellSwap $ intercalate " && " + [ "cd " ++ takeDirectory remotetarball + , "tar xf " ++ remotetarball + , "rm -f " ++ remotetarball + ] + -- Shim for git push over the propellor ssh channel. -- Reads from stdin and sends it to hout; -- reads from hin and sends it to stdout. From 7fd338f96f5852ea309bd4a2964bf93d06062086 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:25:00 -0400 Subject: [PATCH 12/27] propellor spin --- src/Propellor/CmdLine.hs | 2 +- src/Propellor/Server.hs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index ec2ca7e..99826c3 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -176,7 +176,7 @@ spin hn hst = do updatecmd = mkcmd [ "if [ ! -d " ++ localdir ++ " ]" , "then (" ++ intercalate " && " - [ "apt-get update" + [ "apt-get-fail update" , "apt-get --no-install-recommends --no-upgrade -y install git make" , "echo " ++ toMarked statusMarker (show NeedGitClone) ] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 786d121..a8cd6a0 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -8,7 +8,9 @@ import Data.List import System.Exit import System.PosixCompat import System.Posix.IO +import System.Posix.Directory import Control.Concurrent.Async +import Control.Exception (bracket) import qualified Data.ByteString as B import Propellor @@ -131,15 +133,15 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor let shimdir = "propellor" let me = localdir "propellor" void $ Shim.setup me shimdir - withTmpFile "propellor.tar" $ \tarball -> allM id + withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tmp, File shimdir] - , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball) + , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] ] where remotetarball = "/usr/local/propellor.tar" - unpackcmd = shellSwap $ intercalate " && " + unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball , "tar xf " ++ remotetarball , "rm -f " ++ remotetarball From 99e0a5fea733fbeb18ee8fa4556994a50e470b7b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:27:35 -0400 Subject: [PATCH 13/27] propellor spin --- src/Propellor/Server.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index a8cd6a0..eff5699 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -131,7 +131,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do changeWorkingDirectory tmpdir let shimdir = "propellor" - let me = localdir "propellor" + me <- readSymbolicLink "/proc/self/exe" void $ Shim.setup me shimdir withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] From e5135c19489ee0799f9408c2336ebf98c6ddadf3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:32:04 -0400 Subject: [PATCH 14/27] propellor spin --- src/Propellor/Server.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index eff5699..d6987d2 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -76,7 +76,6 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn - updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () @@ -125,14 +124,16 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do -- This should be reasonably portable, as long as the remote host has the -- same architecture as the build host. sendPrecompiled :: HostName -> IO () -sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort " ++ hn) $ do +sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort") $ do cacheparams <- sshCachingParams hn withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do changeWorkingDirectory tmpdir let shimdir = "propellor" me <- readSymbolicLink "/proc/self/exe" - void $ Shim.setup me shimdir + shim <- Shim.setup me shimdir + when (shim /= shimdir "propellor") $ + renameFile shim (shimdir "propellor") withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] From d3b7c8585e2c290a3e005f17411827c8836bd225 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:32:21 -0400 Subject: [PATCH 15/27] propellor spin From f1fdd6f678d2d5d753407434a53cc6e2876185ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:34:34 -0400 Subject: [PATCH 16/27] propellor spin --- src/Propellor/Server.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index d6987d2..a72c83a 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -128,13 +128,14 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor cacheparams <- sshCachingParams hn withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do - changeWorkingDirectory tmpdir let shimdir = "propellor" + changeWorkingDirectory shimdir me <- readSymbolicLink "/proc/self/exe" - shim <- Shim.setup me shimdir + shim <- Shim.setup me "." + changeWorkingDirectory tmpdir when (shim /= shimdir "propellor") $ renameFile shim (shimdir "propellor") - withTmpFile "propellor.tar" $ \tarball _ -> allM id + withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] From 1af398d33eee40a13d7c310e6f97b170b47d21ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:35:14 -0400 Subject: [PATCH 17/27] propellor spin --- src/Propellor/Server.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index a72c83a..fb37ff1 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -129,6 +129,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do let shimdir = "propellor" + createDirectoryIfMissing True shimdir changeWorkingDirectory shimdir me <- readSymbolicLink "/proc/self/exe" shim <- Shim.setup me "." From 083b82d8100ef1915a00f8fbcd1530b85448dc9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:36:16 -0400 Subject: [PATCH 18/27] propellor spin --- src/Propellor/Server.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index fb37ff1..fde9c2e 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -129,13 +129,13 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do let shimdir = "propellor" - createDirectoryIfMissing True shimdir - changeWorkingDirectory shimdir + createDirectoryIfMissing True (tmpdir shimdir) + changeWorkingDirectory (tmpdir shimdir) me <- readSymbolicLink "/proc/self/exe" shim <- Shim.setup me "." + when (shim /= "propellor") $ + renameFile shim "propellor" changeWorkingDirectory tmpdir - when (shim /= shimdir "propellor") $ - renameFile shim (shimdir "propellor") withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] From 0d65fdeb1dcb414b0fb1d5719dd734a1b04da1e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:44:13 -0400 Subject: [PATCH 19/27] propellor spin --- src/Propellor/Server.hs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index fde9c2e..06fede7 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -27,17 +27,19 @@ import Utility.SafeCommand -- running the updateServer update :: IO () update = do - req NeedRepoUrl repoUrlMarker setRepoUrl + whenM hasOrigin $ + req NeedRepoUrl repoUrlMarker setRepoUrl makePrivDataDir req NeedPrivData privDataMarker $ writeFileProtected privDataLocal - req NeedGitPush gitPushMarker $ \_ -> do - hin <- dup stdInput - hout <- dup stdOutput - hClose stdin - hClose stdout - unlessM (boolSystem "git" (pullparams hin hout)) $ - errorMessage "git pull from client failed" + whenM hasOrigin $ + req NeedGitPush gitPushMarker $ \_ -> do + hin <- dup stdInput + hout <- dup stdOutput + hClose stdin + hClose stdout + unlessM (boolSystem "git" (pullparams hin hout)) $ + errorMessage "git pull from client failed" where pullparams hin hout = [ Param "pull" @@ -76,6 +78,7 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn + loop Nothing -> return () sendRepoUrl :: Handle -> IO () From 5d6e1a27a5a39f05b254a7fc351a2dc34ef4620c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:46:07 -0400 Subject: [PATCH 20/27] propellor spin --- src/Propellor/Server.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 06fede7..eaf9649 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -78,7 +78,7 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn - loop + updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () From aef324f52883b6bbf7fb400ece41af19b85cf2b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:46:27 -0400 Subject: [PATCH 21/27] propellor spin From ea058a5f9f4bf20d82f5d5d0fbbdbf74ce3cbfcc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:47:26 -0400 Subject: [PATCH 22/27] remove test hack --- src/Propellor/CmdLine.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index 99826c3..ec2ca7e 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -176,7 +176,7 @@ spin hn hst = do updatecmd = mkcmd [ "if [ ! -d " ++ localdir ++ " ]" , "then (" ++ intercalate " && " - [ "apt-get-fail update" + [ "apt-get update" , "apt-get --no-install-recommends --no-upgrade -y install git make" , "echo " ++ toMarked statusMarker (show NeedGitClone) ] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled) From cd6ca049b8ba90bdb4a1ba6ebf258fc68809049a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:50:56 -0400 Subject: [PATCH 23/27] cleanup --- src/Propellor/Server.hs | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index eaf9649..001b476 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -128,25 +128,28 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do -- same architecture as the build host. sendPrecompiled :: HostName -> IO () sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort") $ do - cacheparams <- sshCachingParams hn - withTmpDir "propellor" $ \tmpdir -> - bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do - let shimdir = "propellor" - createDirectoryIfMissing True (tmpdir shimdir) - changeWorkingDirectory (tmpdir shimdir) - me <- readSymbolicLink "/proc/self/exe" - shim <- Shim.setup me "." - when (shim /= "propellor") $ - renameFile shim "propellor" - changeWorkingDirectory tmpdir - withTmpFile "propellor.tar." $ \tarball _ -> allM id - [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tarball, File shimdir] - , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] - , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] - ] + bracket getWorkingDirectory changeWorkingDirectory $ \_ -> + withTmpDir "propellor" go where + go tmpdir = do + cacheparams <- sshCachingParams hn + let shimdir = takeFileName localdir + createDirectoryIfMissing True (tmpdir shimdir) + changeWorkingDirectory (tmpdir shimdir) + me <- readSymbolicLink "/proc/self/exe" + shim <- Shim.setup me "." + when (shim /= "propellor") $ + renameFile shim "propellor" + changeWorkingDirectory tmpdir + withTmpFile "propellor.tar." $ \tarball _ -> allM id + [ boolSystem "strip" [File me] + , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] + , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] + ] + remotetarball = "/usr/local/propellor.tar" + unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball , "tar xf " ++ remotetarball From c0662e6258e5fc8f5755f01de7d7184c532c6431 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:52:59 -0400 Subject: [PATCH 24/27] comment --- src/Propellor/Server.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 001b476..743490a 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -1,3 +1,7 @@ +-- When propellor --spin is running, the local host acts as a server, +-- which connects to the remote host's propellor and responds to its +-- requests. + module Propellor.Server ( update, updateServer, From 128e16472a03929b089dcf44e28c651edcefcd43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 01:13:53 -0400 Subject: [PATCH 25/27] compress tarball 11 mb -> 4 mb --- src/Propellor/Server.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 743490a..19a2c90 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -147,7 +147,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor changeWorkingDirectory tmpdir withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "tar" [Param "czf", File tarball, File shimdir] , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] ] @@ -156,7 +156,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball - , "tar xf " ++ remotetarball + , "tar xzf " ++ remotetarball , "rm -f " ++ remotetarball ] From e0c2d6a73af01f138a50b80a6885718ed0b9dad1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 02:04:01 -0400 Subject: [PATCH 26/27] comment --- src/Propellor/Shim.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Propellor/Shim.hs b/src/Propellor/Shim.hs index 5b5aa68..1bfbb0c 100644 --- a/src/Propellor/Shim.hs +++ b/src/Propellor/Shim.hs @@ -45,6 +45,8 @@ setup propellorbin dest = do modifyFileMode shim (addModes executeModes) return shim +-- Called when the shimmed propellor is running, so that commands it runs +-- don't see it. cleanEnv :: IO () cleanEnv = void $ unsetEnv "GCONV_PATH" From 57ec60d6f307dbf3e237b924e635b90ba889af18 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 12:09:53 -0400 Subject: [PATCH 27/27] mdwn man page and more complete docs --- Makefile | 7 +++++-- debian/changelog | 2 ++ doc/documentation.mdwn | 1 + doc/mdwn2man | 44 ++++++++++++++++++++++++++++++++++++++++++ propellor.1 | 15 -------------- 5 files changed, 52 insertions(+), 17 deletions(-) create mode 100755 doc/mdwn2man delete mode 100644 propellor.1 diff --git a/Makefile b/Makefile index 61d3242..459fae3 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dist/setup-config: propellor.cabal @if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi @$(CABAL) configure -install: +install: propellor.1 install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor install -s dist/build/propellor/propellor $(DESTDIR)/usr/bin/propellor mkdir -p dist/gittmp @@ -34,8 +34,11 @@ install: && git show-ref master --hash > $(DESTDIR)/usr/src/propellor/head rm -rf dist/gittmp +propellor.1: doc/usage.mdwn doc/mdwn2man + doc/mdwn2man propellor 1 < doc/usage.mdwn > propellor.1 + clean: - rm -rf dist Setup tags propellor privdata/local + rm -rf dist Setup tags propellor propellor.1 privdata/local find -name \*.o -exec rm {} \; find -name \*.hi -exec rm {} \; diff --git a/debian/changelog b/debian/changelog index 784dcdb..a44d72a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ propellor (1.0.1) UNRELEASED; urgency=medium * propellor --spin can now deploy propellor to hosts that do not have git, ghc, or apt-get. This is accomplished by uploading a fairly portable precompiled tarball of propellor. + * --spin host --via host causes propellor to bounce through an intermediate + host, which handles any necessary provisioning of the host being spun. -- Joey Hess Sat, 22 Nov 2014 00:12:35 -0400 diff --git a/doc/documentation.mdwn b/doc/documentation.mdwn index 79e242f..9028dfa 100644 --- a/doc/documentation.mdwn +++ b/doc/documentation.mdwn @@ -3,6 +3,7 @@ Propellor's modules is the most important docuemntation of propellor. Other documentation: +* [[man page|usage]] * [[Haskell Newbie]] * [[Centralized Git Repository]] * [[Contributing]] diff --git a/doc/mdwn2man b/doc/mdwn2man new file mode 100755 index 0000000..aadb13c --- /dev/null +++ b/doc/mdwn2man @@ -0,0 +1,44 @@ +#!/usr/bin/env perl +# Warning: hack + +my $prog=shift; +my $section=shift; + +print ".TH $prog $section\n"; + +while (<>) { + s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg; + s/\`([^\`]*)\`/\\fB$1\\fP/g; + s/\`//g; + s/^\s*\./\\&./g; + if (/^#\s/) { + s/^#\s/.SH /; + <>; # blank; + } + s/^[ \n]+//; + s/^\t/ /; + s/-/\\-/g; + s/^Warning:.*//g; + s/^$/.PP\n/; + s/^\*\s+(.*)/.IP "$1"/; + next if $_ eq ".PP\n" && $skippara; + if (/^.IP /) { + $inlist=1; + $spippara=0; + } + elsif (/^.SH/) { + $skippara=0; + $inlist=0; + } + elsif (/^\./) { + $skippara=1; + } + else { + $skippara=0; + } + if ($inlist && $_ eq ".PP\n") { + $_=".IP\n"; + } + + print $_; +} diff --git a/propellor.1 b/propellor.1 deleted file mode 100644 index 27547f1..0000000 --- a/propellor.1 +++ /dev/null @@ -1,15 +0,0 @@ -.\" -*- nroff -*- -.TH propellor 1 "Commands" -.SH NAME -propellor \- property-based host configuration management in haskell -.SH SYNOPSIS -.B propellor [options] host -.SH DESCRIPTION -.I propellor -is a property-based host configuration management program written -and configured in haskell. -.PP -The first time you run propellor, it will set up a ~/.propellor/ -repository. Edit ~/.propellor/config.hs to configure it. -.SH AUTHOR -Joey Hess