Merge branch 'joeyconfig'

Makefile

@@ -16,10 +16,10 @@ deps:
 	@if [ $$(whoami) = root ]; then apt-get --no-upgrade --no-install-recommends -y install libghc-async-dev || (cabal update; cabal install async); fi || true
 
 dist/setup-config: propellor.cabal
-	if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
+	@if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
-	$(CABAL) configure
+	@$(CABAL) configure
 
-install:
+install: propellor.1
 	install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor
 	install -s dist/build/propellor/propellor $(DESTDIR)/usr/bin/propellor
 	mkdir -p dist/gittmp
@@ -34,8 +34,11 @@ install:
 		&& git show-ref master --hash > $(DESTDIR)/usr/src/propellor/head
 	rm -rf dist/gittmp
 
+propellor.1: doc/usage.mdwn doc/mdwn2man
+	doc/mdwn2man propellor 1 < doc/usage.mdwn > propellor.1
+
 clean:
-	rm -rf dist Setup tags propellor privdata/local
+	rm -rf dist Setup tags propellor propellor.1 privdata/local
 	find -name \*.o -exec rm {} \;
 	find -name \*.hi -exec rm {} \;
 

config-joey.hs

@@ -101,6 +101,7 @@ orca = standardSystem "orca.kitenet.net" Unstable "amd64"
 
 	& Apt.unattendedUpgrades
 	& Postfix.satellite
+	& Systemd.persistentJournal
 	& Docker.configured
 	& Docker.docked (GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h")
 	& Docker.docked (GitAnnexBuilder.standardAutoBuilderContainer dockerImage "i386" 45 "2h")
@@ -125,7 +126,8 @@ kite = standardSystemUnhardened "kite.kitenet.net" Unstable "amd64"
 	& Apt.installed ["linux-image-amd64"]
 	& Linode.chainPVGrub 5
 	& Apt.unattendedUpgrades
-	& Apt.installed ["systemd"]
+	& Systemd.installed
+	& Systemd.persistentJournal
 	& Ssh.hostKeys (Context "kitenet.net")
 	& Ssh.passwordAuthentication True
 	-- Since ssh password authentication is allowed:
@@ -241,20 +243,25 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
 	, "(Encrypt all data stored here.)"
 	]
 	& ipv4 "193.234.225.114"
-		& Grub.chainPVGrub "hd0,0" "xen/xvda1" 30
+
+	& Grub.chainPVGrub "hd0,0" "xen/xvda1" 30
 	& Postfix.satellite
 	& Apt.unattendedUpgrades
+	& Systemd.installed
+	& Systemd.persistentJournal
 	& Ssh.hostKeys ctx
 	& sshPubKey "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAJkoPRhUGT8EId6m37uBdYEtq42VNwslKnc9mmO+89ody066q6seHKeFY6ImfwjcyIjM30RTzEwftuVNQnbEB0="
 	& Ssh.keyImported SshRsa "joey" ctx
 	& Apt.serviceInstalledRunning "swapspace"
-		& alias "eubackup.kitenet.net"
+
+	& alias "eubackup.kitenet.net"
 	& Apt.installed ["obnam", "sshfs", "rsync"]
 	& JoeySites.obnamRepos ["wren", "pell", "kite"]
 	& JoeySites.githubBackup
 	& JoeySites.rsyncNetBackup hosts
 	& JoeySites.backupsBackedupTo hosts "usbackup.kitenet.net" "lib/backup/eubackup"
-		& alias "podcatcher.kitenet.net"
+
+	& alias "podcatcher.kitenet.net"
 	& JoeySites.podcatcher
 	
 	& alias "znc.kitenet.net"
@@ -262,7 +269,8 @@ elephant = standardSystem "elephant.kitenet.net" Unstable "amd64"
 	-- I'd rather this were on diatom, but it needs unstable.
 	& alias "kgb.kitenet.net"
 	& JoeySites.kgbServer
-		& alias "mumble.kitenet.net"
+	
+	& alias "mumble.kitenet.net"
 	& JoeySites.mumbleServer hosts
 	
 	& alias "ns3.kitenet.net"

debian/changelog

@@ -1,3 +1,13 @@
+propellor (1.0.1) UNRELEASED; urgency=medium
+
+  * propellor --spin can now deploy propellor to hosts that do not have 
+    git, ghc, or apt-get. This is accomplished by uploading a fairly
+    portable precompiled tarball of propellor.
+  * --spin host --via host causes propellor to bounce through an intermediate
+    host, which handles any necessary provisioning of the host being spun.
+
+ -- Joey Hess <joeyh@debian.org>  Sat, 22 Nov 2014 00:12:35 -0400
+
 propellor (1.0.0) unstable; urgency=medium
 
   * propellor --spin can now be used to update remote hosts, without

debian/copyright

@@ -2,7 +2,7 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Source: native package
 
 Files: *
-Copyright: © 2010-2014 Joey Hess <id@joeyh.name>
+Copyright: © 2010-2014 Joey Hess <id@joeyh.name> and contributors
 License: BSD-2-clause
 
 License: BSD-2-clause

doc/documentation.mdwn

@@ -3,6 +3,7 @@ Propellor's modules is the most important docuemntation of propellor.
 
 Other documentation:
 
+* [[man page|usage]]
 * [[Haskell Newbie]]
 * [[Centralized Git Repository]]
 * [[Contributing]]

doc/mdwn2man

@@ -0,0 +1,44 @@
+#!/usr/bin/env perl
+# Warning: hack
+
+my $prog=shift;
+my $section=shift;
+
+print ".TH $prog $section\n";
+
+while (<>) {
+	s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg;
+	s/\`([^\`]*)\`/\\fB$1\\fP/g;
+	s/\`//g;
+	s/^\s*\./\\&./g;
+	if (/^#\s/) {
+		s/^#\s/.SH /;
+		<>; # blank;
+	}
+	s/^[ \n]+//;
+	s/^\t/ /;
+	s/-/\\-/g;
+	s/^Warning:.*//g;
+	s/^$/.PP\n/;
+	s/^\*\s+(.*)/.IP "$1"/;
+	next if $_ eq ".PP\n" && $skippara;
+	if (/^.IP /) {
+		$inlist=1;
+		$spippara=0;
+	}
+	elsif (/^.SH/) {
+		$skippara=0;
+		$inlist=0;
+	}
+	elsif (/^\./) {
+		$skippara=1;
+	}
+	else {
+		$skippara=0;
+	}
+	if ($inlist && $_ eq ".PP\n") {
+		$_=".IP\n";
+	}
+
+	print $_;
+}

propellor.1

@@ -1,15 +0,0 @@
-.\" -*- nroff -*-
-.TH propellor 1 "Commands"
-.SH NAME
-propellor \- property-based host configuration management in haskell
-.SH SYNOPSIS
-.B propellor [options] host
-.SH DESCRIPTION
-.I propellor
-is a property-based host configuration management program written 
-and configured in haskell.
-.PP
-The first time you run propellor, it will set up a ~/.propellor/
-repository. Edit ~/.propellor/config.hs to configure it.
-.SH AUTHOR 
-Joey Hess <joey@kitenet.net>

propellor.cabal

@@ -2,7 +2,7 @@ Name: propellor
 Version: 1.0.0
 Cabal-Version: >= 1.6
 License: BSD3
-Maintainer: Joey Hess <joey@kitenet.net>
+Maintainer: Joey Hess <id@joeyh.name>
 Author: Joey Hess
 Stability: Stable
 Copyright: 2014 Joey Hess

src/Propellor/CmdLine.hs

@@ -114,16 +114,19 @@ unknownhost h hosts = errorMessage $ unlines
 	]
 
 buildFirst :: CmdLine -> IO () -> IO ()
-buildFirst cmdline next = do
+buildFirst cmdline next = ifM (doesFileExist "Makefile")
-	oldtime <- getmtime
+	( do
-	ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
+		oldtime <- getmtime
-		( do
+		ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
-			newtime <- getmtime
+			( do
-			if newtime == oldtime
+				newtime <- getmtime
-				then next
+				if newtime == oldtime
-				else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
+					then next
-		, errorMessage "Propellor build failed!" 
+					else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
-		)
+			, errorMessage "Propellor build failed!" 
+			)
+	, next
+	)
   where
 	getmtime = catchMaybeIO $ getModificationTime "propellor"
 
@@ -172,11 +175,11 @@ spin hn hst = do
 
 	updatecmd = mkcmd
 		[ "if [ ! -d " ++ localdir ++ " ]"
-		, "then " ++ intercalate " && "
+		, "then (" ++ intercalate " && "
 			[ "apt-get update"
 			, "apt-get --no-install-recommends --no-upgrade -y install git make"
 			, "echo " ++ toMarked statusMarker (show NeedGitClone)
-			]
+			] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled)
 		, "else " ++ intercalate " && "
 			[ "cd " ++ localdir
 			, "if ! test -x ./propellor; then make deps build; fi"

src/Propellor/Engine.hs

@@ -11,7 +11,6 @@ import "mtl" Control.Monad.Reader
 import Control.Exception (bracket)
 import System.PosixCompat
 import System.Posix.IO
-import Data.Maybe
 
 import Propellor.Types
 import Propellor.Message
@@ -28,7 +27,9 @@ mainProperties :: Host -> IO ()
 mainProperties host = do
 	r <- runPropellor host $
 		ensureProperties [Property "overall" (ensureProperties $ hostProperties host) mempty]
-	setTitle "propellor: done"
+	h <- mkMessageHandle
+        whenConsole h $
+		setTitle "propellor: done"
 	hFlush stdout
 	case r of
 		FailedChange -> exitWith (ExitFailure 1)
@@ -74,8 +75,14 @@ processChainOutput h = go Nothing
 	go lastline = do
 		v <- catchMaybeIO (hGetLine h)
 		case v of
-			Nothing -> pure $ fromMaybe FailedChange $
+			Nothing -> case lastline of
-				readish =<< lastline
+				Nothing -> pure FailedChange
+				Just l -> case readish l of
+					Just r -> pure r
+					Nothing -> do
+						putStrLn l
+						hFlush stdout
+						return FailedChange
 			Just s -> do
 				maybe noop (\l -> unless (null l) (putStrLn l)) lastline
 				hFlush stdout

src/Propellor/Git.hs

@@ -38,7 +38,7 @@ getRepoUrl = getM get urls
 			_ -> Nothing
 
 hasOrigin :: IO Bool
-hasOrigin = do
+hasOrigin = catchDefaultIO False $ do
 	rs <- lines <$> readProcess "git" ["remote"]
 	return $ "origin" `elem` rs
 

src/Propellor/Protocol.hs

@@ -13,7 +13,7 @@ import Data.List
 
 import Propellor
 
-data Stage = NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush
+data Stage = NeedGitClone | NeedRepoUrl | NeedPrivData | NeedGitPush | NeedPrecompiled
 	deriving (Read, Show, Eq)
 
 type Marker = String

src/Propellor/Server.hs

@@ -1,3 +1,7 @@
+-- When propellor --spin is running, the local host acts as a server,
+-- which connects to the remote host's propellor and responds to its
+-- requests.
+
 module Propellor.Server (
 	update,
 	updateServer,
@@ -8,7 +12,9 @@ import Data.List
 import System.Exit
 import System.PosixCompat
 import System.Posix.IO
+import System.Posix.Directory
 import Control.Concurrent.Async
+import Control.Exception (bracket)
 import qualified Data.ByteString as B
 
 import Propellor
@@ -16,6 +22,7 @@ import Propellor.Protocol
 import Propellor.PrivData.Paths
 import Propellor.Git
 import Propellor.Ssh
+import qualified Propellor.Shim as Shim
 import Utility.FileMode
 import Utility.SafeCommand
 
@@ -24,17 +31,19 @@ import Utility.SafeCommand
 -- running the updateServer
 update :: IO ()
 update = do
-	req NeedRepoUrl repoUrlMarker setRepoUrl
+	whenM hasOrigin $
+		req NeedRepoUrl repoUrlMarker setRepoUrl
 	makePrivDataDir
 	req NeedPrivData privDataMarker $
 		writeFileProtected privDataLocal
-	req NeedGitPush gitPushMarker $ \_ -> do
+	whenM hasOrigin $
-		hin <- dup stdInput
+		req NeedGitPush gitPushMarker $ \_ -> do
-		hout <- dup stdOutput
+			hin <- dup stdInput
-		hClose stdin
+			hout <- dup stdOutput
-		hClose stdout
+			hClose stdin
-		unlessM (boolSystem "git" (pullparams hin hout)) $
+			hClose stdout
-			errorMessage "git pull from client failed"
+			unlessM (boolSystem "git" (pullparams hin hout)) $
+				errorMessage "git pull from client failed"
   where
 	pullparams hin hout =
 		[ Param "pull"
@@ -69,6 +78,11 @@ updateServer hn hst connect = connect go
 				hClose fromh
 				sendGitClone hn
 				updateServer hn hst connect
+			(Just NeedPrecompiled) -> do
+				hClose toh
+				hClose fromh
+				sendPrecompiled hn
+				updateServer hn hst connect
 			Nothing -> return ()
 
 sendRepoUrl :: Handle -> IO ()
@@ -113,6 +127,39 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do
 		, "rm -f " ++ remotebundle
 		]
 
+-- Send a tarball containing the precompiled propellor, and libraries.
+-- This should be reasonably portable, as long as the remote host has the
+-- same architecture as the build host.
+sendPrecompiled :: HostName -> IO ()
+sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort") $ do
+	bracket getWorkingDirectory changeWorkingDirectory $ \_ ->
+		withTmpDir "propellor" go
+  where
+	go tmpdir = do
+		cacheparams <- sshCachingParams hn
+		let shimdir = takeFileName localdir
+		createDirectoryIfMissing True (tmpdir </> shimdir)
+		changeWorkingDirectory (tmpdir </> shimdir)
+		me <- readSymbolicLink "/proc/self/exe"
+		shim <- Shim.setup me "."
+		when (shim /= "propellor") $
+			renameFile shim "propellor"
+		changeWorkingDirectory tmpdir
+		withTmpFile "propellor.tar." $ \tarball _ -> allM id
+			[ boolSystem "strip" [File me]
+			, boolSystem "tar" [Param "czf", File tarball, File shimdir]
+			, boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)]
+			, boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd]
+			]
+
+	remotetarball = "/usr/local/propellor.tar"
+
+	unpackcmd = shellWrap $ intercalate " && "
+		[ "cd " ++ takeDirectory remotetarball
+		, "tar xzf " ++ remotetarball
+		, "rm -f " ++ remotetarball
+		]
+
 -- Shim for git push over the propellor ssh channel.
 -- Reads from stdin and sends it to hout;
 -- reads from hin and sends it to stdout.

src/Propellor/Shim.hs

@@ -45,6 +45,8 @@ setup propellorbin dest = do
 	modifyFileMode shim (addModes executeModes)
 	return shim
 
+-- Called when the shimmed propellor is running, so that commands it runs
+-- don't see it.
 cleanEnv :: IO ()
 cleanEnv = void $ unsetEnv "GCONV_PATH"
 

src/Utility/Applicative.hs

@@ -1,6 +1,6 @@
 {- applicative stuff
  -
- - Copyright 2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Data.hs

@@ -1,6 +1,6 @@
 {- utilities for simple data types
  -
- - Copyright 2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2013 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Directory.hs

@@ -1,6 +1,6 @@
 {- directory manipulation
  -
- - Copyright 2011-2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2011-2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Env.hs

@@ -1,6 +1,6 @@
 {- portable environment variables
  -
- - Copyright 2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2013 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Exception.hs

@@ -1,6 +1,6 @@
 {- Simple IO exception handling (and some more)
  -
- - Copyright 2011-2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2011-2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/FileMode.hs

@@ -1,6 +1,6 @@
 {- File mode utilities.
  -
- - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/FileSystemEncoding.hs

@@ -1,6 +1,6 @@
 {- GHC File system encoding handling.
  -
- - Copyright 2012-2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2012-2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/LinuxMkLibs.hs

@@ -1,6 +1,6 @@
 {- Linux library copier and binary shimmer
  -
- - Copyright 2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2013 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Misc.hs

@@ -1,6 +1,6 @@
 {- misc utility functions
  -
- - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2011 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Monad.hs

@@ -1,6 +1,6 @@
 {- monadic stuff
  -
- - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Path.hs

@@ -1,6 +1,6 @@
 {- path manipulation
  -
- - Copyright 2010-2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/PosixFiles.hs

@@ -2,7 +2,7 @@
  -
  - This is like System.PosixCompat.Files, except with a fixed rename.
  -
- - Copyright 2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Process.hs

@@ -1,7 +1,7 @@
 {- System.Process enhancements, including additional ways of running
  - processes, and logging.
  -
- - Copyright 2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/QuickCheck.hs

@@ -1,6 +1,6 @@
 {- QuickCheck with additional instances
  -
- - Copyright 2012-2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2012-2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/SafeCommand.hs

@@ -1,6 +1,6 @@
 {- safely running shell commands
  -
- - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2013 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Scheduled.hs

@@ -1,6 +1,6 @@
 {- scheduled activities
  - 
- - Copyright 2013-2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2013-2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/Table.hs

@@ -1,6 +1,6 @@
 {- text based table generation
  -
- - Copyright 2014 Joey Hess <joey@kitenet.net>
+ - Copyright 2014 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/ThreadScheduler.hs

@@ -1,6 +1,6 @@
 {- thread scheduling
  -
- - Copyright 2012, 2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2012, 2013 Joey Hess <id@joeyh.name>
  - Copyright 2011 Bas van Dijk & Roel van Dijk
  -
  - License: BSD-2-clause

src/Utility/Tmp.hs

@@ -1,6 +1,6 @@
 {- Temporary files and directories.
  -
- - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2013 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}

src/Utility/UserInfo.hs

@@ -1,6 +1,6 @@
 {- user info
  -
- - Copyright 2012 Joey Hess <joey@kitenet.net>
+ - Copyright 2012 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}