cadey
/
route
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

don't panic on a backend disconnect

This commit is contained in:
Cadey Ratio 2017-09-30 10:33:19 -07:00
parent 61af71fdd3
commit d2c0a530ab
No known key found for this signature in database
GPG Key ID: D607EE27C2E7F89A
6 changed files with 72 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/server/backend.go
View File

@ -42,6 +42,7 @@ func (b *Backend) List(ctx context.Context, sel *proto.BackendSelector) (*proto.
Domain: bk.Domain,
Phi: bk.Phi,
Host: bk.Host,
Usable: bk.Usable,
})
}

1
lib/tun2/backend.go
View File

@ -8,4 +8,5 @@ type Backend struct {
Domain string
Phi float32
Host string
Usable bool
}

12
lib/tun2/connection.go
View File

@ -24,9 +24,15 @@ type Connection struct {
controlStream *smux.Stream
user string
domain string
cancel context.CancelFunc
cf context.CancelFunc
detector *failure.Detector
Auth *Auth
usable bool
}
func (c *Connection) cancel() {
c.cf()
c.usable = false
}
// F logs key->value pairs as an ln.Fer
@ -63,6 +69,10 @@ func (c *Connection) Ping() error {
// OpenStream creates a new stream (connection) to the backend server.
func (c *Connection) OpenStream() (net.Conn, error) {
if !c.usable {
return nil, ErrNoSuchBackend
}
err := c.conn.SetDeadline(time.Now().Add(time.Second))
if err != nil {
ln.Error(err, c.F())

17
lib/tun2/server.go
View File

@ -102,6 +102,7 @@ func (s *Server) getBackendsForMatcher(bm backendMatcher) []Backend {
Domain: c.domain,
Phi: float32(c.detector.Phi(time.Now())),
Host: c.conn.RemoteAddr().String(),
Usable: c.usable,
})
}
@ -334,7 +335,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
session: session,
user: tokenUser,
domain: auth.Domain,
cancel: cancel,
cf: cancel,
detector: failure.New(15, 1),
Auth: auth,
}
@ -362,6 +363,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
conns = append(conns, connection)
s.domains.Set(auth.Domain, conns)
connection.usable = true
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
@ -371,7 +373,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
case <-ticker.C:
err := connection.Ping()
if err != nil {
cancel()
connection.cancel()
}
case <-ctx.Done():
s.RemoveConn(connection)
@ -440,7 +442,14 @@ func (s *Server) RoundTrip(req *http.Request) (*http.Response, error) {
}
}
if len(conns) == 0 {
var goodConns []*Connection
for _, conn := range conns {
if conn.usable {
goodConns = append(goodConns, conn)
}
}
if len(goodConns) == 0 {
ln.Error(ErrNoSuchBackend, ln.F{
"action": "no_backend_connected",
"remote": req.RemoteAddr,
@ -451,7 +460,7 @@ func (s *Server) RoundTrip(req *http.Request) (*http.Response, error) {
return nil, ErrNoSuchBackend
}
c := conns[rand.Intn(len(conns))]
c := goodConns[rand.Intn(len(goodConns))]
resp, err := c.RoundTrip(req)
if err != nil {

81
proto/route.pb.go
View File

@ -219,6 +219,7 @@ type Backend struct {
Domain string `protobuf:"bytes,4,opt,name=domain" json:"domain,omitempty"`
Phi float32 `protobuf:"fixed32,5,opt,name=phi" json:"phi,omitempty"`
Host string `protobuf:"bytes,6,opt,name=host" json:"host,omitempty"`
Usable bool `protobuf:"varint,7,opt,name=usable" json:"usable,omitempty"`
}
func (m *Backend) Reset() { *m = Backend{} }
@ -268,6 +269,13 @@ func (m *Backend) GetHost() string {
return ""
}
func (m *Backend) GetUsable() bool {
if m != nil {
return m.Usable
}
return false
}
type BackendList struct {
Bs *BackendSelector `protobuf:"bytes,1,opt,name=bs" json:"bs,omitempty"`
Backends []*Backend `protobuf:"bytes,2,rep,name=backends" json:"backends,omitempty"`
@ -814,40 +822,41 @@ var _Backends_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("route.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 553 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xd3, 0x4e,
0x10, 0xf5, 0x9f, 0xd8, 0x4d, 0x26, 0x3f, 0x35, 0xfd, 0x8d, 0xa2, 0xca, 0x4a, 0x39, 0x44, 0x4b,
0x29, 0x16, 0x88, 0x0a, 0xa5, 0x48, 0x5c, 0xe0, 0x00, 0x0a, 0x8a, 0x22, 0x50, 0x85, 0x36, 0xdc,
0x38, 0x39, 0xc9, 0x48, 0xb1, 0x6a, 0xb2, 0xc1, 0xbb, 0x41, 0xe2, 0xc8, 0x67, 0xe3, 0x33, 0xf0,
0x7d, 0x90, 0xc7, 0x9b, 0xc4, 0x8e, 0x0b, 0xe2, 0xb6, 0x33, 0xef, 0xed, 0xf8, 0xbd, 0xd9, 0x67,
0xe8, 0xe6, 0x6a, 0x6b, 0xe8, 0x7a, 0x93, 0x2b, 0xa3, 0x30, 0xe0, 0x42, 0x04, 0xe0, 0xdf, 0xa6,
0x99, 0x78, 0x04, 0xbd, 0x09, 0x19, 0x59, 0xb4, 0x24, 0x7d, 0xdd, 0x92, 0x36, 0x88, 0xd0, 0x5a,
0x29, 0x6d, 0x22, 0x77, 0xe8, 0xc6, 0x1d, 0xc9, 0x67, 0xf1, 0x0e, 0x02, 0xe6, 0xe0, 0x29, 0x78,
0xe9, 0xd2, 0x42, 0x5e, 0xba, 0xc4, 0x08, 0x4e, 0x16, 0x39, 0x25, 0x46, 0xe5, 0x91, 0xc7, 0xcd,
0x5d, 0xb9, 0x1f, 0xe3, 0x57, 0xc6, 0xbc, 0x82, 0xfe, 0x84, 0xcc, 0x9b, 0x2c, 0xe3, 0x61, 0x5a,
0x92, 0xde, 0xa8, 0xb5, 0x26, 0xbc, 0x84, 0x90, 0x55, 0xe9, 0xc8, 0x1d, 0xfa, 0x71, 0x77, 0xf4,
0xdf, 0x75, 0xa9, 0xb8, 0xd4, 0x65, 0x31, 0xf1, 0x00, 0x60, 0x3a, 0xde, 0xdf, 0x39, 0x52, 0x22,
0x3e, 0x43, 0xf0, 0x49, 0xdd, 0xd1, 0xba, 0x21, 0x11, 0xa1, 0x35, 0x57, 0xcb, 0xef, 0x56, 0x1f,
0x9f, 0xf1, 0x1c, 0x42, 0xbd, 0x50, 0x1b, 0xd2, 0x91, 0x3f, 0xf4, 0xe3, 0x8e, 0xb4, 0x55, 0xd1,
0x4f, 0x16, 0x26, 0xfd, 0x46, 0x51, 0x6b, 0xe8, 0xc6, 0x6d, 0x69, 0x2b, 0xf1, 0x1c, 0xda, 0x3c,
0x7c, 0x46, 0xa6, 0x10, 0x6b, 0x8a, 0xf3, 0xb1, 0x58, 0x26, 0x48, 0x8b, 0x89, 0x97, 0xbc, 0xd8,
0xb2, 0x67, 0x17, 0xdb, 0x87, 0x80, 0x41, 0xab, 0xad, 0x2c, 0xac, 0x5c, 0x6f, 0xef, 0xe3, 0x87,
0x0b, 0x27, 0x6f, 0x93, 0xc5, 0x1d, 0xad, 0x97, 0x0d, 0x2b, 0x7d, 0x08, 0xf8, 0x11, 0x2d, 0xbd,
0x2c, 0x0a, 0x83, 0x5b, 0x4d, 0xf9, 0x6e, 0xd3, 0xc5, 0xb9, 0x30, 0xb2, 0x54, 0x5f, 0x92, 0x74,
0xcd, 0x46, 0x3a, 0xd2, 0x56, 0x78, 0x06, 0xfe, 0x66, 0x95, 0x46, 0xc1, 0xd0, 0x8d, 0x3d, 0x59,
0x1c, 0xf7, 0xef, 0x14, 0x56, 0xde, 0x29, 0x81, 0xae, 0x95, 0xf0, 0x21, 0xd5, 0x06, 0xaf, 0xc0,
0x9b, 0x6b, 0x96, 0xd1, 0x1d, 0x9d, 0x5b, 0xb7, 0x16, 0x9f, 0x51, 0x46, 0x0b, 0xa3, 0x72, 0xe9,
0xcd, 0x35, 0x3e, 0x81, 0xf6, 0xbc, 0x6c, 0xeb, 0xc8, 0xe3, 0xdd, 0x9c, 0xd6, 0xd9, 0x72, 0x8f,
0x8b, 0xd7, 0xd0, 0x3b, 0x1a, 0x51, 0xd1, 0xec, 0xd6, 0x34, 0xef, 0xfc, 0x79, 0x07, 0x7f, 0xe2,
0x02, 0x3a, 0xf6, 0xfa, 0x74, 0x7c, 0xbc, 0xa6, 0xd1, 0x4f, 0x17, 0xc2, 0x32, 0x61, 0xf8, 0x0c,
0xfc, 0x09, 0x19, 0xdc, 0xa9, 0x3e, 0xca, 0xfa, 0xa0, 0x16, 0x34, 0xe1, 0xe0, 0x0d, 0x84, 0x65,
0x40, 0x11, 0x2c, 0x72, 0x9b, 0x66, 0x83, 0x8b, 0xc3, 0xed, 0x46, 0x76, 0x85, 0x83, 0x31, 0xf8,
0x1f, 0xb7, 0x06, 0x6b, 0xb3, 0x06, 0xff, 0xdb, 0xea, 0x90, 0x58, 0xe1, 0xe0, 0x53, 0x08, 0xc7,
0x94, 0x91, 0xa1, 0x7f, 0x20, 0x8f, 0x7e, 0xb9, 0x10, 0x72, 0x7e, 0xee, 0x73, 0x51, 0x0d, 0xd6,
0xa0, 0x96, 0x40, 0xe1, 0xe0, 0xe3, 0x7b, 0x5d, 0xf4, 0xaa, 0xac, 0x19, 0x19, 0xe1, 0xe0, 0xc3,
0xba, 0x72, 0x46, 0x1a, 0xd3, 0x2e, 0x1b, 0xa2, 0x4b, 0x5e, 0x65, 0x36, 0x2f, 0x01, 0xc6, 0xc4,
0x7f, 0x4b, 0xf2, 0x77, 0xe6, 0x68, 0x05, 0x6d, 0xfb, 0x74, 0x1a, 0x5f, 0x40, 0x8b, 0x13, 0xf6,
0x87, 0x54, 0x0d, 0xb0, 0xde, 0x2f, 0xb8, 0xc2, 0xc1, 0x2b, 0x68, 0xbd, 0x4f, 0xb3, 0x0c, 0xcf,
0xea, 0xe8, 0x74, 0x5c, 0xff, 0xd2, 0x3c, 0xe4, 0xff, 0xe3, 0xe6, 0x77, 0x00, 0x00, 0x00, 0xff,
0xff, 0x11, 0x4c, 0x9d, 0x1a, 0x00, 0x05, 0x00, 0x00,
// 565 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x5d, 0x8b, 0xd3, 0x40,
0x14, 0xcd, 0x47, 0x93, 0xb6, 0xb7, 0xb2, 0x5d, 0x2f, 0x65, 0x09, 0x5d, 0x1f, 0xca, 0xb8, 0xae,
0x41, 0x71, 0x91, 0xae, 0xe0, 0x8b, 0x3e, 0x28, 0x95, 0x52, 0x94, 0x45, 0x52, 0xdf, 0x7c, 0x4a,
0xda, 0x0b, 0x0d, 0x1b, 0x3b, 0x35, 0x33, 0x15, 0xfc, 0x29, 0xfe, 0x1e, 0x7f, 0x83, 0xff, 0x47,
0x72, 0x67, 0xfa, 0x91, 0x76, 0x95, 0x7d, 0x9b, 0x7b, 0xcf, 0x99, 0x9b, 0x73, 0xee, 0x9c, 0x40,
0xa7, 0x94, 0x6b, 0x4d, 0x57, 0xab, 0x52, 0x6a, 0x89, 0x01, 0x17, 0x22, 0x00, 0xff, 0x26, 0x2f,
0xc4, 0x13, 0xe8, 0x8e, 0x49, 0x27, 0x55, 0x2b, 0xa1, 0xef, 0x6b, 0x52, 0x1a, 0x11, 0x1a, 0x0b,
0xa9, 0x74, 0xe4, 0x0e, 0xdc, 0xb8, 0x9d, 0xf0, 0x59, 0x7c, 0x80, 0x80, 0x39, 0x78, 0x02, 0x5e,
0x3e, 0xb7, 0x90, 0x97, 0xcf, 0x31, 0x82, 0xe6, 0xac, 0xa4, 0x54, 0xcb, 0x32, 0xf2, 0xb8, 0xb9,
0x29, 0xb7, 0x63, 0xfc, 0xbd, 0x31, 0x6f, 0xa0, 0x37, 0x26, 0xfd, 0xae, 0x28, 0x78, 0x98, 0x4a,
0x48, 0xad, 0xe4, 0x52, 0x11, 0x5e, 0x40, 0xc8, 0xaa, 0x54, 0xe4, 0x0e, 0xfc, 0xb8, 0x33, 0x7c,
0x70, 0x65, 0x14, 0x1b, 0x5d, 0x16, 0x13, 0x8f, 0x00, 0x26, 0xa3, 0xed, 0x9d, 0x03, 0x25, 0xe2,
0x2b, 0x04, 0x5f, 0xe4, 0x2d, 0x2d, 0x8f, 0x24, 0x22, 0x34, 0x32, 0x39, 0xff, 0x69, 0xf5, 0xf1,
0x19, 0xcf, 0x20, 0x54, 0x33, 0xb9, 0x22, 0x15, 0xf9, 0x03, 0x3f, 0x6e, 0x27, 0xb6, 0xaa, 0xfa,
0xe9, 0x4c, 0xe7, 0x3f, 0x28, 0x6a, 0x0c, 0xdc, 0xb8, 0x95, 0xd8, 0x4a, 0xbc, 0x84, 0x16, 0x0f,
0x9f, 0x92, 0xae, 0xc4, 0xea, 0xea, 0x7c, 0x28, 0x96, 0x09, 0x89, 0xc5, 0xc4, 0x6b, 0x5e, 0xac,
0xe9, 0xd9, 0xc5, 0xf6, 0x20, 0x60, 0xd0, 0x6a, 0x33, 0x85, 0x95, 0xeb, 0x6d, 0x7d, 0xfc, 0x72,
0xa1, 0xf9, 0x3e, 0x9d, 0xdd, 0xd2, 0x72, 0x7e, 0x64, 0xa5, 0x07, 0x01, 0x3f, 0xa2, 0xa5, 0x9b,
0xa2, 0x32, 0xb8, 0x56, 0x54, 0x6e, 0x36, 0x5d, 0x9d, 0x2b, 0x23, 0x73, 0xf9, 0x2d, 0xcd, 0x97,
0x6c, 0xa4, 0x9d, 0xd8, 0x0a, 0x4f, 0xc1, 0x5f, 0x2d, 0xf2, 0x28, 0x18, 0xb8, 0xb1, 0x97, 0x54,
0xc7, 0xed, 0x3b, 0x85, 0xbb, 0x77, 0xaa, 0x6e, 0xaf, 0x55, 0x9a, 0x15, 0x14, 0x35, 0xcd, 0x1a,
0x4c, 0x25, 0x52, 0xe8, 0x58, 0x69, 0x9f, 0x72, 0xa5, 0xf1, 0x12, 0xbc, 0x4c, 0xb1, 0xbc, 0xce,
0xf0, 0xcc, 0x6e, 0xc1, 0xe2, 0x53, 0x2a, 0x68, 0xa6, 0x65, 0x99, 0x78, 0x99, 0xc2, 0x67, 0xd0,
0xca, 0x4c, 0x5b, 0x45, 0x1e, 0xef, 0xec, 0xa4, 0xce, 0x4e, 0xb6, 0xb8, 0x78, 0x0b, 0xdd, 0x83,
0x11, 0x7b, 0x5e, 0xdc, 0x9a, 0x97, 0x8d, 0x6f, 0x6f, 0xe7, 0x5b, 0x9c, 0x43, 0xdb, 0x5e, 0x9f,
0x8c, 0x0e, 0xd7, 0x37, 0xfc, 0xed, 0x42, 0x68, 0x92, 0x87, 0x2f, 0xc0, 0x1f, 0x93, 0xc6, 0x8d,
0xea, 0x83, 0x7f, 0xa0, 0x5f, 0x0b, 0xa0, 0x70, 0xf0, 0x1a, 0x42, 0x13, 0x5c, 0x04, 0x8b, 0xdc,
0xe4, 0x45, 0xff, 0x7c, 0x77, 0xfb, 0x28, 0xd3, 0xc2, 0xc1, 0x18, 0xfc, 0xcf, 0x6b, 0x8d, 0xb5,
0x59, 0xfd, 0x87, 0xb6, 0xda, 0x25, 0x59, 0x38, 0xf8, 0x1c, 0xc2, 0x11, 0x15, 0xa4, 0xe9, 0x1e,
0xe4, 0xe1, 0x1f, 0x17, 0x42, 0xce, 0xd5, 0x5d, 0x2e, 0xf6, 0x03, 0xd7, 0xaf, 0x25, 0x53, 0x38,
0xf8, 0xf4, 0x4e, 0x17, 0xdd, 0x7d, 0xd6, 0x94, 0xb4, 0x70, 0xf0, 0x71, 0x5d, 0x39, 0x23, 0x47,
0xd3, 0x2e, 0x8e, 0x44, 0x1b, 0xde, 0xde, 0x6c, 0x5e, 0x02, 0x8c, 0x88, 0xff, 0xa2, 0xf4, 0xff,
0xcc, 0xe1, 0x02, 0x5a, 0xf6, 0xe9, 0x14, 0xbe, 0x82, 0x06, 0x27, 0xec, 0x1f, 0xa9, 0xea, 0x63,
0xbd, 0x5f, 0x71, 0x85, 0x83, 0x97, 0xd0, 0xf8, 0x98, 0x17, 0x05, 0x9e, 0xd6, 0xd1, 0xc9, 0xa8,
0xfe, 0xa5, 0x2c, 0xe4, 0xff, 0xe6, 0xfa, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x00, 0xdd,
0xf1, 0x18, 0x05, 0x00, 0x00,
}

1
proto/route.proto
View File

@ -121,6 +121,7 @@ message Backend {
string domain = 4;
float phi = 5;
string host = 6;
bool usable = 7;
}
message BackendList {