cadey
/
route
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

don't panic on a backend disconnect

This commit is contained in:
Cadey Ratio 2017-09-30 10:33:19 -07:00
parent 61af71fdd3
commit d2c0a530ab
No known key found for this signature in database
GPG Key ID: D607EE27C2E7F89A
6 changed files with 72 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/server/backend.go
View File

@ -42,6 +42,7 @@ func (b *Backend) List(ctx context.Context, sel *proto.BackendSelector) (*proto.
Domain: bk.Domain, Domain: bk.Domain,
Phi: bk.Phi, Phi: bk.Phi,
Host: bk.Host, Host: bk.Host,
Usable: bk.Usable,
}) })
} }

1
lib/tun2/backend.go
View File

@ -8,4 +8,5 @@ type Backend struct {
Domain string Domain string
Phi float32 Phi float32
Host string Host string
Usable bool
} }

12
lib/tun2/connection.go
View File

@ -24,9 +24,15 @@ type Connection struct {
controlStream *smux.Stream controlStream *smux.Stream
user string user string
domain string domain string
cancel context.CancelFunc cf context.CancelFunc
detector *failure.Detector detector *failure.Detector
Auth *Auth Auth *Auth
usable bool
}
func (c *Connection) cancel() {
c.cf()
c.usable = false
} }
// F logs key->value pairs as an ln.Fer // F logs key->value pairs as an ln.Fer
@ -63,6 +69,10 @@ func (c *Connection) Ping() error {
// OpenStream creates a new stream (connection) to the backend server. // OpenStream creates a new stream (connection) to the backend server.
func (c *Connection) OpenStream() (net.Conn, error) { func (c *Connection) OpenStream() (net.Conn, error) {
if !c.usable {
return nil, ErrNoSuchBackend
}
err := c.conn.SetDeadline(time.Now().Add(time.Second)) err := c.conn.SetDeadline(time.Now().Add(time.Second))
if err != nil { if err != nil {
ln.Error(err, c.F()) ln.Error(err, c.F())

17
lib/tun2/server.go
View File

@ -102,6 +102,7 @@ func (s *Server) getBackendsForMatcher(bm backendMatcher) []Backend {
Domain: c.domain, Domain: c.domain,
Phi: float32(c.detector.Phi(time.Now())), Phi: float32(c.detector.Phi(time.Now())),
Host: c.conn.RemoteAddr().String(), Host: c.conn.RemoteAddr().String(),
Usable: c.usable,
}) })
} }
@ -334,7 +335,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
session: session, session: session,
user: tokenUser, user: tokenUser,
domain: auth.Domain, domain: auth.Domain,
cancel: cancel, cf: cancel,
detector: failure.New(15, 1), detector: failure.New(15, 1),
Auth: auth, Auth: auth,
} }
@ -362,6 +363,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
conns = append(conns, connection) conns = append(conns, connection)
s.domains.Set(auth.Domain, conns) s.domains.Set(auth.Domain, conns)
connection.usable = true
ticker := time.NewTicker(5 * time.Second) ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop() defer ticker.Stop()
@ -371,7 +373,7 @@ func (s *Server) HandleConn(c net.Conn, isKCP bool) {
case <-ticker.C: case <-ticker.C:
err := connection.Ping() err := connection.Ping()
if err != nil { if err != nil {
cancel() connection.cancel()
} }
case <-ctx.Done(): case <-ctx.Done():
s.RemoveConn(connection) s.RemoveConn(connection)
@ -440,7 +442,14 @@ func (s *Server) RoundTrip(req *http.Request) (*http.Response, error) {
} }
} }
if len(conns) == 0 { var goodConns []*Connection
for _, conn := range conns {
if conn.usable {
goodConns = append(goodConns, conn)
}
}
if len(goodConns) == 0 {
ln.Error(ErrNoSuchBackend, ln.F{ ln.Error(ErrNoSuchBackend, ln.F{
"action": "no_backend_connected", "action": "no_backend_connected",
"remote": req.RemoteAddr, "remote": req.RemoteAddr,
@ -451,7 +460,7 @@ func (s *Server) RoundTrip(req *http.Request) (*http.Response, error) {
return nil, ErrNoSuchBackend return nil, ErrNoSuchBackend
} }
c := conns[rand.Intn(len(conns))] c := goodConns[rand.Intn(len(goodConns))]
resp, err := c.RoundTrip(req) resp, err := c.RoundTrip(req)
if err != nil { if err != nil {

81
proto/route.pb.go
View File

@ -219,6 +219,7 @@ type Backend struct {
Domain string `protobuf:"bytes,4,opt,name=domain" json:"domain,omitempty"` Domain string `protobuf:"bytes,4,opt,name=domain" json:"domain,omitempty"`
Phi float32 `protobuf:"fixed32,5,opt,name=phi" json:"phi,omitempty"` Phi float32 `protobuf:"fixed32,5,opt,name=phi" json:"phi,omitempty"`
Host string `protobuf:"bytes,6,opt,name=host" json:"host,omitempty"` Host string `protobuf:"bytes,6,opt,name=host" json:"host,omitempty"`
Usable bool `protobuf:"varint,7,opt,name=usable" json:"usable,omitempty"`
} }
func (m *Backend) Reset() { *m = Backend{} } func (m *Backend) Reset() { *m = Backend{} }
@ -268,6 +269,13 @@ func (m *Backend) GetHost() string {
return "" return ""
} }
func (m *Backend) GetUsable() bool {
if m != nil {
return m.Usable
}
return false
}
type BackendList struct { type BackendList struct {
Bs *BackendSelector `protobuf:"bytes,1,opt,name=bs" json:"bs,omitempty"` Bs *BackendSelector `protobuf:"bytes,1,opt,name=bs" json:"bs,omitempty"`
Backends []*Backend `protobuf:"bytes,2,rep,name=backends" json:"backends,omitempty"` Backends []*Backend `protobuf:"bytes,2,rep,name=backends" json:"backends,omitempty"`
@ -814,40 +822,41 @@ var _Backends_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("route.proto", fileDescriptor0) } func init() { proto.RegisterFile("route.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 553 bytes of a gzipped FileDescriptorProto // 565 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xd3, 0x4e, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x5d, 0x8b, 0xd3, 0x40,
0x10, 0xf5, 0x9f, 0xd8, 0x4d, 0x26, 0x3f, 0x35, 0xfd, 0x8d, 0xa2, 0xca, 0x4a, 0x39, 0x44, 0x4b, 0x14, 0xcd, 0x47, 0x93, 0xb6, 0xb7, 0xb2, 0x5d, 0x2f, 0x65, 0x09, 0x5d, 0x1f, 0xca, 0xb8, 0xae,
0x29, 0x16, 0x88, 0x0a, 0xa5, 0x48, 0x5c, 0xe0, 0x00, 0x0a, 0x8a, 0x22, 0x50, 0x85, 0x36, 0xdc, 0x41, 0x71, 0x91, 0xae, 0xe0, 0x8b, 0x3e, 0x28, 0x95, 0x52, 0x94, 0x45, 0x52, 0xdf, 0x7c, 0x4a,
0x38, 0x39, 0xc9, 0x48, 0xb1, 0x6a, 0xb2, 0xc1, 0xbb, 0x41, 0xe2, 0xc8, 0x67, 0xe3, 0x33, 0xf0, 0xda, 0x0b, 0x0d, 0x1b, 0x3b, 0x35, 0x33, 0x15, 0xfc, 0x29, 0xfe, 0x1e, 0x7f, 0x83, 0xff, 0x47,
0x7d, 0x90, 0xc7, 0x9b, 0xc4, 0x8e, 0x0b, 0xe2, 0xb6, 0x33, 0xef, 0xed, 0xf8, 0xbd, 0xd9, 0x67, 0x72, 0x67, 0xfa, 0x91, 0x76, 0x95, 0x7d, 0x9b, 0x7b, 0xcf, 0x99, 0x9b, 0x73, 0xee, 0x9c, 0x40,
0xe8, 0xe6, 0x6a, 0x6b, 0xe8, 0x7a, 0x93, 0x2b, 0xa3, 0x30, 0xe0, 0x42, 0x04, 0xe0, 0xdf, 0xa6, 0xa7, 0x94, 0x6b, 0x4d, 0x57, 0xab, 0x52, 0x6a, 0x89, 0x01, 0x17, 0x22, 0x00, 0xff, 0x26, 0x2f,
0x99, 0x78, 0x04, 0xbd, 0x09, 0x19, 0x59, 0xb4, 0x24, 0x7d, 0xdd, 0x92, 0x36, 0x88, 0xd0, 0x5a, 0xc4, 0x13, 0xe8, 0x8e, 0x49, 0x27, 0x55, 0x2b, 0xa1, 0xef, 0x6b, 0x52, 0x1a, 0x11, 0x1a, 0x0b,
0x29, 0x6d, 0x22, 0x77, 0xe8, 0xc6, 0x1d, 0xc9, 0x67, 0xf1, 0x0e, 0x02, 0xe6, 0xe0, 0x29, 0x78, 0xa9, 0x74, 0xe4, 0x0e, 0xdc, 0xb8, 0x9d, 0xf0, 0x59, 0x7c, 0x80, 0x80, 0x39, 0x78, 0x02, 0x5e,
0xe9, 0xd2, 0x42, 0x5e, 0xba, 0xc4, 0x08, 0x4e, 0x16, 0x39, 0x25, 0x46, 0xe5, 0x91, 0xc7, 0xcd, 0x3e, 0xb7, 0x90, 0x97, 0xcf, 0x31, 0x82, 0xe6, 0xac, 0xa4, 0x54, 0xcb, 0x32, 0xf2, 0xb8, 0xb9,
0x5d, 0xb9, 0x1f, 0xe3, 0x57, 0xc6, 0xbc, 0x82, 0xfe, 0x84, 0xcc, 0x9b, 0x2c, 0xe3, 0x61, 0x5a, 0x29, 0xb7, 0x63, 0xfc, 0xbd, 0x31, 0x6f, 0xa0, 0x37, 0x26, 0xfd, 0xae, 0x28, 0x78, 0x98, 0x4a,
0x92, 0xde, 0xa8, 0xb5, 0x26, 0xbc, 0x84, 0x90, 0x55, 0xe9, 0xc8, 0x1d, 0xfa, 0x71, 0x77, 0xf4, 0x48, 0xad, 0xe4, 0x52, 0x11, 0x5e, 0x40, 0xc8, 0xaa, 0x54, 0xe4, 0x0e, 0xfc, 0xb8, 0x33, 0x7c,
0xdf, 0x75, 0xa9, 0xb8, 0xd4, 0x65, 0x31, 0xf1, 0x00, 0x60, 0x3a, 0xde, 0xdf, 0x39, 0x52, 0x22, 0x70, 0x65, 0x14, 0x1b, 0x5d, 0x16, 0x13, 0x8f, 0x00, 0x26, 0xa3, 0xed, 0x9d, 0x03, 0x25, 0xe2,
0x3e, 0x43, 0xf0, 0x49, 0xdd, 0xd1, 0xba, 0x21, 0x11, 0xa1, 0x35, 0x57, 0xcb, 0xef, 0x56, 0x1f, 0x2b, 0x04, 0x5f, 0xe4, 0x2d, 0x2d, 0x8f, 0x24, 0x22, 0x34, 0x32, 0x39, 0xff, 0x69, 0xf5, 0xf1,
0x9f, 0xf1, 0x1c, 0x42, 0xbd, 0x50, 0x1b, 0xd2, 0x91, 0x3f, 0xf4, 0xe3, 0x8e, 0xb4, 0x55, 0xd1, 0x19, 0xcf, 0x20, 0x54, 0x33, 0xb9, 0x22, 0x15, 0xf9, 0x03, 0x3f, 0x6e, 0x27, 0xb6, 0xaa, 0xfa,
0x4f, 0x16, 0x26, 0xfd, 0x46, 0x51, 0x6b, 0xe8, 0xc6, 0x6d, 0x69, 0x2b, 0xf1, 0x1c, 0xda, 0x3c, 0xe9, 0x4c, 0xe7, 0x3f, 0x28, 0x6a, 0x0c, 0xdc, 0xb8, 0x95, 0xd8, 0x4a, 0xbc, 0x84, 0x16, 0x0f,
0x7c, 0x46, 0xa6, 0x10, 0x6b, 0x8a, 0xf3, 0xb1, 0x58, 0x26, 0x48, 0x8b, 0x89, 0x97, 0xbc, 0xd8, 0x9f, 0x92, 0xae, 0xc4, 0xea, 0xea, 0x7c, 0x28, 0x96, 0x09, 0x89, 0xc5, 0xc4, 0x6b, 0x5e, 0xac,
0xb2, 0x67, 0x17, 0xdb, 0x87, 0x80, 0x41, 0xab, 0xad, 0x2c, 0xac, 0x5c, 0x6f, 0xef, 0xe3, 0x87, 0xe9, 0xd9, 0xc5, 0xf6, 0x20, 0x60, 0xd0, 0x6a, 0x33, 0x85, 0x95, 0xeb, 0x6d, 0x7d, 0xfc, 0x72,
0x0b, 0x27, 0x6f, 0x93, 0xc5, 0x1d, 0xad, 0x97, 0x0d, 0x2b, 0x7d, 0x08, 0xf8, 0x11, 0x2d, 0xbd, 0xa1, 0xf9, 0x3e, 0x9d, 0xdd, 0xd2, 0x72, 0x7e, 0x64, 0xa5, 0x07, 0x01, 0x3f, 0xa2, 0xa5, 0x9b,
0x2c, 0x0a, 0x83, 0x5b, 0x4d, 0xf9, 0x6e, 0xd3, 0xc5, 0xb9, 0x30, 0xb2, 0x54, 0x5f, 0x92, 0x74, 0xa2, 0x32, 0xb8, 0x56, 0x54, 0x6e, 0x36, 0x5d, 0x9d, 0x2b, 0x23, 0x73, 0xf9, 0x2d, 0xcd, 0x97,
0xcd, 0x46, 0x3a, 0xd2, 0x56, 0x78, 0x06, 0xfe, 0x66, 0x95, 0x46, 0xc1, 0xd0, 0x8d, 0x3d, 0x59, 0x6c, 0xa4, 0x9d, 0xd8, 0x0a, 0x4f, 0xc1, 0x5f, 0x2d, 0xf2, 0x28, 0x18, 0xb8, 0xb1, 0x97, 0x54,
0x1c, 0xf7, 0xef, 0x14, 0x56, 0xde, 0x29, 0x81, 0xae, 0x95, 0xf0, 0x21, 0xd5, 0x06, 0xaf, 0xc0, 0xc7, 0xed, 0x3b, 0x85, 0xbb, 0x77, 0xaa, 0x6e, 0xaf, 0x55, 0x9a, 0x15, 0x14, 0x35, 0xcd, 0x1a,
0x9b, 0x6b, 0x96, 0xd1, 0x1d, 0x9d, 0x5b, 0xb7, 0x16, 0x9f, 0x51, 0x46, 0x0b, 0xa3, 0x72, 0xe9, 0x4c, 0x25, 0x52, 0xe8, 0x58, 0x69, 0x9f, 0x72, 0xa5, 0xf1, 0x12, 0xbc, 0x4c, 0xb1, 0xbc, 0xce,
0xcd, 0x35, 0x3e, 0x81, 0xf6, 0xbc, 0x6c, 0xeb, 0xc8, 0xe3, 0xdd, 0x9c, 0xd6, 0xd9, 0x72, 0x8f, 0xf0, 0xcc, 0x6e, 0xc1, 0xe2, 0x53, 0x2a, 0x68, 0xa6, 0x65, 0x99, 0x78, 0x99, 0xc2, 0x67, 0xd0,
0x8b, 0xd7, 0xd0, 0x3b, 0x1a, 0x51, 0xd1, 0xec, 0xd6, 0x34, 0xef, 0xfc, 0x79, 0x07, 0x7f, 0xe2, 0xca, 0x4c, 0x5b, 0x45, 0x1e, 0xef, 0xec, 0xa4, 0xce, 0x4e, 0xb6, 0xb8, 0x78, 0x0b, 0xdd, 0x83,
0x02, 0x3a, 0xf6, 0xfa, 0x74, 0x7c, 0xbc, 0xa6, 0xd1, 0x4f, 0x17, 0xc2, 0x32, 0x61, 0xf8, 0x0c, 0x11, 0x7b, 0x5e, 0xdc, 0x9a, 0x97, 0x8d, 0x6f, 0x6f, 0xe7, 0x5b, 0x9c, 0x43, 0xdb, 0x5e, 0x9f,
0xfc, 0x09, 0x19, 0xdc, 0xa9, 0x3e, 0xca, 0xfa, 0xa0, 0x16, 0x34, 0xe1, 0xe0, 0x0d, 0x84, 0x65, 0x8c, 0x0e, 0xd7, 0x37, 0xfc, 0xed, 0x42, 0x68, 0x92, 0x87, 0x2f, 0xc0, 0x1f, 0x93, 0xc6, 0x8d,
0x40, 0x11, 0x2c, 0x72, 0x9b, 0x66, 0x83, 0x8b, 0xc3, 0xed, 0x46, 0x76, 0x85, 0x83, 0x31, 0xf8, 0xea, 0x83, 0x7f, 0xa0, 0x5f, 0x0b, 0xa0, 0x70, 0xf0, 0x1a, 0x42, 0x13, 0x5c, 0x04, 0x8b, 0xdc,
0x1f, 0xb7, 0x06, 0x6b, 0xb3, 0x06, 0xff, 0xdb, 0xea, 0x90, 0x58, 0xe1, 0xe0, 0x53, 0x08, 0xc7, 0xe4, 0x45, 0xff, 0x7c, 0x77, 0xfb, 0x28, 0xd3, 0xc2, 0xc1, 0x18, 0xfc, 0xcf, 0x6b, 0x8d, 0xb5,
0x94, 0x91, 0xa1, 0x7f, 0x20, 0x8f, 0x7e, 0xb9, 0x10, 0x72, 0x7e, 0xee, 0x73, 0x51, 0x0d, 0xd6, 0x59, 0xfd, 0x87, 0xb6, 0xda, 0x25, 0x59, 0x38, 0xf8, 0x1c, 0xc2, 0x11, 0x15, 0xa4, 0xe9, 0x1e,
0xa0, 0x96, 0x40, 0xe1, 0xe0, 0xe3, 0x7b, 0x5d, 0xf4, 0xaa, 0xac, 0x19, 0x19, 0xe1, 0xe0, 0xc3, 0xe4, 0xe1, 0x1f, 0x17, 0x42, 0xce, 0xd5, 0x5d, 0x2e, 0xf6, 0x03, 0xd7, 0xaf, 0x25, 0x53, 0x38,
0xba, 0x72, 0x46, 0x1a, 0xd3, 0x2e, 0x1b, 0xa2, 0x4b, 0x5e, 0x65, 0x36, 0x2f, 0x01, 0xc6, 0xc4, 0xf8, 0xf4, 0x4e, 0x17, 0xdd, 0x7d, 0xd6, 0x94, 0xb4, 0x70, 0xf0, 0x71, 0x5d, 0x39, 0x23, 0x47,
0x7f, 0x4b, 0xf2, 0x77, 0xe6, 0x68, 0x05, 0x6d, 0xfb, 0x74, 0x1a, 0x5f, 0x40, 0x8b, 0x13, 0xf6, 0xd3, 0x2e, 0x8e, 0x44, 0x1b, 0xde, 0xde, 0x6c, 0x5e, 0x02, 0x8c, 0x88, 0xff, 0xa2, 0xf4, 0xff,
0x87, 0x54, 0x0d, 0xb0, 0xde, 0x2f, 0xb8, 0xc2, 0xc1, 0x2b, 0x68, 0xbd, 0x4f, 0xb3, 0x0c, 0xcf, 0xcc, 0xe1, 0x02, 0x5a, 0xf6, 0xe9, 0x14, 0xbe, 0x82, 0x06, 0x27, 0xec, 0x1f, 0xa9, 0xea, 0x63,
0xea, 0xe8, 0x74, 0x5c, 0xff, 0xd2, 0x3c, 0xe4, 0xff, 0xe3, 0xe6, 0x77, 0x00, 0x00, 0x00, 0xff, 0xbd, 0x5f, 0x71, 0x85, 0x83, 0x97, 0xd0, 0xf8, 0x98, 0x17, 0x05, 0x9e, 0xd6, 0xd1, 0xc9, 0xa8,
0xff, 0x11, 0x4c, 0x9d, 0x1a, 0x00, 0x05, 0x00, 0x00, 0xfe, 0xa5, 0x2c, 0xe4, 0xff, 0xe6, 0xfa, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x00, 0xdd,
0xf1, 0x18, 0x05, 0x00, 0x00,
} }

1
proto/route.proto
View File

@ -121,6 +121,7 @@ message Backend {
string domain = 4; string domain = 4;
float phi = 5; float phi = 5;
string host = 6; string host = 6;
bool usable = 7;
} }
message BackendList { message BackendList {