From 62c711c66e49e892d7d7ed0592823b842d3cff46 Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Sat, 15 Jun 2024 02:08:17 +0300 Subject: [PATCH 1/6] core: add modular `network_proxy` support Co-authored-by: @ImpostorKeanu Signed-off-by: Mohammed Al Sahaf --- .../caddyhttp/reverseproxy/httptransport.go | 19 ++- modules/caddytls/acmeissuer.go | 19 ++- network.go | 128 ++++++++++++++++++ 3 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 network.go diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 80a498066..ab3a47d94 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -135,6 +135,9 @@ type HTTPTransport struct { // The pre-configured underlying HTTP transport. Transport *http.Transport `json:"-"` + // Forward proxy module + NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` + h2cTransport *http2.Transport h3Transport *http3.RoundTripper // TODO: EXPERIMENTAL (May 2024) } @@ -297,7 +300,19 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e } // negotiate any HTTP/SOCKS proxy for the HTTP transport - var proxy func(*http.Request) (*url.URL, error) + proxy := http.ProxyFromEnvironment + if len(h.NetworkProxyRaw) != 0 { + proxyMod, err := caddyCtx.LoadModule(h, "ForwardProxyRaw") + if err != nil { + return nil, fmt.Errorf("failed to load network_proxy module: %v", err) + } + if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok { + proxy = m.ProxyFunc() + } else { + return nil, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``") + } + } + if h.ForwardProxyURL != "" { pUrl, err := url.Parse(h.ForwardProxyURL) if err != nil { @@ -305,8 +320,6 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e } caddyCtx.Logger().Info("setting transport proxy url", zap.String("url", h.ForwardProxyURL)) proxy = http.ProxyURL(pUrl) - } else { - proxy = http.ProxyFromEnvironment } rt := &http.Transport{ diff --git a/modules/caddytls/acmeissuer.go b/modules/caddytls/acmeissuer.go index f46e296e3..38fae46eb 100644 --- a/modules/caddytls/acmeissuer.go +++ b/modules/caddytls/acmeissuer.go @@ -97,6 +97,9 @@ type ACMEIssuer struct { // be used. EXPERIMENTAL: Subject to change. CertificateLifetime caddy.Duration `json:"certificate_lifetime,omitempty"` + // Forward proxy module + NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` + rootPool *x509.CertPool logger *zap.Logger @@ -170,7 +173,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error { } var err error - iss.template, err = iss.makeIssuerTemplate() + iss.template, err = iss.makeIssuerTemplate(ctx) if err != nil { return err } @@ -178,7 +181,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error { return nil } -func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) { +func (iss *ACMEIssuer) makeIssuerTemplate(ctx caddy.Context) (certmagic.ACMEIssuer, error) { template := certmagic.ACMEIssuer{ CA: iss.CA, TestCA: iss.TestCA, @@ -191,6 +194,18 @@ func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) { Logger: iss.logger, } + if len(iss.NetworkProxyRaw) != 0 { + proxyMod, err := ctx.LoadModule(iss, "ForwardProxyRaw") + if err != nil { + return template, fmt.Errorf("failed to load network_proxy module: %v", err) + } + if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok { + template.HTTPProxy = m.ProxyFunc() + } else { + return template, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``") + } + } + if iss.Challenges != nil { if iss.Challenges.HTTP != nil { template.DisableHTTPChallenge = iss.Challenges.HTTP.Disabled diff --git a/network.go b/network.go new file mode 100644 index 000000000..72c612762 --- /dev/null +++ b/network.go @@ -0,0 +1,128 @@ +package caddy + +import ( + "errors" + "net/http" + "net/url" + "strings" + + "go.uber.org/zap" +) + +func init() { + RegisterModule(ProxyFromEnvironment{}) + RegisterModule(ProxyFromURL{}) +} + +type ProxyFuncProducer interface { + ProxyFunc() func(*http.Request) (*url.URL, error) +} + +type ProxyFromEnvironment struct{} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromEnvironment) ProxyFunc() func(*http.Request) (*url.URL, error) { + return http.ProxyFromEnvironment +} + +// CaddyModule implements Module. +func (p ProxyFromEnvironment) CaddyModule() ModuleInfo { + return ModuleInfo{ + ID: "caddy.network_proxy.source.environment", + New: func() Module { + return ProxyFromEnvironment{} + }, + } +} + +type ProxyFromURL struct { + URL string `json:"url"` + + ctx Context + logger *zap.Logger +} + +// CaddyModule implements Module. +func (p ProxyFromURL) CaddyModule() ModuleInfo { + return ModuleInfo{ + ID: "caddy.network_proxy.source.url", + New: func() Module { + return &ProxyFromURL{} + }, + } +} + +func (p *ProxyFromURL) Provision(ctx Context) error { + p.ctx = ctx + p.logger = ctx.Logger() + return nil +} + +// Validate implements Validator. +func (p ProxyFromURL) Validate() error { + if _, err := url.Parse(p.URL); err != nil { + return err + } + return nil +} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { + if strings.Contains(p.URL, "{") && strings.Contains(p.URL, "}") { + // courtesy of @ImpostorKeanu: https://github.com/caddyserver/caddy/pull/6397 + return func(r *http.Request) (*url.URL, error) { + // retrieve the replacer from context. + repl, ok := r.Context().Value(ReplacerCtxKey).(*Replacer) + if !ok { + err := errors.New("failed to obtain replacer from request") + p.logger.Error(err.Error()) + return nil, err + } + + // apply placeholders to the value + // note: h.ForwardProxyURL should never be empty at this point + s := repl.ReplaceAll(p.URL, "") + if s == "" { + p.logger.Error("forward_proxy_url was empty after applying placeholders", + zap.String("initial_value", p.URL), + zap.String("final_value", s), + zap.String("hint", "check for invalid placeholders")) + return nil, errors.New("empty value for forward_proxy_url") + } + + // parse the url + pUrl, err := url.Parse(s) + if err != nil { + p.logger.Warn("failed to derive transport proxy from forward_proxy_url") + pUrl = nil + } else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" { + // url.Parse does not return an error on these values: + // + // - http://:80 + // - pUrl.Host == ":80" + // - /some/path + // - pUrl.Host == "" + // + // Super edge cases, but humans are human. + err = errors.New("supplied forward_proxy_url is missing a host value") + pUrl = nil + } else { + p.logger.Debug("setting transport proxy url", zap.String("url", s)) + } + + return pUrl, err + } + } + return func(*http.Request) (*url.URL, error) { + return url.Parse(p.URL) + } +} + +var ( + _ Module = ProxyFromEnvironment{} + _ ProxyFuncProducer = ProxyFromEnvironment{} + _ Module = ProxyFromURL{} + _ Provisioner = &ProxyFromURL{} + _ Validator = ProxyFromURL{} + _ ProxyFuncProducer = ProxyFromURL{} +) From 54367746a3cfeb0f22d48c05e27ad2b859839210 Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Sat, 15 Jun 2024 14:08:27 +0300 Subject: [PATCH 2/6] move modules around Signed-off-by: Mohammed Al Sahaf --- modules/standard/proxy.go | 126 ++++++++++++++++++++++++++++++++++++++ network.go | 118 ----------------------------------- 2 files changed, 126 insertions(+), 118 deletions(-) create mode 100644 modules/standard/proxy.go diff --git a/modules/standard/proxy.go b/modules/standard/proxy.go new file mode 100644 index 000000000..d3ada40ea --- /dev/null +++ b/modules/standard/proxy.go @@ -0,0 +1,126 @@ +package standard + +import ( + "errors" + "net/http" + "net/url" + "strings" + + "go.uber.org/zap" + + "github.com/caddyserver/caddy/v2" +) + +func init() { + caddy.RegisterModule(ProxyFromEnvironment{}) + caddy.RegisterModule(ProxyFromURL{}) +} + +type ProxyFromEnvironment struct{} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromEnvironment) ProxyFunc() func(*http.Request) (*url.URL, error) { + return http.ProxyFromEnvironment +} + +// CaddyModule implements Module. +func (p ProxyFromEnvironment) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.network_proxy.source.environment", + New: func() caddy.Module { + return ProxyFromEnvironment{} + }, + } +} + +type ProxyFromURL struct { + URL string `json:"url"` + + ctx caddy.Context + logger *zap.Logger +} + +// CaddyModule implements Module. +func (p ProxyFromURL) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.network_proxy.source.url", + New: func() caddy.Module { + return &ProxyFromURL{} + }, + } +} + +func (p *ProxyFromURL) Provision(ctx caddy.Context) error { + p.ctx = ctx + p.logger = ctx.Logger() + return nil +} + +// Validate implements Validator. +func (p ProxyFromURL) Validate() error { + if _, err := url.Parse(p.URL); err != nil { + return err + } + return nil +} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { + if strings.Contains(p.URL, "{") && strings.Contains(p.URL, "}") { + // courtesy of @ImpostorKeanu: https://github.com/caddyserver/caddy/pull/6397 + return func(r *http.Request) (*url.URL, error) { + // retrieve the replacer from context. + repl, ok := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer) + if !ok { + err := errors.New("failed to obtain replacer from request") + p.logger.Error(err.Error()) + return nil, err + } + + // apply placeholders to the value + // note: h.ForwardProxyURL should never be empty at this point + s := repl.ReplaceAll(p.URL, "") + if s == "" { + p.logger.Error("forward_proxy_url was empty after applying placeholders", + zap.String("initial_value", p.URL), + zap.String("final_value", s), + zap.String("hint", "check for invalid placeholders")) + return nil, errors.New("empty value for forward_proxy_url") + } + + // parse the url + pUrl, err := url.Parse(s) + if err != nil { + p.logger.Warn("failed to derive transport proxy from forward_proxy_url") + pUrl = nil + } else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" { + // url.Parse does not return an error on these values: + // + // - http://:80 + // - pUrl.Host == ":80" + // - /some/path + // - pUrl.Host == "" + // + // Super edge cases, but humans are human. + err = errors.New("supplied forward_proxy_url is missing a host value") + pUrl = nil + } else { + p.logger.Debug("setting transport proxy url", zap.String("url", s)) + } + + return pUrl, err + } + } + return func(*http.Request) (*url.URL, error) { + return url.Parse(p.URL) + } +} + +var ( + _ caddy.Module = ProxyFromEnvironment{} + _ caddy.ProxyFuncProducer = ProxyFromEnvironment{} + _ caddy.Module = ProxyFromURL{} + _ caddy.Provisioner = &ProxyFromURL{} + _ caddy.Validator = ProxyFromURL{} + _ caddy.ProxyFuncProducer = ProxyFromURL{} +) diff --git a/network.go b/network.go index 72c612762..c7f65ecbe 100644 --- a/network.go +++ b/network.go @@ -1,128 +1,10 @@ package caddy import ( - "errors" "net/http" "net/url" - "strings" - - "go.uber.org/zap" ) -func init() { - RegisterModule(ProxyFromEnvironment{}) - RegisterModule(ProxyFromURL{}) -} - type ProxyFuncProducer interface { ProxyFunc() func(*http.Request) (*url.URL, error) } - -type ProxyFromEnvironment struct{} - -// ProxyFunc implements ProxyFuncProducer. -func (p ProxyFromEnvironment) ProxyFunc() func(*http.Request) (*url.URL, error) { - return http.ProxyFromEnvironment -} - -// CaddyModule implements Module. -func (p ProxyFromEnvironment) CaddyModule() ModuleInfo { - return ModuleInfo{ - ID: "caddy.network_proxy.source.environment", - New: func() Module { - return ProxyFromEnvironment{} - }, - } -} - -type ProxyFromURL struct { - URL string `json:"url"` - - ctx Context - logger *zap.Logger -} - -// CaddyModule implements Module. -func (p ProxyFromURL) CaddyModule() ModuleInfo { - return ModuleInfo{ - ID: "caddy.network_proxy.source.url", - New: func() Module { - return &ProxyFromURL{} - }, - } -} - -func (p *ProxyFromURL) Provision(ctx Context) error { - p.ctx = ctx - p.logger = ctx.Logger() - return nil -} - -// Validate implements Validator. -func (p ProxyFromURL) Validate() error { - if _, err := url.Parse(p.URL); err != nil { - return err - } - return nil -} - -// ProxyFunc implements ProxyFuncProducer. -func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { - if strings.Contains(p.URL, "{") && strings.Contains(p.URL, "}") { - // courtesy of @ImpostorKeanu: https://github.com/caddyserver/caddy/pull/6397 - return func(r *http.Request) (*url.URL, error) { - // retrieve the replacer from context. - repl, ok := r.Context().Value(ReplacerCtxKey).(*Replacer) - if !ok { - err := errors.New("failed to obtain replacer from request") - p.logger.Error(err.Error()) - return nil, err - } - - // apply placeholders to the value - // note: h.ForwardProxyURL should never be empty at this point - s := repl.ReplaceAll(p.URL, "") - if s == "" { - p.logger.Error("forward_proxy_url was empty after applying placeholders", - zap.String("initial_value", p.URL), - zap.String("final_value", s), - zap.String("hint", "check for invalid placeholders")) - return nil, errors.New("empty value for forward_proxy_url") - } - - // parse the url - pUrl, err := url.Parse(s) - if err != nil { - p.logger.Warn("failed to derive transport proxy from forward_proxy_url") - pUrl = nil - } else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" { - // url.Parse does not return an error on these values: - // - // - http://:80 - // - pUrl.Host == ":80" - // - /some/path - // - pUrl.Host == "" - // - // Super edge cases, but humans are human. - err = errors.New("supplied forward_proxy_url is missing a host value") - pUrl = nil - } else { - p.logger.Debug("setting transport proxy url", zap.String("url", s)) - } - - return pUrl, err - } - } - return func(*http.Request) (*url.URL, error) { - return url.Parse(p.URL) - } -} - -var ( - _ Module = ProxyFromEnvironment{} - _ ProxyFuncProducer = ProxyFromEnvironment{} - _ Module = ProxyFromURL{} - _ Provisioner = &ProxyFromURL{} - _ Validator = ProxyFromURL{} - _ ProxyFuncProducer = ProxyFromURL{} -) From 18cd1c0525a00363a91e9cecd38572607c52cedb Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Sat, 24 Aug 2024 15:54:19 +0300 Subject: [PATCH 3/6] add caddyfile implementation Signed-off-by: Mohammed Al Sahaf --- ...everse_proxy_http_transport_none_proxy.txt | 40 +++++++++++++ ...reverse_proxy_http_transport_url_proxy.txt | 41 +++++++++++++ modules/caddyhttp/reverseproxy/caddyfile.go | 16 +++++ modules/caddytls/proxy.go | 3 + .../network/networkproxy.go} | 60 ++++++++++++------- 5 files changed, 138 insertions(+), 22 deletions(-) create mode 100644 caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt create mode 100644 caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt create mode 100644 modules/caddytls/proxy.go rename modules/{standard/proxy.go => internal/network/networkproxy.go} (75%) diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt new file mode 100644 index 000000000..6d41ed5b9 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt @@ -0,0 +1,40 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + forward_proxy none + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "none" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt new file mode 100644 index 000000000..c4134eec1 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt @@ -0,0 +1,41 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + forward_proxy url http://localhost:8080 + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "url", + "url": "http://localhost:8080" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index 12e2b9b97..9fe6a1ebe 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -1074,6 +1074,22 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } h.ForwardProxyURL = d.Val() + case "forward_proxy": + if !d.NextArg() { + return d.ArgErr() + } + modStem := d.Val() + modID := "caddy.network_proxy.source." + modStem + unm, err := caddyfile.UnmarshalModule(d, modID) + if err != nil { + return err + } + // TODO: should we validate here? + // ca, ok := unm.(caddy.ProxyFuncProducer) + // if !ok { + // return d.Errf("module %s is not a caddy.ProxyFuncProducer", modID) + // } + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(unm, "from", modStem, nil) case "dial_timeout": if !d.NextArg() { return d.ArgErr() diff --git a/modules/caddytls/proxy.go b/modules/caddytls/proxy.go new file mode 100644 index 000000000..7a5e9c5c7 --- /dev/null +++ b/modules/caddytls/proxy.go @@ -0,0 +1,3 @@ +package caddytls + +import _ "github.com/caddyserver/caddy/v2/modules/internal/network" diff --git a/modules/standard/proxy.go b/modules/internal/network/networkproxy.go similarity index 75% rename from modules/standard/proxy.go rename to modules/internal/network/networkproxy.go index d3ada40ea..f75168e81 100644 --- a/modules/standard/proxy.go +++ b/modules/internal/network/networkproxy.go @@ -1,4 +1,4 @@ -package standard +package network import ( "errors" @@ -9,28 +9,12 @@ import ( "go.uber.org/zap" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" ) func init() { - caddy.RegisterModule(ProxyFromEnvironment{}) caddy.RegisterModule(ProxyFromURL{}) -} - -type ProxyFromEnvironment struct{} - -// ProxyFunc implements ProxyFuncProducer. -func (p ProxyFromEnvironment) ProxyFunc() func(*http.Request) (*url.URL, error) { - return http.ProxyFromEnvironment -} - -// CaddyModule implements Module. -func (p ProxyFromEnvironment) CaddyModule() caddy.ModuleInfo { - return caddy.ModuleInfo{ - ID: "caddy.network_proxy.source.environment", - New: func() caddy.Module { - return ProxyFromEnvironment{} - }, - } + caddy.RegisterModule(ProxyFromNone{}) } type ProxyFromURL struct { @@ -111,16 +95,48 @@ func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { return pUrl, err } } - return func(*http.Request) (*url.URL, error) { + return func(r *http.Request) (*url.URL, error) { return url.Parse(p.URL) } } +// UnmarshalCaddyfile implements caddyfile.Unmarshaler. +func (p *ProxyFromURL) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + d.Next() + d.Next() + p.URL = d.Val() + return nil +} + +type ProxyFromNone struct{} + +func (p ProxyFromNone) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.network_proxy.source.none", + New: func() caddy.Module { + return &ProxyFromNone{} + }, + } +} + +// UnmarshalCaddyfile implements caddyfile.Unmarshaler. +func (p ProxyFromNone) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + return nil +} + +// ProxyFunc implements ProxyFuncProducer. +func (p ProxyFromNone) ProxyFunc() func(*http.Request) (*url.URL, error) { + return nil +} + var ( - _ caddy.Module = ProxyFromEnvironment{} - _ caddy.ProxyFuncProducer = ProxyFromEnvironment{} _ caddy.Module = ProxyFromURL{} _ caddy.Provisioner = &ProxyFromURL{} _ caddy.Validator = ProxyFromURL{} _ caddy.ProxyFuncProducer = ProxyFromURL{} + _ caddyfile.Unmarshaler = &ProxyFromURL{} + + _ caddy.Module = ProxyFromNone{} + _ caddy.ProxyFuncProducer = ProxyFromNone{} + _ caddyfile.Unmarshaler = ProxyFromNone{} ) From 38bb05c6c6b99dce2d543df7183a92817c8d0caa Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Mon, 26 Aug 2024 17:46:27 +0000 Subject: [PATCH 4/6] address feedbcak --- .../reverse_proxy_http_transport_none_proxy.txt | 2 +- .../reverse_proxy_http_transport_url_proxy.txt | 2 +- modules/caddyhttp/reverseproxy/caddyfile.go | 17 +++++++++-------- modules/caddyhttp/reverseproxy/httptransport.go | 17 ++++++++++++++++- modules/internal/network/networkproxy.go | 10 ++++++---- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt index 6d41ed5b9..3805448d9 100644 --- a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt @@ -1,7 +1,7 @@ :8884 reverse_proxy 127.0.0.1:65535 { transport http { - forward_proxy none + network_proxy none } } ---------- diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt index c4134eec1..9397458e9 100644 --- a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt @@ -1,7 +1,7 @@ :8884 reverse_proxy 127.0.0.1:65535 { transport http { - forward_proxy url http://localhost:8080 + network_proxy url http://localhost:8080 } } ---------- diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index 9fe6a1ebe..59a7ff756 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -980,7 +980,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // read_buffer // write_buffer // max_response_header -// forward_proxy_url +// network_proxy { +// ... +// } // dial_timeout // dial_fallback_delay // response_header_timeout @@ -991,6 +993,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // tls_insecure_skip_verify // tls_timeout // tls_trusted_ca_certs +// tls_trust_pool { +// ... +// } // tls_server_name // tls_renegotiation // tls_except_ports @@ -1069,12 +1074,13 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } case "forward_proxy_url": + caddy.Log().Warn("The 'forward_proxy_url' field is deprecated. Use the 'network_proxy' field instead.") if !d.NextArg() { return d.ArgErr() } - h.ForwardProxyURL = d.Val() - case "forward_proxy": + h.ForwardProxyURL = d.Val() + case "network_proxy": if !d.NextArg() { return d.ArgErr() } @@ -1084,11 +1090,6 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if err != nil { return err } - // TODO: should we validate here? - // ca, ok := unm.(caddy.ProxyFuncProducer) - // if !ok { - // return d.Errf("module %s is not a caddy.ProxyFuncProducer", modID) - // } h.NetworkProxyRaw = caddyconfig.JSONModuleObject(unm, "from", modStem, nil) case "dial_timeout": if !d.NextArg() { diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 55b25a48f..e9e31b1ca 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -88,6 +88,7 @@ type HTTPTransport struct { // forward_proxy_url -> upstream // // Default: http.ProxyFromEnvironment + // DEPRECATED: Use NetworkProxyRaw|`network_proxy` instead. Subject to removal. ForwardProxyURL string `json:"forward_proxy_url,omitempty"` // How long to wait before timing out trying to connect to @@ -139,7 +140,20 @@ type HTTPTransport struct { // The pre-configured underlying HTTP transport. Transport *http.Transport `json:"-"` - // Forward proxy module + // The module that provides the network (forward) proxy + // URL that the HTTP transport will use to proxy + // requests to the upstream. See [http.Transport.Proxy](https://pkg.go.dev/net/http#Transport.Proxy) + // for information regarding supported protocols. + // + // Providing a value to this parameter results in + // requests flowing through the reverse_proxy in the following + // way: + // + // User Agent -> + // reverse_proxy -> + // [proxy provided by the module] -> upstream + // + // If nil/empty, default to reading the `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY`. NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` h2cTransport *http2.Transport @@ -343,6 +357,7 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e } if h.ForwardProxyURL != "" { + caddyCtx.Logger().Warn("forward_proxy_url is deprecated; use network_proxy instead") pUrl, err := url.Parse(h.ForwardProxyURL) if err != nil { return nil, fmt.Errorf("failed to parse transport proxy url: %v", err) diff --git a/modules/internal/network/networkproxy.go b/modules/internal/network/networkproxy.go index f75168e81..fa0f5009a 100644 --- a/modules/internal/network/networkproxy.go +++ b/modules/internal/network/networkproxy.go @@ -17,6 +17,7 @@ func init() { caddy.RegisterModule(ProxyFromNone{}) } +// The "url" proxy source uses the defined URL as the proxy type ProxyFromURL struct { URL string `json:"url"` @@ -65,17 +66,17 @@ func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { // note: h.ForwardProxyURL should never be empty at this point s := repl.ReplaceAll(p.URL, "") if s == "" { - p.logger.Error("forward_proxy_url was empty after applying placeholders", + p.logger.Error("network_proxy URL was empty after applying placeholders", zap.String("initial_value", p.URL), zap.String("final_value", s), zap.String("hint", "check for invalid placeholders")) - return nil, errors.New("empty value for forward_proxy_url") + return nil, errors.New("empty value for network_proxy URL") } // parse the url pUrl, err := url.Parse(s) if err != nil { - p.logger.Warn("failed to derive transport proxy from forward_proxy_url") + p.logger.Warn("failed to derive transport proxy from network_proxy URL") pUrl = nil } else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" { // url.Parse does not return an error on these values: @@ -86,7 +87,7 @@ func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) { // - pUrl.Host == "" // // Super edge cases, but humans are human. - err = errors.New("supplied forward_proxy_url is missing a host value") + err = errors.New("supplied network_proxy URL is missing a host value") pUrl = nil } else { p.logger.Debug("setting transport proxy url", zap.String("url", s)) @@ -108,6 +109,7 @@ func (p *ProxyFromURL) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil } +// The "none" proxy source module disables the use of network proxy. type ProxyFromNone struct{} func (p ProxyFromNone) CaddyModule() caddy.ModuleInfo { From 908fa5e103eeb9ca13ec22a31a8814d20cfcc8ea Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Tue, 27 Aug 2024 10:48:59 +0300 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Francis Lavoie --- modules/caddyhttp/reverseproxy/caddyfile.go | 16 +++++++++------- modules/caddyhttp/reverseproxy/httptransport.go | 8 ++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index 59a7ff756..4e8ea674b 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -980,9 +980,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // read_buffer // write_buffer // max_response_header -// network_proxy { -// ... -// } +// network_proxy { +// ... +// } // dial_timeout // dial_fallback_delay // response_header_timeout @@ -993,9 +993,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // tls_insecure_skip_verify // tls_timeout // tls_trusted_ca_certs -// tls_trust_pool { -// ... -// } +// tls_trust_pool { +// ... +// } // tls_server_name // tls_renegotiation // tls_except_ports @@ -1074,12 +1074,13 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } case "forward_proxy_url": - caddy.Log().Warn("The 'forward_proxy_url' field is deprecated. Use the 'network_proxy' field instead.") + caddy.Log().Warn("The 'forward_proxy_url' field is deprecated. Use 'network_proxy url ' instead.") if !d.NextArg() { return d.ArgErr() } h.ForwardProxyURL = d.Val() + case "network_proxy": if !d.NextArg() { return d.ArgErr() @@ -1091,6 +1092,7 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return err } h.NetworkProxyRaw = caddyconfig.JSONModuleObject(unm, "from", modStem, nil) + case "dial_timeout": if !d.NextArg() { return d.ArgErr() diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index e9e31b1ca..1c8911a3c 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -145,15 +145,15 @@ type HTTPTransport struct { // requests to the upstream. See [http.Transport.Proxy](https://pkg.go.dev/net/http#Transport.Proxy) // for information regarding supported protocols. // - // Providing a value to this parameter results in - // requests flowing through the reverse_proxy in the following - // way: + // Providing a value to this parameter results in requests + // flowing through the reverse_proxy in the following way: // // User Agent -> // reverse_proxy -> // [proxy provided by the module] -> upstream // - // If nil/empty, default to reading the `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY`. + // If nil, defaults to reading the `HTTP_PROXY`, + // `HTTPS_PROXY`, and `NO_PROXY` environment variables. NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"` h2cTransport *http2.Transport From b8a9870410b4c3ea42f2313a188b53c7d03cc678 Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Wed, 28 Aug 2024 00:47:04 +0300 Subject: [PATCH 6/6] adapt ForwardProxyURL to use the NetworkProxyRaw Signed-off-by: Mohammed Al Sahaf --- ...proxy_http_transport_forward_proxy_url.txt | 41 +++++++++++++++++++ modules/caddyhttp/reverseproxy/caddyfile.go | 5 ++- .../caddyhttp/reverseproxy/httptransport.go | 18 ++++---- 3 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt new file mode 100644 index 000000000..9fc445283 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt @@ -0,0 +1,41 @@ +:8884 +reverse_proxy 127.0.0.1:65535 { + transport http { + forward_proxy_url http://localhost:8080 + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "transport": { + "network_proxy": { + "from": "url", + "url": "http://localhost:8080" + }, + "protocol": "http" + }, + "upstreams": [ + { + "dial": "127.0.0.1:65535" + } + ] + } + ] + } + ] + } + } + } + } +} diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index 4e8ea674b..ae35723ea 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -33,6 +33,7 @@ import ( "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/caddyserver/caddy/v2/modules/internal/network" ) func init() { @@ -1078,8 +1079,8 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if !d.NextArg() { return d.ArgErr() } - - h.ForwardProxyURL = d.Val() + u := network.ProxyFromURL{URL: d.Val()} + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil) case "network_proxy": if !d.NextArg() { diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 1c8911a3c..534eef469 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -24,7 +24,6 @@ import ( weakrand "math/rand" "net" "net/http" - "net/url" "os" "reflect" "strings" @@ -36,8 +35,10 @@ import ( "golang.org/x/net/http2" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/caddyserver/caddy/v2/modules/internal/network" ) func init() { @@ -344,6 +345,11 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e // negotiate any HTTP/SOCKS proxy for the HTTP transport proxy := http.ProxyFromEnvironment + if h.ForwardProxyURL != "" { + caddyCtx.Logger().Warn("forward_proxy_url is deprecated; use network_proxy instead") + u := network.ProxyFromURL{URL: h.ForwardProxyURL} + h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil) + } if len(h.NetworkProxyRaw) != 0 { proxyMod, err := caddyCtx.LoadModule(h, "ForwardProxyRaw") if err != nil { @@ -356,16 +362,6 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e } } - if h.ForwardProxyURL != "" { - caddyCtx.Logger().Warn("forward_proxy_url is deprecated; use network_proxy instead") - pUrl, err := url.Parse(h.ForwardProxyURL) - if err != nil { - return nil, fmt.Errorf("failed to parse transport proxy url: %v", err) - } - caddyCtx.Logger().Info("setting transport proxy url", zap.String("url", h.ForwardProxyURL)) - proxy = http.ProxyURL(pUrl) - } - rt := &http.Transport{ Proxy: proxy, DialContext: dialContext,