Merge 0896d92d030a39a50a2e5193bb553613b7536308 into 8861eae22350d9e8f94653db951faf85a50a82da

Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com>

caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_forward_proxy_url.txt

@@ -0,0 +1,41 @@
+:8884
+reverse_proxy 127.0.0.1:65535 {
+	transport http {
+		forward_proxy_url http://localhost:8080
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"network_proxy": {
+											"from": "url",
+											"url": "http://localhost:8080"
+										},
+										"protocol": "http"
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_none_proxy.txt

@@ -0,0 +1,40 @@
+:8884
+reverse_proxy 127.0.0.1:65535 {
+	transport http {
+		network_proxy none
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"network_proxy": {
+											"from": "none"
+										},
+										"protocol": "http"
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/reverse_proxy_http_transport_url_proxy.txt

@@ -0,0 +1,41 @@
+:8884
+reverse_proxy 127.0.0.1:65535 {
+	transport http {
+		network_proxy url http://localhost:8080
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8884"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "reverse_proxy",
+									"transport": {
+										"network_proxy": {
+											"from": "url",
+											"url": "http://localhost:8080"
+										},
+										"protocol": "http"
+									},
+									"upstreams": [
+										{
+											"dial": "127.0.0.1:65535"
+										}
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

modules/caddyhttp/reverseproxy/caddyfile.go

@@ -33,6 +33,7 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/caddyserver/caddy/v2/modules/internal/network"
 )
 
 func init() {
@@ -979,7 +980,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error
 //	    read_buffer             <size>
 //	    write_buffer            <size>
 //	    max_response_header     <size>
-//	    forward_proxy_url       <url>
+//	    network_proxy           <module> {
+//	        ...
+//	    }
 //	    dial_timeout            <duration>
 //	    dial_fallback_delay     <duration>
 //	    response_header_timeout <duration>
@@ -990,6 +993,9 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error
 //	    tls_insecure_skip_verify
 //	    tls_timeout <duration>
 //	    tls_trusted_ca_certs <cert_files...>
+//	    tls_trust_pool <module> {
+//	        ...
+//	    }
 //	    tls_server_name <sni>
 //	    tls_renegotiation <level>
 //	    tls_except_ports <ports...>
@@ -1068,10 +1074,24 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			}
 
 		case "forward_proxy_url":
+			caddy.Log().Warn("The 'forward_proxy_url' field is deprecated. Use 'network_proxy url <url>' instead.")
 			if !d.NextArg() {
 				return d.ArgErr()
 			}
-			h.ForwardProxyURL = d.Val()
+			u := network.ProxyFromURL{URL: d.Val()}
+			h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil)
+
+		case "network_proxy":
+			if !d.NextArg() {
+				return d.ArgErr()
+			}
+			modStem := d.Val()
+			modID := "caddy.network_proxy.source." + modStem
+			unm, err := caddyfile.UnmarshalModule(d, modID)
+			if err != nil {
+				return err
+			}
+			h.NetworkProxyRaw = caddyconfig.JSONModuleObject(unm, "from", modStem, nil)
 
 		case "dial_timeout":
 			if !d.NextArg() {

modules/caddyhttp/reverseproxy/httptransport.go

@@ -24,7 +24,6 @@ import (
 	weakrand "math/rand"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"reflect"
 	"slices"
@@ -38,8 +37,10 @@ import (
 	"golang.org/x/net/http2"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/caddyserver/caddy/v2/modules/internal/network"
 )
 
 func init() {
@@ -90,6 +91,7 @@ type HTTPTransport struct {
 	//  forward_proxy_url -> upstream
 	//
 	// Default: http.ProxyFromEnvironment
+	// DEPRECATED: Use NetworkProxyRaw|`network_proxy` instead. Subject to removal.
 	ForwardProxyURL string `json:"forward_proxy_url,omitempty"`
 
 	// How long to wait before timing out trying to connect to
@@ -141,6 +143,22 @@ type HTTPTransport struct {
 	// The pre-configured underlying HTTP transport.
 	Transport *http.Transport `json:"-"`
 
+	// The module that provides the network (forward) proxy
+	// URL that the HTTP transport will use to proxy
+	// requests to the upstream. See [http.Transport.Proxy](https://pkg.go.dev/net/http#Transport.Proxy)
+	// for information regarding supported protocols.
+	//
+	// Providing a value to this parameter results in requests
+	// flowing through the reverse_proxy in the following way:
+	//
+	// User Agent ->
+	//  reverse_proxy ->
+	//  [proxy provided by the module] -> upstream
+	//
+	// If nil, defaults to reading the `HTTP_PROXY`,
+	// `HTTPS_PROXY`, and `NO_PROXY` environment variables.
+	NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"`
+
 	h2cTransport *http2.Transport
 	h3Transport  *http3.Transport // TODO: EXPERIMENTAL (May 2024)
 }
@@ -328,16 +346,22 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e
 	}
 
 	// negotiate any HTTP/SOCKS proxy for the HTTP transport
-	var proxy func(*http.Request) (*url.URL, error)
+	proxy := http.ProxyFromEnvironment
 	if h.ForwardProxyURL != "" {
-		pUrl, err := url.Parse(h.ForwardProxyURL)
+		caddyCtx.Logger().Warn("forward_proxy_url is deprecated; use network_proxy instead")
+		u := network.ProxyFromURL{URL: h.ForwardProxyURL}
+		h.NetworkProxyRaw = caddyconfig.JSONModuleObject(u, "from", "url", nil)
+	}
+	if len(h.NetworkProxyRaw) != 0 {
+		proxyMod, err := caddyCtx.LoadModule(h, "ForwardProxyRaw")
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse transport proxy url: %v", err)
+			return nil, fmt.Errorf("failed to load network_proxy module: %v", err)
+		}
+		if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok {
+			proxy = m.ProxyFunc()
+		} else {
+			return nil, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``")
 		}
-		caddyCtx.Logger().Info("setting transport proxy url", zap.String("url", h.ForwardProxyURL))
-		proxy = http.ProxyURL(pUrl)
-	} else {
-		proxy = http.ProxyFromEnvironment
 	}
 
 	rt := &http.Transport{

modules/caddytls/acmeissuer.go

@@ -106,6 +106,9 @@ type ACMEIssuer struct {
 	// be used. EXPERIMENTAL: Subject to change.
 	CertificateLifetime caddy.Duration `json:"certificate_lifetime,omitempty"`
 
+	// Forward proxy module
+	NetworkProxyRaw json.RawMessage `json:"network_proxy,omitempty" caddy:"namespace=caddy.network_proxy.source inline_key=from"`
+
 	rootPool *x509.CertPool
 	logger   *zap.Logger
 
@@ -179,7 +182,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error {
 	}
 
 	var err error
-	iss.template, err = iss.makeIssuerTemplate()
+	iss.template, err = iss.makeIssuerTemplate(ctx)
 	if err != nil {
 		return err
 	}
@@ -187,7 +190,7 @@ func (iss *ACMEIssuer) Provision(ctx caddy.Context) error {
 	return nil
 }
 
-func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) {
+func (iss *ACMEIssuer) makeIssuerTemplate(ctx caddy.Context) (certmagic.ACMEIssuer, error) {
 	template := certmagic.ACMEIssuer{
 		CA:                iss.CA,
 		TestCA:            iss.TestCA,
@@ -201,6 +204,18 @@ func (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) {
 		Logger:            iss.logger,
 	}
 
+	if len(iss.NetworkProxyRaw) != 0 {
+		proxyMod, err := ctx.LoadModule(iss, "ForwardProxyRaw")
+		if err != nil {
+			return template, fmt.Errorf("failed to load network_proxy module: %v", err)
+		}
+		if m, ok := proxyMod.(caddy.ProxyFuncProducer); ok {
+			template.HTTPProxy = m.ProxyFunc()
+		} else {
+			return template, fmt.Errorf("network_proxy module is not `(func(*http.Request) (*url.URL, error))``")
+		}
+	}
+
 	if iss.Challenges != nil {
 		if iss.Challenges.HTTP != nil {
 			template.DisableHTTPChallenge = iss.Challenges.HTTP.Disabled

modules/caddytls/proxy.go

@@ -0,0 +1,3 @@
+package caddytls
+
+import _ "github.com/caddyserver/caddy/v2/modules/internal/network"

modules/internal/network/networkproxy.go

@@ -0,0 +1,144 @@
+package network
+
+import (
+	"errors"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"go.uber.org/zap"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func init() {
+	caddy.RegisterModule(ProxyFromURL{})
+	caddy.RegisterModule(ProxyFromNone{})
+}
+
+// The "url" proxy source uses the defined URL as the proxy
+type ProxyFromURL struct {
+	URL string `json:"url"`
+
+	ctx    caddy.Context
+	logger *zap.Logger
+}
+
+// CaddyModule implements Module.
+func (p ProxyFromURL) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID: "caddy.network_proxy.source.url",
+		New: func() caddy.Module {
+			return &ProxyFromURL{}
+		},
+	}
+}
+
+func (p *ProxyFromURL) Provision(ctx caddy.Context) error {
+	p.ctx = ctx
+	p.logger = ctx.Logger()
+	return nil
+}
+
+// Validate implements Validator.
+func (p ProxyFromURL) Validate() error {
+	if _, err := url.Parse(p.URL); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ProxyFunc implements ProxyFuncProducer.
+func (p ProxyFromURL) ProxyFunc() func(*http.Request) (*url.URL, error) {
+	if strings.Contains(p.URL, "{") && strings.Contains(p.URL, "}") {
+		// courtesy of @ImpostorKeanu: https://github.com/caddyserver/caddy/pull/6397
+		return func(r *http.Request) (*url.URL, error) {
+			// retrieve the replacer from context.
+			repl, ok := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+			if !ok {
+				err := errors.New("failed to obtain replacer from request")
+				p.logger.Error(err.Error())
+				return nil, err
+			}
+
+			// apply placeholders to the value
+			// note: h.ForwardProxyURL should never be empty at this point
+			s := repl.ReplaceAll(p.URL, "")
+			if s == "" {
+				p.logger.Error("network_proxy URL was empty after applying placeholders",
+					zap.String("initial_value", p.URL),
+					zap.String("final_value", s),
+					zap.String("hint", "check for invalid placeholders"))
+				return nil, errors.New("empty value for network_proxy URL")
+			}
+
+			// parse the url
+			pUrl, err := url.Parse(s)
+			if err != nil {
+				p.logger.Warn("failed to derive transport proxy from network_proxy URL")
+				pUrl = nil
+			} else if pUrl.Host == "" || strings.Split("", pUrl.Host)[0] == ":" {
+				// url.Parse does not return an error on these values:
+				//
+				// - http://:80
+				//   - pUrl.Host == ":80"
+				// - /some/path
+				//   - pUrl.Host == ""
+				//
+				// Super edge cases, but humans are human.
+				err = errors.New("supplied network_proxy URL is missing a host value")
+				pUrl = nil
+			} else {
+				p.logger.Debug("setting transport proxy url", zap.String("url", s))
+			}
+
+			return pUrl, err
+		}
+	}
+	return func(r *http.Request) (*url.URL, error) {
+		return url.Parse(p.URL)
+	}
+}
+
+// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
+func (p *ProxyFromURL) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	d.Next()
+	d.Next()
+	p.URL = d.Val()
+	return nil
+}
+
+// The "none" proxy source module disables the use of network proxy.
+type ProxyFromNone struct{}
+
+func (p ProxyFromNone) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID: "caddy.network_proxy.source.none",
+		New: func() caddy.Module {
+			return &ProxyFromNone{}
+		},
+	}
+}
+
+// UnmarshalCaddyfile implements caddyfile.Unmarshaler.
+func (p ProxyFromNone) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	return nil
+}
+
+// ProxyFunc implements ProxyFuncProducer.
+func (p ProxyFromNone) ProxyFunc() func(*http.Request) (*url.URL, error) {
+	return nil
+}
+
+var (
+	_ caddy.Module            = ProxyFromURL{}
+	_ caddy.Provisioner       = &ProxyFromURL{}
+	_ caddy.Validator         = ProxyFromURL{}
+	_ caddy.ProxyFuncProducer = ProxyFromURL{}
+	_ caddyfile.Unmarshaler   = &ProxyFromURL{}
+
+	_ caddy.Module            = ProxyFromNone{}
+	_ caddy.ProxyFuncProducer = ProxyFromNone{}
+	_ caddyfile.Unmarshaler   = ProxyFromNone{}
+)

network.go

@@ -0,0 +1,10 @@
+package caddy
+
+import (
+	"net/http"
+	"net/url"
+)
+
+type ProxyFuncProducer interface {
+	ProxyFunc() func(*http.Request) (*url.URL, error)
+}