Skip to content

add optional IP anonymization #2798

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions context.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ type Context interface {
// The behavior can be configured using `Echo#IPExtractor`.
RealIP() string

// AnonymizedIP returns the client's network address with anonymization applied.
// It uses RealIP() to get the original IP address and then applies anonymization.
AnonymizedIP() string

// Path returns the registered path for the handler.
Path() string

Expand Down Expand Up @@ -317,6 +321,32 @@ func (c *context) RealIP() string {
return ra
}

func (c *context) AnonymizedIP() string {
ip := c.RealIP()

if ip == "" {
return "" // safeguard
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return ip // safeguard, not an ip
}

// IPv4
if parsedIP.To4() != nil {
ipParts := strings.Split(parsedIP.String(), ".")
if len(ipParts) == 4 {
ipParts[3] = "0"
return strings.Join(ipParts, ".")
}
}

// IPv6
ipParts := strings.Split(parsedIP.String(), ":")
ipParts[len(ipParts)-1] = "0"
return strings.Join(ipParts, ":")
}

func (c *context) Path() string {
return c.path
}
Expand Down
89 changes: 89 additions & 0 deletions context_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1121,3 +1121,92 @@ func TestContext_RealIP(t *testing.T) {
assert.Equal(t, tt.s, tt.c.RealIP())
}
}

func TestContext_AnonymizedIP(t *testing.T) {
tests := []struct {
c Context
s string
}{
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"127.0.0.1, 127.0.1.1, "}},
},
},
"127.0.0.0",
},
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"127.0.0.1,127.0.1.1"}},
},
},
"127.0.0.0",
},
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"127.0.0.1"}},
},
},
"127.0.0.0",
},
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"[2001:db8:85a3:8d3:1319:8a2e:370:7348], 2001:db8::1, "}},
},
},
"2001:db8:85a3:8d3:1319:8a2e:370:0",
},
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"[2001:db8:85a3:8d3:1319:8a2e:370:7348],[2001:db8::1]"}},
},
},
"2001:db8:85a3:8d3:1319:8a2e:370:0",
},
{
&context{
request: &http.Request{
Header: http.Header{HeaderXForwardedFor: []string{"2001:db8:85a3:8d3:1319:8a2e:370:7348"}},
},
},
"2001:db8:85a3:8d3:1319:8a2e:370:0",
},
{
&context{
request: &http.Request{
Header: http.Header{
"X-Real-Ip": []string{"192.168.0.1"},
},
},
},
"192.168.0.0",
},
{
&context{
request: &http.Request{
Header: http.Header{
"X-Real-Ip": []string{"[2001:db8::1]"},
},
},
},
"2001:db8::0",
},

{
&context{
request: &http.Request{
RemoteAddr: "89.89.89.89:1654",
},
},
"89.89.89.0",
},
}

for _, tt := range tests {
assert.Equal(t, tt.s, tt.c.AnonymizedIP())
}
}
3 changes: 3 additions & 0 deletions middleware/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type LoggerConfig struct {
// - time_custom
// - id (Request ID)
// - remote_ip
// - remote_ip_anon (Anonymized remote IP address, e.g. "1.2.3.0")
// - uri
// - host
// - method
Expand Down Expand Up @@ -161,6 +162,8 @@ func LoggerWithConfig(config LoggerConfig) echo.MiddlewareFunc {
return buf.WriteString(id)
case "remote_ip":
return buf.WriteString(c.RealIP())
case "remote_ip_anon":
return buf.WriteString(c.AnonymizedIP())
case "host":
return buf.WriteString(req.Host)
case "uri":
Expand Down
8 changes: 4 additions & 4 deletions middleware/logger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func TestLoggerTemplate(t *testing.T) {

e := echo.New()
e.Use(LoggerWithConfig(LoggerConfig{
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","user_agent":"${user_agent}",` +
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","remote_ip_anon":"${remote_ip_anon}","host":"${host}","user_agent":"${user_agent}",` +
`"method":"${method}","uri":"${uri}","status":${status}, "latency":${latency},` +
`"latency_human":"${latency_human}","bytes_in":${bytes_in}, "path":"${path}", "route":"${route}", "referer":"${referer}",` +
`"bytes_out":${bytes_out},"ch":"${header:X-Custom-Header}", "protocol":"${protocol}"` +
Expand Down Expand Up @@ -151,7 +151,7 @@ func TestLoggerCustomTimestamp(t *testing.T) {
customTimeFormat := "2006-01-02 15:04:05.00000"
e := echo.New()
e.Use(LoggerWithConfig(LoggerConfig{
Format: `{"time":"${time_custom}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","user_agent":"${user_agent}",` +
Format: `{"time":"${time_custom}","id":"${id}","remote_ip":"${remote_ip}","remote_ip_anon":"${remote_ip_anon}","host":"${host}","user_agent":"${user_agent}",` +
`"method":"${method}","uri":"${uri}","status":${status}, "latency":${latency},` +
`"latency_human":"${latency_human}","bytes_in":${bytes_in}, "path":"${path}", "referer":"${referer}",` +
`"bytes_out":${bytes_out},"ch":"${header:X-Custom-Header}",` +
Expand Down Expand Up @@ -204,7 +204,7 @@ func BenchmarkLoggerWithConfig_withoutMapFields(b *testing.B) {

buf := new(bytes.Buffer)
mw := LoggerWithConfig(LoggerConfig{
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","user_agent":"${user_agent}",` +
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","remote_ip_anon":"${remote_ip_anon}","host":"${host}","user_agent":"${user_agent}",` +
`"method":"${method}","uri":"${uri}","status":${status}, "latency":${latency},` +
`"latency_human":"${latency_human}","bytes_in":${bytes_in}, "path":"${path}", "referer":"${referer}",` +
`"bytes_out":${bytes_out}, "protocol":"${protocol}"}` + "\n",
Expand Down Expand Up @@ -240,7 +240,7 @@ func BenchmarkLoggerWithConfig_withMapFields(b *testing.B) {

buf := new(bytes.Buffer)
mw := LoggerWithConfig(LoggerConfig{
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","host":"${host}","user_agent":"${user_agent}",` +
Format: `{"time":"${time_rfc3339_nano}","id":"${id}","remote_ip":"${remote_ip}","remote_ip_anon":"${remote_ip_anon}","host":"${host}","user_agent":"${user_agent}",` +
`"method":"${method}","uri":"${uri}","status":${status}, "latency":${latency},` +
`"latency_human":"${latency_human}","bytes_in":${bytes_in}, "path":"${path}", "referer":"${referer}",` +
`"bytes_out":${bytes_out},"ch":"${header:X-Custom-Header}", "protocol":"${protocol}"` +
Expand Down
8 changes: 7 additions & 1 deletion middleware/request_logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ type RequestLoggerConfig struct {
LogProtocol bool
// LogRemoteIP instructs logger to extract request remote IP. See `echo.Context.RealIP()` for implementation details.
LogRemoteIP bool
// AnonymizeRemoteIP instructs logger to anonymize remote IP. See `echo.Context.AnonymizedIP()` for implementation details.
AnonymizeRemoteIP bool
// LogHost instructs logger to extract request host value (i.e. `example.com`)
LogHost bool
// LogMethod instructs logger to extract request method value (i.e. `GET` etc)
Expand Down Expand Up @@ -298,7 +300,11 @@ func (config RequestLoggerConfig) ToMiddleware() (echo.MiddlewareFunc, error) {
v.Protocol = req.Proto
}
if config.LogRemoteIP {
v.RemoteIP = c.RealIP()
if config.AnonymizeRemoteIP {
v.RemoteIP = c.AnonymizedIP()
} else {
v.RemoteIP = c.RealIP()
}
}
if config.LogHost {
v.Host = req.Host
Expand Down
44 changes: 23 additions & 21 deletions middleware/request_logger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@
package middleware

import (
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"

"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
)

func TestRequestLoggerWithConfig(t *testing.T) {
Expand Down Expand Up @@ -292,24 +293,25 @@ func TestRequestLogger_allFields(t *testing.T) {
expect = values
return nil
},
LogLatency: true,
LogProtocol: true,
LogRemoteIP: true,
LogHost: true,
LogMethod: true,
LogURI: true,
LogURIPath: true,
LogRoutePath: true,
LogRequestID: true,
LogReferer: true,
LogUserAgent: true,
LogStatus: true,
LogError: true,
LogContentLength: true,
LogResponseSize: true,
LogHeaders: []string{"accept-encoding", "User-Agent"},
LogQueryParams: []string{"lang", "checked"},
LogFormValues: []string{"csrf", "multiple"},
LogLatency: true,
LogProtocol: true,
LogRemoteIP: true,
LogHost: true,
LogMethod: true,
LogURI: true,
LogURIPath: true,
LogRoutePath: true,
LogRequestID: true,
LogReferer: true,
LogUserAgent: true,
LogStatus: true,
LogError: true,
LogContentLength: true,
LogResponseSize: true,
LogHeaders: []string{"accept-encoding", "User-Agent"},
LogQueryParams: []string{"lang", "checked"},
LogFormValues: []string{"csrf", "multiple"},
AnonymizeRemoteIP: true,
timeNow: func() time.Time {
if isFirstNowCall {
isFirstNowCall = false
Expand Down Expand Up @@ -346,7 +348,7 @@ func TestRequestLogger_allFields(t *testing.T) {
assert.Equal(t, time.Unix(1631045377, 0), expect.StartTime)
assert.Equal(t, 10*time.Second, expect.Latency)
assert.Equal(t, "HTTP/1.1", expect.Protocol)
assert.Equal(t, "8.8.8.8", expect.RemoteIP)
assert.Equal(t, "8.8.8.0", expect.RemoteIP)
assert.Equal(t, "example.com", expect.Host)
assert.Equal(t, http.MethodPost, expect.Method)
assert.Equal(t, "/test?lang=en&checked=1&checked=2", expect.URI)
Expand Down