diff --git a/WORKSPACE b/WORKSPACE index 7e44591d8..e7d1e676e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,7 +26,7 @@ # # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime. -ESP_TOOL = "008e8c0203578d2ee48aa175b58c611fbecc4ca4" +ESP_TOOL = "7c1cac2aa0613f40200acb64342b23823e5a3621" git_repository( name = "nginx", diff --git a/src/api_manager/context/BUILD b/src/api_manager/context/BUILD index 1d82770d8..467ef91fe 100644 --- a/src/api_manager/context/BUILD +++ b/src/api_manager/context/BUILD @@ -56,3 +56,15 @@ cc_library( "//src/api_manager/utils", ], ) + +cc_test( + name = "client_ip_extraction_test", + size = "small", + srcs = [ + "client_ip_extraction_test.cc", + ], + deps = [ + "//src/api_manager:api_manager", + "//external:googletest_main", + ], +) diff --git a/src/api_manager/context/client_ip_extraction_test.cc b/src/api_manager/context/client_ip_extraction_test.cc new file mode 100644 index 000000000..3fd44a8f5 --- /dev/null +++ b/src/api_manager/context/client_ip_extraction_test.cc @@ -0,0 +1,300 @@ +// Copyright (C) Extensible Service Proxy Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +//#include "src/api_manager/mock_api_manager_environment.h" +#include "src/api_manager/context/request_context.h" +#include "src/api_manager/context/service_context.h" +//#include "src/api_manager/mock_request.h" +//#include "src/api_manager/api_manager_impl.h" +#include "include/api_manager/request.h" + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Mock; +using ::testing::Return; + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { +namespace context { + +namespace { + +const char kServiceConfig1[] = + R"( +{ + "name": "bookstore.test.appspot.com", + "title": "Bookstore", + "http": { + "rules": [ + { + "selector": "EchoGetMessage", + "get": "/echo" + } + ] + }, + "usage": { + "rules": [ + { + "selector": "EchoGetMessage", + "allowUnregisteredCalls": true + } + ] + }, + "control": { + "environment": "servicecontrol.googleapis.com" + }, + "id": "2017-05-01r0" +} +)"; + +// Simulate periodic timer event on creation +class MockPeriodicTimer : public PeriodicTimer { + public: + MockPeriodicTimer() {} + MockPeriodicTimer(std::function continuation) + : continuation_(continuation) { + continuation_(); + } + + virtual ~MockPeriodicTimer() {} + void Stop() {} + + private: + std::function continuation_; +}; + +class MockApiManagerEnvironment : public ApiManagerEnvInterface { + public: + virtual ~MockApiManagerEnvironment() {} + + void Log(LogLevel level, const char *message) override { + // std::cout << __FILE__ << ":" << __LINE__ << " " << message << std::endl; + } + std::unique_ptr StartPeriodicTimer( + std::chrono::milliseconds interval, std::function continuation) { + return std::unique_ptr(new MockPeriodicTimer(continuation)); + } + + void RunHTTPRequest(std::unique_ptr request) override {} + + virtual void RunGRPCRequest(std::unique_ptr request) override {} +}; + +class MockRequest : public Request { + public: + MockRequest(const std::string &client_ip, + const std::map &header) + : client_ip_(client_ip), header_(header) {} + + virtual ~MockRequest() {} + + bool FindHeader(const std::string &name, std::string *header) override { + auto it = header_.find(name); + if (it != header_.end()) { + header->assign(it->second); + return true; + } + + return false; + } + + std::string GetClientIP() { return client_ip_; } + + std::string GetRequestHTTPMethod() override { return "GET"; } + + std::string GetQueryParameters() override { return ""; } + + std::string GetUnparsedRequestPath() override { return "/echo"; } + + bool FindQuery(const std::string &name, std::string *query) override { + return false; + } + + MOCK_METHOD2(AddHeaderToBackend, + utils::Status(const std::string &, const std::string &)); + MOCK_METHOD1(SetAuthToken, void(const std::string &)); + MOCK_METHOD0(GetFrontendProtocol, + ::google::api_manager::protocol::Protocol()); + MOCK_METHOD0(GetBackendProtocol, ::google::api_manager::protocol::Protocol()); + MOCK_METHOD0(GetRequestPath, std::string()); + MOCK_METHOD0(GetInsecureCallerID, std::string()); + MOCK_METHOD0(GetRequestHeaders, std::multimap *()); + MOCK_METHOD0(GetGrpcRequestBytes, int64_t()); + MOCK_METHOD0(GetGrpcResponseBytes, int64_t()); + MOCK_METHOD0(GetGrpcRequestMessageCounts, int64_t()); + MOCK_METHOD0(GetGrpcResponseMessageCounts, int64_t()); + + private: + const std::string client_ip_; + const std::map header_; +}; +} + +class ClientIPExtractionTest : public ::testing::Test { + protected: + ClientIPExtractionTest() : callback_run_count_(0) {} + + void SetUp() { + callback_run_count_ = 0; + call_history_.clear(); + } + + protected: + std::vector call_history_; + int callback_run_count_; +}; + +// Extracts client IP address from the request based on the server configuration +std::string ExtractClientIP(std::string serverConfig, std::string remote_ip, + std::map headers) { + std::unique_ptr env(new MockApiManagerEnvironment()); + + std::shared_ptr global_context( + new context::GlobalContext(std::move(env), serverConfig)); + + std::unique_ptr config = + Config::Create(global_context->env(), std::string(kServiceConfig1)); + + std::shared_ptr service_context( + new context::ServiceContext(global_context, std::move(config))); + + std::unique_ptr request(new MockRequest(remote_ip, headers)); + + RequestContext context(service_context, std::move(request)); + + service_control::CheckRequestInfo info; + + context.FillCheckRequestInfo(&info); + + return info.client_ip; +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressNoOverrideTest) { + const char kServerConfigWithoutClientIPExperiment[] = + R"( + { + "experimental": { + "disable_log_status": false + } + } + )"; + + EXPECT_EQ("4.4.4.4", + ExtractClientIP(kServerConfigWithoutClientIPExperiment, "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressOverrideTest) { + const char kServerConfigWithClientIPExperimentSecondFromLast[] = + R"( + { + "client_ip_extraction_config": { + "client_ip_header": "X-Forwarded-For", + "client_ip_position": -2 + } + } + )"; + + EXPECT_EQ("2.2.2.2", + ExtractClientIP(kServerConfigWithClientIPExperimentSecondFromLast, + "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressOverrideLastTest) { + const char kServerConfigWithClientIPExperimentLast[] = + R"( + { + "client_ip_extraction_config": { + "client_ip_header": "X-Forwarded-For", + "client_ip_position": -1 + } + } + )"; + + EXPECT_EQ("3.3.3.3", + ExtractClientIP(kServerConfigWithClientIPExperimentLast, "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressOverrideOutOfIndexTest) { + const char kServerConfigWithClientIPExperimentOutOfIndex[] = + R"( + { + "client_ip_extraction_config": { + "client_ip_header": "X-Forwarded-For", + "client_ip_position": -5 + } + } + )"; + + EXPECT_EQ( + "4.4.4.4", + ExtractClientIP(kServerConfigWithClientIPExperimentOutOfIndex, "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressOverrideFirstIndexTest) { + const char kServerConfigWithClientIPExperimentFirst[] = + R"( + { + "client_ip_extraction_config": { + "client_ip_header": "X-Forwarded-For", + "client_ip_position": 0 + } + } + )"; + + EXPECT_EQ("1.1.1.1", + ExtractClientIP(kServerConfigWithClientIPExperimentFirst, "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +TEST_F(ClientIPExtractionTest, ClientIPAddressOverrideSecondIndexTest) { + const char kServerConfigWithClientIPExperimentSecond[] = + R"( + { + "client_ip_extraction_config": { + "client_ip_header": "X-Forwarded-For", + "client_ip_position": 1 + } + } + )"; + + EXPECT_EQ("2.2.2.2", ExtractClientIP( + kServerConfigWithClientIPExperimentSecond, "4.4.4.4", + {{"X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3"}, + {"apiKey", "test-api-key"}})); +} + +} // namespace context +} // namespace api_manager +} // namespace google diff --git a/src/api_manager/context/request_context.cc b/src/api_manager/context/request_context.cc index 78e9aa8b2..bb509bcbf 100644 --- a/src/api_manager/context/request_context.cc +++ b/src/api_manager/context/request_context.cc @@ -54,6 +54,9 @@ const char kDefaultApiKeyQueryName1[] = "key"; const char kDefaultApiKeyQueryName2[] = "api_key"; const char kDefaultApiKeyHeaderName[] = "x-api-key"; +// Delimiter of the IP addresses in the XFF header +const char kClientIPHeaderDelimeter = ','; + // Header for android package name, used for api key restriction check. const char kXAndroidPackage[] = "x-android-package"; @@ -73,6 +76,21 @@ std::string GenerateUUID() { return uuid_buf; } +inline void split(const std::string &s, char delim, + std::vector *elems) { + std::stringstream ss(s); + std::string item; + while (std::getline(ss, item, delim)) { + elems->push_back(item); + } +} + +inline const std::string trim(std::string &str) { + str.erase(0, str.find_first_not_of(' ')); // heading spaces + str.erase(str.find_last_not_of(' ') + 1); // tailing spaces + return str; +} + } // namespace using context::ServiceContext; @@ -198,7 +216,7 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->producer_project_id = service_context()->project_id(); info->referer = http_referer_; info->request_start_time = start_time_; - info->client_ip = request_->GetClientIP(); + info->client_ip = FindClientIPAddress(); info->client_host = request_->GetClientHost(); } @@ -259,7 +277,6 @@ void RequestContext::FillAllocateQuotaRequestInfo( service_control::QuotaRequestInfo *info) { FillOperationInfo(info); - info->client_ip = request_->GetClientIP(); info->method_name = this->method_call_.method_info->name(); info->metric_cost_vector = &this->method_call_.method_info->metric_cost_vector(); @@ -326,6 +343,35 @@ void RequestContext::FillReportRequestInfo( } } +const std::string RequestContext::FindClientIPAddress() { + auto serverConfig = service_context_->config()->server_config(); + std::string client_ip_header; + + if (serverConfig->has_client_ip_extraction_config() && + serverConfig->client_ip_extraction_config().client_ip_header().length() > + 0 && + request_->FindHeader( + serverConfig->client_ip_extraction_config().client_ip_header(), + &client_ip_header)) { + // split headers + std::vector secments; + split(client_ip_header, kClientIPHeaderDelimeter, &secments); + int client_ip_header_position = + serverConfig->client_ip_extraction_config().client_ip_position(); + + if (client_ip_header_position < 0) { + client_ip_header_position = secments.size() + client_ip_header_position; + } + + if (client_ip_header_position >= 0 && + client_ip_header_position < (int)secments.size()) { + return trim(secments[client_ip_header_position]); + } + } + + return request_->GetClientIP(); +} + void RequestContext::StartBackendSpanAndSetTraceContext() { backend_span_.reset(CreateSpan(cloud_trace_.get(), "Backend")); diff --git a/src/api_manager/context/request_context.h b/src/api_manager/context/request_context.h index c1e98d713..96f62afb3 100644 --- a/src/api_manager/context/request_context.h +++ b/src/api_manager/context/request_context.h @@ -154,6 +154,11 @@ class RequestContext { // Extracts api-key void ExtractApiKey(); + // Find client IP address based on the + // ServerConfig.client_ip_extraction_config. If it is not configured or + // doesn't match, returns request_->GetClientIP() + const std::string FindClientIPAddress(); + // The ApiManagerImpl object. std::shared_ptr service_context_; diff --git a/src/api_manager/proto/server_config.proto b/src/api_manager/proto/server_config.proto index 4d4b01c34..789d39310 100644 --- a/src/api_manager/proto/server_config.proto +++ b/src/api_manager/proto/server_config.proto @@ -47,6 +47,9 @@ message ServerConfig { // Common service API configuration ApiServiceConfig api_service_config = 13; + // Get client IP address from the static header with position configuration + ClientIPExtractionConfig client_ip_extraction_config = 14; + // The service config rollout strategy, [fixed|managed] // fixed: never change service config dynamically. // managed: follow service management service config rollout. @@ -228,6 +231,16 @@ message ApiServiceConfig { repeated string rewrite = 1; } +// Get client IP address from the header with position configuration +message ClientIPExtractionConfig { + // Defines HTTP header name where client IP will be extracted. + string client_ip_header = 1; + + // Defines the position of the client IP address. + // Same as the array index in many languages, such as Python. + int32 client_ip_position = 2; +} + message Experimental { // Disable timed printouts of ESP status to the error log. bool disable_log_status = 1; diff --git a/src/nginx/t/check_key_restriction.t b/src/nginx/t/check_key_restriction.t index 961a5d263..466348c57 100644 --- a/src/nginx/t/check_key_restriction.t +++ b/src/nginx/t/check_key_restriction.t @@ -56,6 +56,10 @@ service_control_config { flush_interval_ms: 1000 } } +client_ip_extraction_config { + client_ip_header: "X-Forwarded-For" + client_ip_position: -2 +} EOF # Save service name in the service configuration protocol buffer file. @@ -75,10 +79,7 @@ events { http { %%TEST_GLOBALS_HTTP%% server_tokens off; - set_real_ip_from 0.0.0.0/1; - set_real_ip_from 0::/1; - real_ip_header X-Forwarded-For; - real_ip_recursive on; + server { listen 127.0.0.1:${NginxPort}; server_name localhost; @@ -111,7 +112,7 @@ my $response = ApiManager::http($NginxPort,<<'EOF'); GET /shelves?key=this-is-an-api-key HTTP/1.0 Referer: http://google.com/bookstore/root Host: localhost -X-Forwarded-For: 10.20.30.40 +X-Forwarded-For: 1.1.1.1, 2.2.2.2, 3.3.3.3 X-Android-Package: com.goolge.cloud.esp X-Android-Cert: AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0 X-Ios-Bundle-Identifier: 5b40ad6af9a806305a0a56d7cb91b82a27c26909 @@ -159,7 +160,7 @@ my $check_request = decode_json(ServiceControl::convert_proto( $r->{body}, 'check_request', 'json' ) ); is( $check_request->{operation}->{labels}-> - {'servicecontrol.googleapis.com/caller_ip'}, "10.20.30.40", + {'servicecontrol.googleapis.com/caller_ip'}, "2.2.2.2", "servicecontrol.googleapis.com/caller_ip was overrode by ". "X-Forwarded-For header" ); is( $check_request->{operation}->{labels}->